[llvm] [AArch64] Implement latency costs for loads (PR #201567)
John Brawn via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 9 09:27:25 PDT 2026
https://github.com/john-brawn-arm updated https://github.com/llvm/llvm-project/pull/201567
>From 3440c4a2d90128089bb0ff5654d23834d6990b77 Mon Sep 17 00:00:00 2001
From: John Brawn <john.brawn at arm.com>
Date: Fri, 8 May 2026 12:41:48 +0100
Subject: [PATCH] [AArch64] Implement latency costs for loads
Adjust AArch64TTIImpl::getMemoryOpCost to query the scheduling model
to get the latency of loads, instead of using BasicTTIImpl which
returns an arbitrary cost of 4.
The test changes here are mostly CostModel tests being updated to the
correct latency, but some selectopt tests also need to be adjusted as
the default scheduling model is cortex-a510 which has low enough load
latency that the optimization isn't applied.
---
.../AArch64/AArch64TargetTransformInfo.cpp | 60 +++++-
.../AArch64/AArch64TargetTransformInfo.h | 6 +-
llvm/test/Analysis/CostModel/AArch64/cast.ll | 20 +-
llvm/test/Analysis/CostModel/AArch64/gep.ll | 72 +++----
.../CostModel/AArch64/insert-extract.ll | 184 +++++++++++++-----
.../CostModel/AArch64/masked_expand_load.ll | 72 +++----
.../Analysis/CostModel/AArch64/masked_ldst.ll | 36 ++--
.../CostModel/AArch64/mem-op-cost-model.ll | 86 ++++----
.../CostModel/AArch64/shuffle-load.ll | 164 ++++++++--------
llvm/test/Analysis/CostModel/AArch64/store.ll | 8 +-
.../Analysis/CostModel/AArch64/sve-cast.ll | 96 ++++-----
.../CostModel/AArch64/sve-intrinsics.ll | 26 +--
.../Analysis/CostModel/AArch64/sve-ldst.ll | 62 +++---
.../Analysis/CostModel/AArch64/vec3-ops.ll | 42 ++--
llvm/test/CodeGen/AArch64/selectopt-cast.ll | 4 +-
.../select-optimize-trailing-dbg-records.ll | 2 +-
16 files changed, 539 insertions(+), 401 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 730bec428a38e..e33311d7ead25 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5029,9 +5029,8 @@ InstructionCost AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty,
TTI::OperandValueInfo OpInfo,
const Instruction *I) const {
EVT VT = TLI->getValueType(DL, Ty, true);
- // Type legalization can't handle structs, and load latency isn't handled here
- if (VT == MVT::Other ||
- (Opcode == Instruction::Load && CostKind == TTI::TCK_Latency))
+ // Type legalization can't handle structs
+ if (VT == MVT::Other)
return BaseT::getMemoryOpCost(Opcode, Ty, Alignment, AddressSpace,
CostKind);
@@ -5055,8 +5054,59 @@ InstructionCost AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty,
if (CostKind == TTI::TCK_CodeSize || CostKind == TTI::TCK_SizeAndLatency)
return LT.first;
- if (CostKind != TTI::TCK_RecipThroughput)
- return 1;
+ if (CostKind == TTI::TCK_Latency) {
+ // Latency doesn't make much sense for stores, so just return 1
+ if (Opcode == Instruction::Store)
+ return 1;
+ // We expect the load to become LT.first loads of type LT.second. The
+ // latency will be the latency of the last load plus the time it gets to get
+ // there, which will be the amount of other loads before that (i.e. total
+ // loads - 1). We get the latency from the SchedModel, and assume that the
+ // loads become the variant with unsigned integer offset.
+ unsigned Inst = 0;
+ if (LT.second.isScalableVector() ||
+ ST->useSVEForFixedLengthVectors(LT.second)) {
+ Inst = AArch64::LDR_ZXI;
+ } else if (LT.second.isVector() || LT.second.isFloatingPoint()) {
+ switch (LT.second.getSizeInBits()) {
+ case 8:
+ Inst = AArch64::LDRBui;
+ break;
+ case 16:
+ Inst = AArch64::LDRHui;
+ break;
+ case 32:
+ Inst = AArch64::LDRSui;
+ break;
+ case 64:
+ Inst = AArch64::LDRDui;
+ break;
+ case 128:
+ Inst = AArch64::LDRQui;
+ break;
+ default:
+ llvm_unreachable("Unexpected float or vector type");
+ }
+ } else {
+ switch (LT.second.getSizeInBits()) {
+ case 8:
+ Inst = AArch64::LDRBBui;
+ break;
+ case 16:
+ Inst = AArch64::LDRHHui;
+ break;
+ case 32:
+ Inst = AArch64::LDRWui;
+ break;
+ case 64:
+ Inst = AArch64::LDRXui;
+ break;
+ default:
+ llvm_unreachable("Unexpected integer type");
+ }
+ }
+ return (LT.first - 1) + SchedModel.computeInstrLatency(Inst);
+ }
if (ST->isMisaligned128StoreSlow() && Opcode == Instruction::Store &&
LT.second.is128BitVector() && Alignment < Align(16)) {
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 200b4c9fba196..cc9f57e94a50e 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -21,6 +21,7 @@
#include "AArch64TargetMachine.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/CodeGen/BasicTTIImpl.h"
+#include "llvm/CodeGen/TargetSchedule.h"
#include "llvm/IR/FMF.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/Intrinsics.h"
@@ -48,6 +49,7 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
const AArch64Subtarget *ST;
const AArch64TargetLowering *TLI;
+ TargetSchedModel SchedModel;
static const FeatureBitset InlineInverseFeatures;
@@ -82,7 +84,9 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
public:
explicit AArch64TTIImpl(const AArch64TargetMachine *TM, const Function &F)
: BaseT(TM, F.getDataLayout()), ST(TM->getSubtargetImpl(F)),
- TLI(ST->getTargetLowering()) {}
+ TLI(ST->getTargetLowering()) {
+ SchedModel.init(ST);
+ }
bool areInlineCompatible(const Function *Caller,
const Function *Callee) const override;
diff --git a/llvm/test/Analysis/CostModel/AArch64/cast.ll b/llvm/test/Analysis/CostModel/AArch64/cast.ll
index 7f5be9f100442..9863f8d3e17e9 100644
--- a/llvm/test/Analysis/CostModel/AArch64/cast.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/cast.ll
@@ -859,16 +859,16 @@ define i32 @bitcasts() {
define i32 @load_extends() {
; CHECK-LABEL: 'load_extends'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadi8 = load i8, ptr undef, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadi16 = load i16, ptr undef, align 2
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadi32 = load i32, ptr undef, align 4
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:4 SizeLat:1 for: %loadv2i8 = load <2 x i8>, ptr undef, align 2
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %loadv4i8 = load <4 x i8>, ptr undef, align 4
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv8i8 = load <8 x i8>, ptr undef, align 8
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:4 SizeLat:1 for: %loadv2i16 = load <2 x i16>, ptr undef, align 4
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv4i16 = load <4 x i16>, ptr undef, align 8
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv2i32 = load <2 x i32>, ptr undef, align 8
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv4i32 = load <4 x i32>, ptr undef, align 16
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadi8 = load i8, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadi16 = load i16, ptr undef, align 2
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadi32 = load i32, ptr undef, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:2 SizeLat:1 for: %loadv2i8 = load <2 x i8>, ptr undef, align 2
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %loadv4i8 = load <4 x i8>, ptr undef, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadv8i8 = load <8 x i8>, ptr undef, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:2 SizeLat:1 for: %loadv2i16 = load <2 x i16>, ptr undef, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadv4i16 = load <4 x i16>, ptr undef, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadv2i32 = load <2 x i32>, ptr undef, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadv4i32 = load <4 x i32>, ptr undef, align 16
; CHECK-NEXT: Cost Model: Found costs of 0 for: %r0 = sext i8 %loadi8 to i16
; CHECK-NEXT: Cost Model: Found costs of 0 for: %r1 = zext i8 %loadi8 to i16
; CHECK-NEXT: Cost Model: Found costs of 0 for: %r2 = sext i8 %loadi8 to i32
diff --git a/llvm/test/Analysis/CostModel/AArch64/gep.ll b/llvm/test/Analysis/CostModel/AArch64/gep.ll
index a1239b246f2fb..a96d4ebea6a77 100644
--- a/llvm/test/Analysis/CostModel/AArch64/gep.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/gep.ll
@@ -7,7 +7,7 @@ target triple = "aarch64-linux-gnu"
define i8 @test1(ptr %p) {
; CHECK-LABEL: 'test1'
; CHECK-NEXT: Cost Model: Found costs of 0 for: %a = getelementptr inbounds i8, ptr %p, i32 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i8, ptr %a, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i8, ptr %a, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %v
;
%a = getelementptr inbounds i8, ptr %p, i32 1
@@ -18,7 +18,7 @@ define i8 @test1(ptr %p) {
define i16 @test2(ptr %p) {
; CHECK-LABEL: 'test2'
; CHECK-NEXT: Cost Model: Found costs of 0 for: %a = getelementptr inbounds i16, ptr %p, i32 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i16, ptr %a, align 2
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i16, ptr %a, align 2
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i16 %v
;
%a = getelementptr inbounds i16, ptr %p, i32 1
@@ -29,7 +29,7 @@ define i16 @test2(ptr %p) {
define i32 @test3(ptr %p) {
; CHECK-LABEL: 'test3'
; CHECK-NEXT: Cost Model: Found costs of 0 for: %a = getelementptr inbounds i32, ptr %p, i32 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i32, ptr %a, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i32, ptr %a, align 4
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %v
;
%a = getelementptr inbounds i32, ptr %p, i32 1
@@ -40,7 +40,7 @@ define i32 @test3(ptr %p) {
define i64 @test4(ptr %p) {
; CHECK-LABEL: 'test4'
; CHECK-NEXT: Cost Model: Found costs of 0 for: %a = getelementptr inbounds i64, ptr %p, i32 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i64, ptr %a, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i64, ptr %a, align 8
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i64 %v
;
%a = getelementptr inbounds i64, ptr %p, i32 1
@@ -51,7 +51,7 @@ define i64 @test4(ptr %p) {
define i8 @test5(ptr %p) {
; CHECK-LABEL: 'test5'
; CHECK-NEXT: Cost Model: Found costs of 0 for: %a = getelementptr inbounds i8, ptr %p, i32 1024
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i8, ptr %a, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i8, ptr %a, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %v
;
%a = getelementptr inbounds i8, ptr %p, i32 1024
@@ -62,7 +62,7 @@ define i8 @test5(ptr %p) {
define i16 @test6(ptr %p) {
; CHECK-LABEL: 'test6'
; CHECK-NEXT: Cost Model: Found costs of 0 for: %a = getelementptr inbounds i16, ptr %p, i32 1024
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i16, ptr %a, align 2
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i16, ptr %a, align 2
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i16 %v
;
%a = getelementptr inbounds i16, ptr %p, i32 1024
@@ -73,7 +73,7 @@ define i16 @test6(ptr %p) {
define i32 @test7(ptr %p) {
; CHECK-LABEL: 'test7'
; CHECK-NEXT: Cost Model: Found costs of 0 for: %a = getelementptr inbounds i32, ptr %p, i32 1024
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i32, ptr %a, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i32, ptr %a, align 4
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %v
;
%a = getelementptr inbounds i32, ptr %p, i32 1024
@@ -84,7 +84,7 @@ define i32 @test7(ptr %p) {
define i64 @test8(ptr %p) {
; CHECK-LABEL: 'test8'
; CHECK-NEXT: Cost Model: Found costs of 0 for: %a = getelementptr inbounds i64, ptr %p, i32 1024
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i64, ptr %a, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i64, ptr %a, align 8
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i64 %v
;
%a = getelementptr inbounds i64, ptr %p, i32 1024
@@ -95,7 +95,7 @@ define i64 @test8(ptr %p) {
define i8 @test9(ptr %p) {
; CHECK-LABEL: 'test9'
; CHECK-NEXT: Cost Model: Found costs of 1 for: %a = getelementptr inbounds i8, ptr %p, i32 4096
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i8, ptr %a, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i8, ptr %a, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %v
;
%a = getelementptr inbounds i8, ptr %p, i32 4096
@@ -106,7 +106,7 @@ define i8 @test9(ptr %p) {
define i16 @test10(ptr %p) {
; CHECK-LABEL: 'test10'
; CHECK-NEXT: Cost Model: Found costs of 1 for: %a = getelementptr inbounds i16, ptr %p, i32 4096
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i16, ptr %a, align 2
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i16, ptr %a, align 2
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i16 %v
;
%a = getelementptr inbounds i16, ptr %p, i32 4096
@@ -117,7 +117,7 @@ define i16 @test10(ptr %p) {
define i32 @test11(ptr %p) {
; CHECK-LABEL: 'test11'
; CHECK-NEXT: Cost Model: Found costs of 1 for: %a = getelementptr inbounds i32, ptr %p, i32 4096
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i32, ptr %a, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i32, ptr %a, align 4
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %v
;
%a = getelementptr inbounds i32, ptr %p, i32 4096
@@ -128,7 +128,7 @@ define i32 @test11(ptr %p) {
define i64 @test12(ptr %p) {
; CHECK-LABEL: 'test12'
; CHECK-NEXT: Cost Model: Found costs of 1 for: %a = getelementptr inbounds i64, ptr %p, i32 4096
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i64, ptr %a, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i64, ptr %a, align 8
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i64 %v
;
%a = getelementptr inbounds i64, ptr %p, i32 4096
@@ -139,7 +139,7 @@ define i64 @test12(ptr %p) {
define i8 @test13(ptr %p) {
; CHECK-LABEL: 'test13'
; CHECK-NEXT: Cost Model: Found costs of 0 for: %a = getelementptr inbounds i8, ptr %p, i32 -64
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i8, ptr %a, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i8, ptr %a, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %v
;
%a = getelementptr inbounds i8, ptr %p, i32 -64
@@ -150,7 +150,7 @@ define i8 @test13(ptr %p) {
define i16 @test14(ptr %p) {
; CHECK-LABEL: 'test14'
; CHECK-NEXT: Cost Model: Found costs of 0 for: %a = getelementptr inbounds i16, ptr %p, i32 -64
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i16, ptr %a, align 2
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i16, ptr %a, align 2
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i16 %v
;
%a = getelementptr inbounds i16, ptr %p, i32 -64
@@ -161,7 +161,7 @@ define i16 @test14(ptr %p) {
define i32 @test15(ptr %p) {
; CHECK-LABEL: 'test15'
; CHECK-NEXT: Cost Model: Found costs of 0 for: %a = getelementptr inbounds i32, ptr %p, i32 -64
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i32, ptr %a, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i32, ptr %a, align 4
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %v
;
%a = getelementptr inbounds i32, ptr %p, i32 -64
@@ -172,7 +172,7 @@ define i32 @test15(ptr %p) {
define i64 @test16(ptr %p) {
; CHECK-LABEL: 'test16'
; CHECK-NEXT: Cost Model: Found costs of 1 for: %a = getelementptr inbounds i64, ptr %p, i32 -64
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i64, ptr %a, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i64, ptr %a, align 8
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i64 %v
;
%a = getelementptr inbounds i64, ptr %p, i32 -64
@@ -183,7 +183,7 @@ define i64 @test16(ptr %p) {
define i8 @test17(ptr %p) {
; CHECK-LABEL: 'test17'
; CHECK-NEXT: Cost Model: Found costs of 1 for: %a = getelementptr inbounds i8, ptr %p, i32 -1024
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i8, ptr %a, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i8, ptr %a, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %v
;
%a = getelementptr inbounds i8, ptr %p, i32 -1024
@@ -194,7 +194,7 @@ define i8 @test17(ptr %p) {
define i16 @test18(ptr %p) {
; CHECK-LABEL: 'test18'
; CHECK-NEXT: Cost Model: Found costs of 1 for: %a = getelementptr inbounds i16, ptr %p, i32 -1024
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i16, ptr %a, align 2
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i16, ptr %a, align 2
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i16 %v
;
%a = getelementptr inbounds i16, ptr %p, i32 -1024
@@ -205,7 +205,7 @@ define i16 @test18(ptr %p) {
define i32 @test19(ptr %p) {
; CHECK-LABEL: 'test19'
; CHECK-NEXT: Cost Model: Found costs of 1 for: %a = getelementptr inbounds i32, ptr %p, i32 -1024
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i32, ptr %a, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i32, ptr %a, align 4
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %v
;
%a = getelementptr inbounds i32, ptr %p, i32 -1024
@@ -216,7 +216,7 @@ define i32 @test19(ptr %p) {
define i64 @test20(ptr %p) {
; CHECK-LABEL: 'test20'
; CHECK-NEXT: Cost Model: Found costs of 1 for: %a = getelementptr inbounds i64, ptr %p, i32 -1024
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i64, ptr %a, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i64, ptr %a, align 8
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i64 %v
;
%a = getelementptr inbounds i64, ptr %p, i32 -1024
@@ -227,7 +227,7 @@ define i64 @test20(ptr %p) {
define i8 @test21(ptr %p, i32 %i) {
; CHECK-LABEL: 'test21'
; CHECK-NEXT: Cost Model: Found costs of 0 for: %a = getelementptr inbounds i8, ptr %p, i32 %i
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i8, ptr %a, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i8, ptr %a, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %v
;
%a = getelementptr inbounds i8, ptr %p, i32 %i
@@ -238,7 +238,7 @@ define i8 @test21(ptr %p, i32 %i) {
define i16 @test22(ptr %p, i32 %i) {
; CHECK-LABEL: 'test22'
; CHECK-NEXT: Cost Model: Found costs of 0 for: %a = getelementptr inbounds i16, ptr %p, i32 %i
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i16, ptr %a, align 2
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i16, ptr %a, align 2
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i16 %v
;
%a = getelementptr inbounds i16, ptr %p, i32 %i
@@ -249,7 +249,7 @@ define i16 @test22(ptr %p, i32 %i) {
define i32 @test23(ptr %p, i32 %i) {
; CHECK-LABEL: 'test23'
; CHECK-NEXT: Cost Model: Found costs of 0 for: %a = getelementptr inbounds i32, ptr %p, i32 %i
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i32, ptr %a, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i32, ptr %a, align 4
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %v
;
%a = getelementptr inbounds i32, ptr %p, i32 %i
@@ -260,7 +260,7 @@ define i32 @test23(ptr %p, i32 %i) {
define i64 @test24(ptr %p, i32 %i) {
; CHECK-LABEL: 'test24'
; CHECK-NEXT: Cost Model: Found costs of 0 for: %a = getelementptr inbounds i64, ptr %p, i32 %i
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i64, ptr %a, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i64, ptr %a, align 8
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i64 %v
;
%a = getelementptr inbounds i64, ptr %p, i32 %i
@@ -271,7 +271,7 @@ define i64 @test24(ptr %p, i32 %i) {
define i8 @test25(ptr %p) {
; CHECK-LABEL: 'test25'
; CHECK-NEXT: Cost Model: Found costs of 0 for: %a = getelementptr inbounds i8, ptr %p, i32 -128
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i8, ptr %a, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i8, ptr %a, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %v
;
%a = getelementptr inbounds i8, ptr %p, i32 -128
@@ -282,7 +282,7 @@ define i8 @test25(ptr %p) {
define i16 @test26(ptr %p) {
; CHECK-LABEL: 'test26'
; CHECK-NEXT: Cost Model: Found costs of 0 for: %a = getelementptr inbounds i16, ptr %p, i32 -128
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i16, ptr %a, align 2
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i16, ptr %a, align 2
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i16 %v
;
%a = getelementptr inbounds i16, ptr %p, i32 -128
@@ -293,7 +293,7 @@ define i16 @test26(ptr %p) {
define i32 @test27(ptr %p) {
; CHECK-LABEL: 'test27'
; CHECK-NEXT: Cost Model: Found costs of 1 for: %a = getelementptr inbounds i32, ptr %p, i32 -128
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i32, ptr %a, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i32, ptr %a, align 4
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %v
;
%a = getelementptr inbounds i32, ptr %p, i32 -128
@@ -304,7 +304,7 @@ define i32 @test27(ptr %p) {
define i64 @test28(ptr %p) {
; CHECK-LABEL: 'test28'
; CHECK-NEXT: Cost Model: Found costs of 1 for: %a = getelementptr inbounds i64, ptr %p, i32 -128
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i64, ptr %a, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i64, ptr %a, align 8
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i64 %v
;
%a = getelementptr inbounds i64, ptr %p, i32 -128
@@ -315,7 +315,7 @@ define i64 @test28(ptr %p) {
define i8 @test29(ptr %p) {
; CHECK-LABEL: 'test29'
; CHECK-NEXT: Cost Model: Found costs of 0 for: %a = getelementptr inbounds i8, ptr %p, i32 -256
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i8, ptr %a, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i8, ptr %a, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %v
;
%a = getelementptr inbounds i8, ptr %p, i32 -256
@@ -326,7 +326,7 @@ define i8 @test29(ptr %p) {
define i16 @test30(ptr %p) {
; CHECK-LABEL: 'test30'
; CHECK-NEXT: Cost Model: Found costs of 1 for: %a = getelementptr inbounds i16, ptr %p, i32 -256
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i16, ptr %a, align 2
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i16, ptr %a, align 2
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i16 %v
;
%a = getelementptr inbounds i16, ptr %p, i32 -256
@@ -337,7 +337,7 @@ define i16 @test30(ptr %p) {
define i32 @test31(ptr %p) {
; CHECK-LABEL: 'test31'
; CHECK-NEXT: Cost Model: Found costs of 1 for: %a = getelementptr inbounds i32, ptr %p, i32 -256
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i32, ptr %a, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i32, ptr %a, align 4
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %v
;
%a = getelementptr inbounds i32, ptr %p, i32 -256
@@ -348,7 +348,7 @@ define i32 @test31(ptr %p) {
define i64 @test32(ptr %p) {
; CHECK-LABEL: 'test32'
; CHECK-NEXT: Cost Model: Found costs of 1 for: %a = getelementptr inbounds i64, ptr %p, i32 -256
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i64, ptr %a, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i64, ptr %a, align 8
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i64 %v
;
%a = getelementptr inbounds i64, ptr %p, i32 -256
@@ -359,7 +359,7 @@ define i64 @test32(ptr %p) {
define i8 @test33(ptr %p) {
; CHECK-LABEL: 'test33'
; CHECK-NEXT: Cost Model: Found costs of 1 for: %a = getelementptr inbounds i8, ptr %p, i32 -512
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i8, ptr %a, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i8, ptr %a, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %v
;
%a = getelementptr inbounds i8, ptr %p, i32 -512
@@ -370,7 +370,7 @@ define i8 @test33(ptr %p) {
define i16 @test34(ptr %p) {
; CHECK-LABEL: 'test34'
; CHECK-NEXT: Cost Model: Found costs of 1 for: %a = getelementptr inbounds i16, ptr %p, i32 -512
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i16, ptr %a, align 2
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i16, ptr %a, align 2
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i16 %v
;
%a = getelementptr inbounds i16, ptr %p, i32 -512
@@ -381,7 +381,7 @@ define i16 @test34(ptr %p) {
define i32 @test35(ptr %p) {
; CHECK-LABEL: 'test35'
; CHECK-NEXT: Cost Model: Found costs of 1 for: %a = getelementptr inbounds i32, ptr %p, i32 -512
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i32, ptr %a, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i32, ptr %a, align 4
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %v
;
%a = getelementptr inbounds i32, ptr %p, i32 -512
@@ -392,7 +392,7 @@ define i32 @test35(ptr %p) {
define i64 @test36(ptr %p) {
; CHECK-LABEL: 'test36'
; CHECK-NEXT: Cost Model: Found costs of 1 for: %a = getelementptr inbounds i64, ptr %p, i32 -512
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i64, ptr %a, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v = load i64, ptr %a, align 8
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i64 %v
;
%a = getelementptr inbounds i64, ptr %p, i32 -512
diff --git a/llvm/test/Analysis/CostModel/AArch64/insert-extract.ll b/llvm/test/Analysis/CostModel/AArch64/insert-extract.ll
index 8cd5b54c2a4d7..2476d31ea100f 100644
--- a/llvm/test/Analysis/CostModel/AArch64/insert-extract.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/insert-extract.ll
@@ -1,17 +1,17 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output | FileCheck --check-prefixes=CHECK,GENERIC %s
-; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=neoverse-n1 | FileCheck --check-prefixes=CHECK,GENERIC %s
-; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=neoverse-n2 | FileCheck --check-prefixes=CHECK,GENERIC %s
-; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=neoverse-v1 | FileCheck --check-prefixes=CHECK,GENERIC %s
-; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=neoverse-v2 | FileCheck --check-prefixes=CHECK,GENERIC %s
-; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=kryo | FileCheck --check-prefixes=CHECK,GENERIC %s
-; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=apple-m1 | FileCheck --check-prefixes=CHECK,FAST-LD1 %s
-; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=apple-m2 | FileCheck --check-prefixes=CHECK,FAST-LD1 %s
-; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=apple-m3 | FileCheck --check-prefixes=CHECK,FAST-LD1 %s
-; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=apple-m4 | FileCheck --check-prefixes=CHECK,FAST-LD1 %s
-; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=apple-m5 | FileCheck --check-prefixes=CHECK,FAST-LD1 %s
-; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=apple-latest | FileCheck --check-prefixes=CHECK,FAST-LD1 %s
-; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mattr=+fast-ld1-single | FileCheck --check-prefixes=CHECK,FAST-LD1 %s
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output | FileCheck --check-prefixes=CHECK,SLOW-LD1-LAT2 %s
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=neoverse-n1 | FileCheck --check-prefixes=CHECK,SLOW-LD1-LAT4 %s
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=neoverse-n2 | FileCheck --check-prefixes=CHECK,SLOW-LD1-LAT4 %s
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=neoverse-v1 | FileCheck --check-prefixes=CHECK,SLOW-LD1-LAT4 %s
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=neoverse-v2 | FileCheck --check-prefixes=CHECK,SLOW-LD1-LAT4 %s
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=kryo | FileCheck --check-prefixes=CHECK,SLOW-LD1-LAT3 %s
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=apple-m1 | FileCheck --check-prefixes=CHECK,FAST-LD1-LAT4 %s
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=apple-m2 | FileCheck --check-prefixes=CHECK,FAST-LD1-LAT4 %s
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=apple-m3 | FileCheck --check-prefixes=CHECK,FAST-LD1-LAT4 %s
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=apple-m4 | FileCheck --check-prefixes=CHECK,FAST-LD1-LAT4 %s
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=apple-m5 | FileCheck --check-prefixes=CHECK,FAST-LD1-LAT4 %s
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=apple-latest | FileCheck --check-prefixes=CHECK,FAST-LD1-LAT4 %s
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mattr=+fast-ld1-single | FileCheck --check-prefixes=CHECK,FAST-LD1-LAT2 %s
target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-gnu"
@@ -91,15 +91,30 @@ define void @vectorInstrCost() {
;; LD1: Load one single-element structure to one lane of one register.
define <8 x i8> @LD1_B(<8 x i8> %vec, ptr noundef %i) {
-; GENERIC-LABEL: 'LD1_B'
-; GENERIC-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v1 = load i8, ptr %i, align 1
-; GENERIC-NEXT: Cost Model: Found costs of RThru:3 CodeSize:0 Lat:3 SizeLat:3 for: %v2 = insertelement <8 x i8> %vec, i8 %v1, i32 1
-; GENERIC-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %v2
+; SLOW-LD1-LAT2-LABEL: 'LD1_B'
+; SLOW-LD1-LAT2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v1 = load i8, ptr %i, align 1
+; SLOW-LD1-LAT2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:0 Lat:3 SizeLat:3 for: %v2 = insertelement <8 x i8> %vec, i8 %v1, i32 1
+; SLOW-LD1-LAT2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %v2
;
-; FAST-LD1-LABEL: 'LD1_B'
-; FAST-LD1-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v1 = load i8, ptr %i, align 1
-; FAST-LD1-NEXT: Cost Model: Found costs of 0 for: %v2 = insertelement <8 x i8> %vec, i8 %v1, i32 1
-; FAST-LD1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %v2
+; SLOW-LD1-LAT4-LABEL: 'LD1_B'
+; SLOW-LD1-LAT4-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v1 = load i8, ptr %i, align 1
+; SLOW-LD1-LAT4-NEXT: Cost Model: Found costs of RThru:3 CodeSize:0 Lat:3 SizeLat:3 for: %v2 = insertelement <8 x i8> %vec, i8 %v1, i32 1
+; SLOW-LD1-LAT4-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %v2
+;
+; SLOW-LD1-LAT3-LABEL: 'LD1_B'
+; SLOW-LD1-LAT3-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %v1 = load i8, ptr %i, align 1
+; SLOW-LD1-LAT3-NEXT: Cost Model: Found costs of RThru:3 CodeSize:0 Lat:3 SizeLat:3 for: %v2 = insertelement <8 x i8> %vec, i8 %v1, i32 1
+; SLOW-LD1-LAT3-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %v2
+;
+; FAST-LD1-LAT4-LABEL: 'LD1_B'
+; FAST-LD1-LAT4-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v1 = load i8, ptr %i, align 1
+; FAST-LD1-LAT4-NEXT: Cost Model: Found costs of 0 for: %v2 = insertelement <8 x i8> %vec, i8 %v1, i32 1
+; FAST-LD1-LAT4-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %v2
+;
+; FAST-LD1-LAT2-LABEL: 'LD1_B'
+; FAST-LD1-LAT2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v1 = load i8, ptr %i, align 1
+; FAST-LD1-LAT2-NEXT: Cost Model: Found costs of 0 for: %v2 = insertelement <8 x i8> %vec, i8 %v1, i32 1
+; FAST-LD1-LAT2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %v2
;
entry:
%v1 = load i8, ptr %i, align 1
@@ -108,15 +123,30 @@ entry:
}
define <4 x i16> @LD1_H(<4 x i16> %vec, ptr noundef %i) {
-; GENERIC-LABEL: 'LD1_H'
-; GENERIC-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v1 = load i16, ptr %i, align 2
-; GENERIC-NEXT: Cost Model: Found costs of RThru:3 CodeSize:0 Lat:3 SizeLat:3 for: %v2 = insertelement <4 x i16> %vec, i16 %v1, i32 2
-; GENERIC-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i16> %v2
+; SLOW-LD1-LAT2-LABEL: 'LD1_H'
+; SLOW-LD1-LAT2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v1 = load i16, ptr %i, align 2
+; SLOW-LD1-LAT2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:0 Lat:3 SizeLat:3 for: %v2 = insertelement <4 x i16> %vec, i16 %v1, i32 2
+; SLOW-LD1-LAT2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i16> %v2
+;
+; SLOW-LD1-LAT4-LABEL: 'LD1_H'
+; SLOW-LD1-LAT4-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v1 = load i16, ptr %i, align 2
+; SLOW-LD1-LAT4-NEXT: Cost Model: Found costs of RThru:3 CodeSize:0 Lat:3 SizeLat:3 for: %v2 = insertelement <4 x i16> %vec, i16 %v1, i32 2
+; SLOW-LD1-LAT4-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i16> %v2
;
-; FAST-LD1-LABEL: 'LD1_H'
-; FAST-LD1-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v1 = load i16, ptr %i, align 2
-; FAST-LD1-NEXT: Cost Model: Found costs of 0 for: %v2 = insertelement <4 x i16> %vec, i16 %v1, i32 2
-; FAST-LD1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i16> %v2
+; SLOW-LD1-LAT3-LABEL: 'LD1_H'
+; SLOW-LD1-LAT3-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %v1 = load i16, ptr %i, align 2
+; SLOW-LD1-LAT3-NEXT: Cost Model: Found costs of RThru:3 CodeSize:0 Lat:3 SizeLat:3 for: %v2 = insertelement <4 x i16> %vec, i16 %v1, i32 2
+; SLOW-LD1-LAT3-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i16> %v2
+;
+; FAST-LD1-LAT4-LABEL: 'LD1_H'
+; FAST-LD1-LAT4-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v1 = load i16, ptr %i, align 2
+; FAST-LD1-LAT4-NEXT: Cost Model: Found costs of 0 for: %v2 = insertelement <4 x i16> %vec, i16 %v1, i32 2
+; FAST-LD1-LAT4-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i16> %v2
+;
+; FAST-LD1-LAT2-LABEL: 'LD1_H'
+; FAST-LD1-LAT2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v1 = load i16, ptr %i, align 2
+; FAST-LD1-LAT2-NEXT: Cost Model: Found costs of 0 for: %v2 = insertelement <4 x i16> %vec, i16 %v1, i32 2
+; FAST-LD1-LAT2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i16> %v2
;
entry:
%v1 = load i16, ptr %i, align 2
@@ -125,15 +155,30 @@ entry:
}
define <4 x i32> @LD1_W(<4 x i32> %vec, ptr noundef %i) {
-; GENERIC-LABEL: 'LD1_W'
-; GENERIC-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v1 = load i32, ptr %i, align 4
-; GENERIC-NEXT: Cost Model: Found costs of RThru:3 CodeSize:0 Lat:3 SizeLat:3 for: %v2 = insertelement <4 x i32> %vec, i32 %v1, i32 3
-; GENERIC-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %v2
+; SLOW-LD1-LAT2-LABEL: 'LD1_W'
+; SLOW-LD1-LAT2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v1 = load i32, ptr %i, align 4
+; SLOW-LD1-LAT2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:0 Lat:3 SizeLat:3 for: %v2 = insertelement <4 x i32> %vec, i32 %v1, i32 3
+; SLOW-LD1-LAT2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %v2
+;
+; SLOW-LD1-LAT4-LABEL: 'LD1_W'
+; SLOW-LD1-LAT4-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v1 = load i32, ptr %i, align 4
+; SLOW-LD1-LAT4-NEXT: Cost Model: Found costs of RThru:3 CodeSize:0 Lat:3 SizeLat:3 for: %v2 = insertelement <4 x i32> %vec, i32 %v1, i32 3
+; SLOW-LD1-LAT4-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %v2
+;
+; SLOW-LD1-LAT3-LABEL: 'LD1_W'
+; SLOW-LD1-LAT3-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %v1 = load i32, ptr %i, align 4
+; SLOW-LD1-LAT3-NEXT: Cost Model: Found costs of RThru:3 CodeSize:0 Lat:3 SizeLat:3 for: %v2 = insertelement <4 x i32> %vec, i32 %v1, i32 3
+; SLOW-LD1-LAT3-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %v2
;
-; FAST-LD1-LABEL: 'LD1_W'
-; FAST-LD1-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v1 = load i32, ptr %i, align 4
-; FAST-LD1-NEXT: Cost Model: Found costs of 0 for: %v2 = insertelement <4 x i32> %vec, i32 %v1, i32 3
-; FAST-LD1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %v2
+; FAST-LD1-LAT4-LABEL: 'LD1_W'
+; FAST-LD1-LAT4-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v1 = load i32, ptr %i, align 4
+; FAST-LD1-LAT4-NEXT: Cost Model: Found costs of 0 for: %v2 = insertelement <4 x i32> %vec, i32 %v1, i32 3
+; FAST-LD1-LAT4-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %v2
+;
+; FAST-LD1-LAT2-LABEL: 'LD1_W'
+; FAST-LD1-LAT2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v1 = load i32, ptr %i, align 4
+; FAST-LD1-LAT2-NEXT: Cost Model: Found costs of 0 for: %v2 = insertelement <4 x i32> %vec, i32 %v1, i32 3
+; FAST-LD1-LAT2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %v2
;
entry:
%v1 = load i32, ptr %i, align 4
@@ -142,15 +187,30 @@ entry:
}
define <2 x i64> @LD1_X(<2 x i64> %vec, ptr noundef %i) {
-; GENERIC-LABEL: 'LD1_X'
-; GENERIC-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v1 = load i64, ptr %i, align 8
-; GENERIC-NEXT: Cost Model: Found costs of RThru:3 CodeSize:0 Lat:3 SizeLat:3 for: %v2 = insertelement <2 x i64> %vec, i64 %v1, i32 0
-; GENERIC-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i64> %v2
+; SLOW-LD1-LAT2-LABEL: 'LD1_X'
+; SLOW-LD1-LAT2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v1 = load i64, ptr %i, align 8
+; SLOW-LD1-LAT2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:0 Lat:3 SizeLat:3 for: %v2 = insertelement <2 x i64> %vec, i64 %v1, i32 0
+; SLOW-LD1-LAT2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i64> %v2
+;
+; SLOW-LD1-LAT4-LABEL: 'LD1_X'
+; SLOW-LD1-LAT4-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v1 = load i64, ptr %i, align 8
+; SLOW-LD1-LAT4-NEXT: Cost Model: Found costs of RThru:3 CodeSize:0 Lat:3 SizeLat:3 for: %v2 = insertelement <2 x i64> %vec, i64 %v1, i32 0
+; SLOW-LD1-LAT4-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i64> %v2
+;
+; SLOW-LD1-LAT3-LABEL: 'LD1_X'
+; SLOW-LD1-LAT3-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %v1 = load i64, ptr %i, align 8
+; SLOW-LD1-LAT3-NEXT: Cost Model: Found costs of RThru:3 CodeSize:0 Lat:3 SizeLat:3 for: %v2 = insertelement <2 x i64> %vec, i64 %v1, i32 0
+; SLOW-LD1-LAT3-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i64> %v2
+;
+; FAST-LD1-LAT4-LABEL: 'LD1_X'
+; FAST-LD1-LAT4-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v1 = load i64, ptr %i, align 8
+; FAST-LD1-LAT4-NEXT: Cost Model: Found costs of 0 for: %v2 = insertelement <2 x i64> %vec, i64 %v1, i32 0
+; FAST-LD1-LAT4-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i64> %v2
;
-; FAST-LD1-LABEL: 'LD1_X'
-; FAST-LD1-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v1 = load i64, ptr %i, align 8
-; FAST-LD1-NEXT: Cost Model: Found costs of 0 for: %v2 = insertelement <2 x i64> %vec, i64 %v1, i32 0
-; FAST-LD1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i64> %v2
+; FAST-LD1-LAT2-LABEL: 'LD1_X'
+; FAST-LD1-LAT2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v1 = load i64, ptr %i, align 8
+; FAST-LD1-LAT2-NEXT: Cost Model: Found costs of 0 for: %v2 = insertelement <2 x i64> %vec, i64 %v1, i32 0
+; FAST-LD1-LAT2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i64> %v2
;
entry:
%v1 = load i64, ptr %i, align 8
@@ -161,11 +221,35 @@ entry:
;; Multi-use tests: Check that Load context is only applied when load has one use
define <8 x i8> @LD1_multi_use_load(<8 x i8> %vec, ptr noundef %i, ptr noundef %out) {
-; CHECK-LABEL: 'LD1_multi_use_load'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v1 = load i8, ptr %i, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v2 = insertelement <8 x i8> %vec, i8 %v1, i32 1
-; CHECK-NEXT: Cost Model: Found costs of 1 for: store i8 %v1, ptr %out, align 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %v2
+; SLOW-LD1-LAT2-LABEL: 'LD1_multi_use_load'
+; SLOW-LD1-LAT2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v1 = load i8, ptr %i, align 1
+; SLOW-LD1-LAT2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v2 = insertelement <8 x i8> %vec, i8 %v1, i32 1
+; SLOW-LD1-LAT2-NEXT: Cost Model: Found costs of 1 for: store i8 %v1, ptr %out, align 1
+; SLOW-LD1-LAT2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %v2
+;
+; SLOW-LD1-LAT4-LABEL: 'LD1_multi_use_load'
+; SLOW-LD1-LAT4-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v1 = load i8, ptr %i, align 1
+; SLOW-LD1-LAT4-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v2 = insertelement <8 x i8> %vec, i8 %v1, i32 1
+; SLOW-LD1-LAT4-NEXT: Cost Model: Found costs of 1 for: store i8 %v1, ptr %out, align 1
+; SLOW-LD1-LAT4-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %v2
+;
+; SLOW-LD1-LAT3-LABEL: 'LD1_multi_use_load'
+; SLOW-LD1-LAT3-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %v1 = load i8, ptr %i, align 1
+; SLOW-LD1-LAT3-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v2 = insertelement <8 x i8> %vec, i8 %v1, i32 1
+; SLOW-LD1-LAT3-NEXT: Cost Model: Found costs of 1 for: store i8 %v1, ptr %out, align 1
+; SLOW-LD1-LAT3-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %v2
+;
+; FAST-LD1-LAT4-LABEL: 'LD1_multi_use_load'
+; FAST-LD1-LAT4-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v1 = load i8, ptr %i, align 1
+; FAST-LD1-LAT4-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v2 = insertelement <8 x i8> %vec, i8 %v1, i32 1
+; FAST-LD1-LAT4-NEXT: Cost Model: Found costs of 1 for: store i8 %v1, ptr %out, align 1
+; FAST-LD1-LAT4-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %v2
+;
+; FAST-LD1-LAT2-LABEL: 'LD1_multi_use_load'
+; FAST-LD1-LAT2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v1 = load i8, ptr %i, align 1
+; FAST-LD1-LAT2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v2 = insertelement <8 x i8> %vec, i8 %v1, i32 1
+; FAST-LD1-LAT2-NEXT: Cost Model: Found costs of 1 for: store i8 %v1, ptr %out, align 1
+; FAST-LD1-LAT2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %v2
;
entry:
%v1 = load i8, ptr %i, align 1
diff --git a/llvm/test/Analysis/CostModel/AArch64/masked_expand_load.ll b/llvm/test/Analysis/CostModel/AArch64/masked_expand_load.ll
index b33f73a1fef58..91f18af74bbaf 100644
--- a/llvm/test/Analysis/CostModel/AArch64/masked_expand_load.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/masked_expand_load.ll
@@ -8,24 +8,24 @@
define void @fixed() {
; SVE2p2-SME2p2-NON-STREAMING-LABEL: 'fixed'
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:10 CodeSize:6 Lat:16 SizeLat:10 for: %v2i8 = call <2 x i8> @llvm.masked.expandload.v2i8(ptr poison, <2 x i1> poison, <2 x i8> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:32 SizeLat:20 for: %v4i8 = call <4 x i8> @llvm.masked.expandload.v4i8(ptr poison, <4 x i1> poison, <4 x i8> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:40 CodeSize:24 Lat:64 SizeLat:40 for: %v8i8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr poison, <8 x i1> poison, <8 x i8> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:80 CodeSize:48 Lat:128 SizeLat:80 for: %v16i8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr poison, <16 x i1> poison, <16 x i8> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:10 CodeSize:6 Lat:16 SizeLat:10 for: %v2i16 = call <2 x i16> @llvm.masked.expandload.v2i16(ptr poison, <2 x i1> poison, <2 x i16> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:32 SizeLat:20 for: %v4i16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr poison, <4 x i1> poison, <4 x i16> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:40 CodeSize:24 Lat:64 SizeLat:40 for: %v8i16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr poison, <8 x i1> poison, <8 x i16> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:10 CodeSize:6 Lat:16 SizeLat:10 for: %v2i32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr poison, <2 x i1> poison, <2 x i32> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:32 SizeLat:20 for: %v4i32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr poison, <4 x i1> poison, <4 x i32> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:10 CodeSize:6 Lat:16 SizeLat:10 for: %v2i64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr poison, <2 x i1> poison, <2 x i64> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:8 CodeSize:5 Lat:14 SizeLat:8 for: %v2f16 = call <2 x half> @llvm.masked.expandload.v2f16(ptr poison, <2 x i1> poison, <2 x half> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:18 CodeSize:11 Lat:30 SizeLat:18 for: %v4f16 = call <4 x half> @llvm.masked.expandload.v4f16(ptr poison, <4 x i1> poison, <4 x half> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:38 CodeSize:23 Lat:62 SizeLat:38 for: %v8f16 = call <8 x half> @llvm.masked.expandload.v8f16(ptr poison, <8 x i1> poison, <8 x half> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:8 CodeSize:5 Lat:14 SizeLat:8 for: %v2f32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr poison, <2 x i1> poison, <2 x float> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:18 CodeSize:11 Lat:30 SizeLat:18 for: %v4f32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr poison, <4 x i1> poison, <4 x float> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:8 CodeSize:5 Lat:14 SizeLat:8 for: %v2f64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr poison, <2 x i1> poison, <2 x double> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:32 SizeLat:20 for: %v4i64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr poison, <4 x i1> poison, <4 x i64> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:152 CodeSize:92 Lat:248 SizeLat:152 for: %v32f16 = call <32 x half> @llvm.masked.expandload.v32f16(ptr poison, <32 x i1> poison, <32 x half> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:10 CodeSize:6 Lat:12 SizeLat:10 for: %v2i8 = call <2 x i8> @llvm.masked.expandload.v2i8(ptr poison, <2 x i1> poison, <2 x i8> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:24 SizeLat:20 for: %v4i8 = call <4 x i8> @llvm.masked.expandload.v4i8(ptr poison, <4 x i1> poison, <4 x i8> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:40 CodeSize:24 Lat:48 SizeLat:40 for: %v8i8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr poison, <8 x i1> poison, <8 x i8> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:80 CodeSize:48 Lat:96 SizeLat:80 for: %v16i8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr poison, <16 x i1> poison, <16 x i8> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:10 CodeSize:6 Lat:12 SizeLat:10 for: %v2i16 = call <2 x i16> @llvm.masked.expandload.v2i16(ptr poison, <2 x i1> poison, <2 x i16> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:24 SizeLat:20 for: %v4i16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr poison, <4 x i1> poison, <4 x i16> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:40 CodeSize:24 Lat:48 SizeLat:40 for: %v8i16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr poison, <8 x i1> poison, <8 x i16> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:10 CodeSize:6 Lat:12 SizeLat:10 for: %v2i32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr poison, <2 x i1> poison, <2 x i32> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:24 SizeLat:20 for: %v4i32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr poison, <4 x i1> poison, <4 x i32> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:10 CodeSize:6 Lat:12 SizeLat:10 for: %v2i64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr poison, <2 x i1> poison, <2 x i64> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:8 CodeSize:5 Lat:10 SizeLat:8 for: %v2f16 = call <2 x half> @llvm.masked.expandload.v2f16(ptr poison, <2 x i1> poison, <2 x half> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:18 CodeSize:11 Lat:22 SizeLat:18 for: %v4f16 = call <4 x half> @llvm.masked.expandload.v4f16(ptr poison, <4 x i1> poison, <4 x half> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:38 CodeSize:23 Lat:46 SizeLat:38 for: %v8f16 = call <8 x half> @llvm.masked.expandload.v8f16(ptr poison, <8 x i1> poison, <8 x half> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:8 CodeSize:5 Lat:10 SizeLat:8 for: %v2f32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr poison, <2 x i1> poison, <2 x float> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:18 CodeSize:11 Lat:22 SizeLat:18 for: %v4f32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr poison, <4 x i1> poison, <4 x float> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:8 CodeSize:5 Lat:10 SizeLat:8 for: %v2f64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr poison, <2 x i1> poison, <2 x double> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:24 SizeLat:20 for: %v4i64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr poison, <4 x i1> poison, <4 x i64> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:152 CodeSize:92 Lat:184 SizeLat:152 for: %v32f16 = call <32 x half> @llvm.masked.expandload.v32f16(ptr poison, <32 x i1> poison, <32 x half> poison)
; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; SVE2p2-SME2p2-STREAMING-LABEL: 'fixed'
@@ -50,24 +50,24 @@ define void @fixed() {
; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; SVE2p2-SME2p2-LABEL: 'fixed'
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:10 CodeSize:6 Lat:16 SizeLat:10 for: %v2i8 = call <2 x i8> @llvm.masked.expandload.v2i8(ptr poison, <2 x i1> poison, <2 x i8> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:32 SizeLat:20 for: %v4i8 = call <4 x i8> @llvm.masked.expandload.v4i8(ptr poison, <4 x i1> poison, <4 x i8> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:40 CodeSize:24 Lat:64 SizeLat:40 for: %v8i8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr poison, <8 x i1> poison, <8 x i8> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:80 CodeSize:48 Lat:128 SizeLat:80 for: %v16i8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr poison, <16 x i1> poison, <16 x i8> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:10 CodeSize:6 Lat:16 SizeLat:10 for: %v2i16 = call <2 x i16> @llvm.masked.expandload.v2i16(ptr poison, <2 x i1> poison, <2 x i16> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:32 SizeLat:20 for: %v4i16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr poison, <4 x i1> poison, <4 x i16> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:40 CodeSize:24 Lat:64 SizeLat:40 for: %v8i16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr poison, <8 x i1> poison, <8 x i16> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:10 CodeSize:6 Lat:16 SizeLat:10 for: %v2i32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr poison, <2 x i1> poison, <2 x i32> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:32 SizeLat:20 for: %v4i32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr poison, <4 x i1> poison, <4 x i32> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:10 CodeSize:6 Lat:16 SizeLat:10 for: %v2i64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr poison, <2 x i1> poison, <2 x i64> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:8 CodeSize:5 Lat:14 SizeLat:8 for: %v2f16 = call <2 x half> @llvm.masked.expandload.v2f16(ptr poison, <2 x i1> poison, <2 x half> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:18 CodeSize:11 Lat:30 SizeLat:18 for: %v4f16 = call <4 x half> @llvm.masked.expandload.v4f16(ptr poison, <4 x i1> poison, <4 x half> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:38 CodeSize:23 Lat:62 SizeLat:38 for: %v8f16 = call <8 x half> @llvm.masked.expandload.v8f16(ptr poison, <8 x i1> poison, <8 x half> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:8 CodeSize:5 Lat:14 SizeLat:8 for: %v2f32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr poison, <2 x i1> poison, <2 x float> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:18 CodeSize:11 Lat:30 SizeLat:18 for: %v4f32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr poison, <4 x i1> poison, <4 x float> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:8 CodeSize:5 Lat:14 SizeLat:8 for: %v2f64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr poison, <2 x i1> poison, <2 x double> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:32 SizeLat:20 for: %v4i64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr poison, <4 x i1> poison, <4 x i64> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:152 CodeSize:92 Lat:248 SizeLat:152 for: %v32f16 = call <32 x half> @llvm.masked.expandload.v32f16(ptr poison, <32 x i1> poison, <32 x half> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:10 CodeSize:6 Lat:12 SizeLat:10 for: %v2i8 = call <2 x i8> @llvm.masked.expandload.v2i8(ptr poison, <2 x i1> poison, <2 x i8> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:24 SizeLat:20 for: %v4i8 = call <4 x i8> @llvm.masked.expandload.v4i8(ptr poison, <4 x i1> poison, <4 x i8> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:40 CodeSize:24 Lat:48 SizeLat:40 for: %v8i8 = call <8 x i8> @llvm.masked.expandload.v8i8(ptr poison, <8 x i1> poison, <8 x i8> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:80 CodeSize:48 Lat:96 SizeLat:80 for: %v16i8 = call <16 x i8> @llvm.masked.expandload.v16i8(ptr poison, <16 x i1> poison, <16 x i8> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:10 CodeSize:6 Lat:12 SizeLat:10 for: %v2i16 = call <2 x i16> @llvm.masked.expandload.v2i16(ptr poison, <2 x i1> poison, <2 x i16> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:24 SizeLat:20 for: %v4i16 = call <4 x i16> @llvm.masked.expandload.v4i16(ptr poison, <4 x i1> poison, <4 x i16> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:40 CodeSize:24 Lat:48 SizeLat:40 for: %v8i16 = call <8 x i16> @llvm.masked.expandload.v8i16(ptr poison, <8 x i1> poison, <8 x i16> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:10 CodeSize:6 Lat:12 SizeLat:10 for: %v2i32 = call <2 x i32> @llvm.masked.expandload.v2i32(ptr poison, <2 x i1> poison, <2 x i32> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:24 SizeLat:20 for: %v4i32 = call <4 x i32> @llvm.masked.expandload.v4i32(ptr poison, <4 x i1> poison, <4 x i32> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:10 CodeSize:6 Lat:12 SizeLat:10 for: %v2i64 = call <2 x i64> @llvm.masked.expandload.v2i64(ptr poison, <2 x i1> poison, <2 x i64> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:8 CodeSize:5 Lat:10 SizeLat:8 for: %v2f16 = call <2 x half> @llvm.masked.expandload.v2f16(ptr poison, <2 x i1> poison, <2 x half> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:18 CodeSize:11 Lat:22 SizeLat:18 for: %v4f16 = call <4 x half> @llvm.masked.expandload.v4f16(ptr poison, <4 x i1> poison, <4 x half> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:38 CodeSize:23 Lat:46 SizeLat:38 for: %v8f16 = call <8 x half> @llvm.masked.expandload.v8f16(ptr poison, <8 x i1> poison, <8 x half> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:8 CodeSize:5 Lat:10 SizeLat:8 for: %v2f32 = call <2 x float> @llvm.masked.expandload.v2f32(ptr poison, <2 x i1> poison, <2 x float> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:18 CodeSize:11 Lat:22 SizeLat:18 for: %v4f32 = call <4 x float> @llvm.masked.expandload.v4f32(ptr poison, <4 x i1> poison, <4 x float> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:8 CodeSize:5 Lat:10 SizeLat:8 for: %v2f64 = call <2 x double> @llvm.masked.expandload.v2f64(ptr poison, <2 x i1> poison, <2 x double> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:24 SizeLat:20 for: %v4i64 = call <4 x i64> @llvm.masked.expandload.v4i64(ptr poison, <4 x i1> poison, <4 x i64> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:152 CodeSize:92 Lat:184 SizeLat:152 for: %v32f16 = call <32 x half> @llvm.masked.expandload.v32f16(ptr poison, <32 x i1> poison, <32 x half> poison)
; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; SVE-ONLY-LABEL: 'fixed'
diff --git a/llvm/test/Analysis/CostModel/AArch64/masked_ldst.ll b/llvm/test/Analysis/CostModel/AArch64/masked_ldst.ll
index c1ee229ae8e2c..f0b5325e5648a 100644
--- a/llvm/test/Analysis/CostModel/AArch64/masked_ldst.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/masked_ldst.ll
@@ -5,24 +5,24 @@ target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
define void @fixed() {
; CHECK-LABEL: 'fixed'
-; CHECK-NEXT: Cost Model: Found costs of RThru:10 CodeSize:8 Lat:18 SizeLat:12 for: %v2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0(ptr align 8 undef, <2 x i1> undef, <2 x i8> undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:20 CodeSize:16 Lat:36 SizeLat:24 for: %v4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 8 undef, <4 x i1> undef, <4 x i8> undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:40 CodeSize:32 Lat:72 SizeLat:48 for: %v8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 8 undef, <8 x i1> undef, <8 x i8> undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:80 CodeSize:64 Lat:144 SizeLat:96 for: %v16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 8 undef, <16 x i1> undef, <16 x i8> undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:10 CodeSize:8 Lat:18 SizeLat:12 for: %v2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0(ptr align 8 undef, <2 x i1> undef, <2 x i16> undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:20 CodeSize:16 Lat:36 SizeLat:24 for: %v4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 8 undef, <4 x i1> undef, <4 x i16> undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:40 CodeSize:32 Lat:72 SizeLat:48 for: %v8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 8 undef, <8 x i1> undef, <8 x i16> undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:10 CodeSize:8 Lat:18 SizeLat:12 for: %v2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr align 8 undef, <2 x i1> undef, <2 x i32> undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:20 CodeSize:16 Lat:36 SizeLat:24 for: %v4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 8 undef, <4 x i1> undef, <4 x i32> undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:10 CodeSize:8 Lat:18 SizeLat:12 for: %v2i64 = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr align 8 undef, <2 x i1> undef, <2 x i64> undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:7 Lat:16 SizeLat:10 for: %v2f16 = call <2 x half> @llvm.masked.load.v2f16.p0(ptr align 8 undef, <2 x i1> undef, <2 x half> undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:18 CodeSize:15 Lat:34 SizeLat:22 for: %v4f16 = call <4 x half> @llvm.masked.load.v4f16.p0(ptr align 8 undef, <4 x i1> undef, <4 x half> undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:38 CodeSize:31 Lat:70 SizeLat:46 for: %v8f16 = call <8 x half> @llvm.masked.load.v8f16.p0(ptr align 8 undef, <8 x i1> undef, <8 x half> undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:7 Lat:16 SizeLat:10 for: %v2f32 = call <2 x float> @llvm.masked.load.v2f32.p0(ptr align 8 undef, <2 x i1> undef, <2 x float> undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:18 CodeSize:15 Lat:34 SizeLat:22 for: %v4f32 = call <4 x float> @llvm.masked.load.v4f32.p0(ptr align 8 undef, <4 x i1> undef, <4 x float> undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:7 Lat:16 SizeLat:10 for: %v2f64 = call <2 x double> @llvm.masked.load.v2f64.p0(ptr align 8 undef, <2 x i1> undef, <2 x double> undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:20 CodeSize:16 Lat:36 SizeLat:24 for: %v4i64 = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 undef, <4 x i1> undef, <4 x i64> undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:152 CodeSize:124 Lat:280 SizeLat:184 for: %v32f16 = call <32 x half> @llvm.masked.load.v32f16.p0(ptr align 8 undef, <32 x i1> undef, <32 x half> undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:10 CodeSize:8 Lat:14 SizeLat:12 for: %v2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0(ptr align 8 undef, <2 x i1> undef, <2 x i8> undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:20 CodeSize:16 Lat:28 SizeLat:24 for: %v4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 8 undef, <4 x i1> undef, <4 x i8> undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:40 CodeSize:32 Lat:56 SizeLat:48 for: %v8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 8 undef, <8 x i1> undef, <8 x i8> undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:80 CodeSize:64 Lat:112 SizeLat:96 for: %v16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 8 undef, <16 x i1> undef, <16 x i8> undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:10 CodeSize:8 Lat:14 SizeLat:12 for: %v2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0(ptr align 8 undef, <2 x i1> undef, <2 x i16> undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:20 CodeSize:16 Lat:28 SizeLat:24 for: %v4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 8 undef, <4 x i1> undef, <4 x i16> undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:40 CodeSize:32 Lat:56 SizeLat:48 for: %v8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 8 undef, <8 x i1> undef, <8 x i16> undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:10 CodeSize:8 Lat:14 SizeLat:12 for: %v2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr align 8 undef, <2 x i1> undef, <2 x i32> undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:20 CodeSize:16 Lat:28 SizeLat:24 for: %v4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 8 undef, <4 x i1> undef, <4 x i32> undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:10 CodeSize:8 Lat:14 SizeLat:12 for: %v2i64 = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr align 8 undef, <2 x i1> undef, <2 x i64> undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:7 Lat:12 SizeLat:10 for: %v2f16 = call <2 x half> @llvm.masked.load.v2f16.p0(ptr align 8 undef, <2 x i1> undef, <2 x half> undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:18 CodeSize:15 Lat:26 SizeLat:22 for: %v4f16 = call <4 x half> @llvm.masked.load.v4f16.p0(ptr align 8 undef, <4 x i1> undef, <4 x half> undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:38 CodeSize:31 Lat:54 SizeLat:46 for: %v8f16 = call <8 x half> @llvm.masked.load.v8f16.p0(ptr align 8 undef, <8 x i1> undef, <8 x half> undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:7 Lat:12 SizeLat:10 for: %v2f32 = call <2 x float> @llvm.masked.load.v2f32.p0(ptr align 8 undef, <2 x i1> undef, <2 x float> undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:18 CodeSize:15 Lat:26 SizeLat:22 for: %v4f32 = call <4 x float> @llvm.masked.load.v4f32.p0(ptr align 8 undef, <4 x i1> undef, <4 x float> undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:7 Lat:12 SizeLat:10 for: %v2f64 = call <2 x double> @llvm.masked.load.v2f64.p0(ptr align 8 undef, <2 x i1> undef, <2 x double> undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:20 CodeSize:16 Lat:28 SizeLat:24 for: %v4i64 = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 undef, <4 x i1> undef, <4 x i64> undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:152 CodeSize:124 Lat:216 SizeLat:184 for: %v32f16 = call <32 x half> @llvm.masked.load.v32f16.p0(ptr align 8 undef, <32 x i1> undef, <32 x half> undef)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
entry:
diff --git a/llvm/test/Analysis/CostModel/AArch64/mem-op-cost-model.ll b/llvm/test/Analysis/CostModel/AArch64/mem-op-cost-model.ll
index 3403e148a505d..b0b42e88d202b 100644
--- a/llvm/test/Analysis/CostModel/AArch64/mem-op-cost-model.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/mem-op-cost-model.ll
@@ -13,19 +13,19 @@ target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
define <16 x i8> @load16(ptr %ptr) {
; CHECK: function 'load16'
; CHECK-NEON-LABEL: 'load16'
-; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <16 x i8>, ptr %ptr, align 16
+; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %out = load <16 x i8>, ptr %ptr, align 16
; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i8> %out
;
; CHECK-SVE-128-LABEL: 'load16'
-; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <16 x i8>, ptr %ptr, align 16
+; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %out = load <16 x i8>, ptr %ptr, align 16
; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i8> %out
;
; CHECK-SVE-256-LABEL: 'load16'
-; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <16 x i8>, ptr %ptr, align 16
+; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %out = load <16 x i8>, ptr %ptr, align 16
; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i8> %out
;
; CHECK-SVE-512-LABEL: 'load16'
-; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <16 x i8>, ptr %ptr, align 16
+; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %out = load <16 x i8>, ptr %ptr, align 16
; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i8> %out
;
%out = load <16 x i8>, ptr %ptr
@@ -57,19 +57,19 @@ define void @store16(ptr %ptr, <16 x i8> %val) {
define <8 x i8> @load8(ptr %ptr) {
; CHECK: function 'load8'
; CHECK-NEON-LABEL: 'load8'
-; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <8 x i8>, ptr %ptr, align 8
+; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %out = load <8 x i8>, ptr %ptr, align 8
; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %out
;
; CHECK-SVE-128-LABEL: 'load8'
-; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <8 x i8>, ptr %ptr, align 8
+; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %out = load <8 x i8>, ptr %ptr, align 8
; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %out
;
; CHECK-SVE-256-LABEL: 'load8'
-; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <8 x i8>, ptr %ptr, align 8
+; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %out = load <8 x i8>, ptr %ptr, align 8
; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %out
;
; CHECK-SVE-512-LABEL: 'load8'
-; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <8 x i8>, ptr %ptr, align 8
+; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %out = load <8 x i8>, ptr %ptr, align 8
; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %out
;
%out = load <8 x i8>, ptr %ptr
@@ -101,19 +101,19 @@ define void @store8(ptr %ptr, <8 x i8> %val) {
define <4 x i8> @load4(ptr %ptr) {
; CHECK: function 'load4'
; CHECK-NEON-LABEL: 'load4'
-; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <4 x i8>, ptr %ptr, align 4
+; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %out = load <4 x i8>, ptr %ptr, align 4
; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %out
;
; CHECK-SVE-128-LABEL: 'load4'
-; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <4 x i8>, ptr %ptr, align 4
+; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %out = load <4 x i8>, ptr %ptr, align 4
; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %out
;
; CHECK-SVE-256-LABEL: 'load4'
-; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <4 x i8>, ptr %ptr, align 4
+; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %out = load <4 x i8>, ptr %ptr, align 4
; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %out
;
; CHECK-SVE-512-LABEL: 'load4'
-; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <4 x i8>, ptr %ptr, align 4
+; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %out = load <4 x i8>, ptr %ptr, align 4
; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %out
;
%out = load <4 x i8>, ptr %ptr
@@ -145,19 +145,19 @@ define void @store4(ptr %ptr, <4 x i8> %val) {
define <16 x i16> @load_256(ptr %ptr) {
; CHECK: function 'load_256'
; CHECK-NEON-LABEL: 'load_256'
-; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %out = load <16 x i16>, ptr %ptr, align 32
+; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:3 SizeLat:2 for: %out = load <16 x i16>, ptr %ptr, align 32
; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i16> %out
;
; CHECK-SVE-128-LABEL: 'load_256'
-; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %out = load <16 x i16>, ptr %ptr, align 32
+; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:3 SizeLat:2 for: %out = load <16 x i16>, ptr %ptr, align 32
; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i16> %out
;
; CHECK-SVE-256-LABEL: 'load_256'
-; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <16 x i16>, ptr %ptr, align 32
+; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %out = load <16 x i16>, ptr %ptr, align 32
; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i16> %out
;
; CHECK-SVE-512-LABEL: 'load_256'
-; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <16 x i16>, ptr %ptr, align 32
+; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %out = load <16 x i16>, ptr %ptr, align 32
; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i16> %out
;
%out = load <16 x i16>, ptr %ptr
@@ -167,11 +167,11 @@ define <16 x i16> @load_256(ptr %ptr) {
define <8 x i64> @load_512(ptr %ptr) {
; CHECK: function 'load_512'
; CHECK-NEON-LABEL: 'load_512'
-; CHECK-NEON-NEXT: Cost Model: Found costs of 4 for: %out = load <8 x i64>, ptr %ptr, align 64
+; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:5 SizeLat:4 for: %out = load <8 x i64>, ptr %ptr, align 64
; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i64> %out
;
; CHECK-SVE-128-LABEL: 'load_512'
-; CHECK-SVE-128-NEXT: Cost Model: Found costs of 4 for: %out = load <8 x i64>, ptr %ptr, align 64
+; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:5 SizeLat:4 for: %out = load <8 x i64>, ptr %ptr, align 64
; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i64> %out
;
; CHECK-SVE-256-LABEL: 'load_512'
@@ -179,7 +179,7 @@ define <8 x i64> @load_512(ptr %ptr) {
; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i64> %out
;
; CHECK-SVE-512-LABEL: 'load_512'
-; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <8 x i64>, ptr %ptr, align 64
+; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %out = load <8 x i64>, ptr %ptr, align 64
; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i64> %out
;
%out = load <8 x i64>, ptr %ptr
@@ -190,19 +190,19 @@ declare <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr>, i32 immarg, <4 x i1>,
define <4 x i8> @gather_load_4xi8_constant_mask(<4 x ptr> %ptrs) {
; CHECK: gather_load_4xi8_constant_mask
; CHECK-NEON-LABEL: 'gather_load_4xi8_constant_mask'
-; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:32 SizeLat:20 for: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true), <4 x i8> undef)
+; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:24 SizeLat:20 for: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true), <4 x i8> undef)
; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %lv
;
; CHECK-SVE-128-LABEL: 'gather_load_4xi8_constant_mask'
-; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:32 SizeLat:20 for: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true), <4 x i8> undef)
+; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:24 SizeLat:20 for: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true), <4 x i8> undef)
; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %lv
;
; CHECK-SVE-256-LABEL: 'gather_load_4xi8_constant_mask'
-; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:40 CodeSize:40 Lat:160 SizeLat:40 for: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true), <4 x i8> undef)
+; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:40 CodeSize:40 Lat:80 SizeLat:40 for: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true), <4 x i8> undef)
; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %lv
;
; CHECK-SVE-512-LABEL: 'gather_load_4xi8_constant_mask'
-; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:40 CodeSize:40 Lat:160 SizeLat:40 for: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true), <4 x i8> undef)
+; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:40 CodeSize:40 Lat:80 SizeLat:40 for: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true), <4 x i8> undef)
; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %lv
;
%lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef)
@@ -212,19 +212,19 @@ define <4 x i8> @gather_load_4xi8_constant_mask(<4 x ptr> %ptrs) {
define <4 x i8> @gather_load_4xi8_variable_mask(<4 x ptr> %ptrs, <4 x i1> %cond) {
; CHECK: gather_load_4xi8_variable_mask
; CHECK-NEON-LABEL: 'gather_load_4xi8_variable_mask'
-; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:28 CodeSize:20 Lat:44 SizeLat:32 for: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> %cond, <4 x i8> undef)
+; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:28 CodeSize:20 Lat:36 SizeLat:32 for: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> %cond, <4 x i8> undef)
; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %lv
;
; CHECK-SVE-128-LABEL: 'gather_load_4xi8_variable_mask'
-; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:28 CodeSize:20 Lat:44 SizeLat:32 for: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> %cond, <4 x i8> undef)
+; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:28 CodeSize:20 Lat:36 SizeLat:32 for: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> %cond, <4 x i8> undef)
; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %lv
;
; CHECK-SVE-256-LABEL: 'gather_load_4xi8_variable_mask'
-; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:40 CodeSize:40 Lat:160 SizeLat:40 for: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> %cond, <4 x i8> undef)
+; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:40 CodeSize:40 Lat:80 SizeLat:40 for: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> %cond, <4 x i8> undef)
; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %lv
;
; CHECK-SVE-512-LABEL: 'gather_load_4xi8_variable_mask'
-; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:40 CodeSize:40 Lat:160 SizeLat:40 for: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> %cond, <4 x i8> undef)
+; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:40 CodeSize:40 Lat:80 SizeLat:40 for: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> %cond, <4 x i8> undef)
; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %lv
;
%lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i8> undef)
@@ -280,19 +280,19 @@ declare <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr>, i32 immarg, <4 x i1>
define <4 x i32> @gather_load_4xi32_constant_mask(<4 x ptr> %ptrs) {
; CHECK: gather_load_4xi32_constant_mask
; CHECK-NEON-LABEL: 'gather_load_4xi32_constant_mask'
-; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:32 SizeLat:20 for: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true), <4 x i32> undef)
+; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:24 SizeLat:20 for: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true), <4 x i32> undef)
; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %lv
;
; CHECK-SVE-128-LABEL: 'gather_load_4xi32_constant_mask'
-; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:32 SizeLat:20 for: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true), <4 x i32> undef)
+; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:24 SizeLat:20 for: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true), <4 x i32> undef)
; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %lv
;
; CHECK-SVE-256-LABEL: 'gather_load_4xi32_constant_mask'
-; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:40 CodeSize:40 Lat:160 SizeLat:40 for: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true), <4 x i32> undef)
+; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:40 CodeSize:40 Lat:80 SizeLat:40 for: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true), <4 x i32> undef)
; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %lv
;
; CHECK-SVE-512-LABEL: 'gather_load_4xi32_constant_mask'
-; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:40 CodeSize:40 Lat:160 SizeLat:40 for: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true), <4 x i32> undef)
+; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:40 CodeSize:40 Lat:80 SizeLat:40 for: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true), <4 x i32> undef)
; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %lv
;
%lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
@@ -302,19 +302,19 @@ define <4 x i32> @gather_load_4xi32_constant_mask(<4 x ptr> %ptrs) {
define <4 x i32> @gather_load_4xi32_variable_mask(<4 x ptr> %ptrs, <4 x i1> %cond) {
; CHECK: gather_load_4xi32_variable_mask
; CHECK-NEON-LABEL: 'gather_load_4xi32_variable_mask'
-; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:28 CodeSize:20 Lat:44 SizeLat:32 for: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> %cond, <4 x i32> undef)
+; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:28 CodeSize:20 Lat:36 SizeLat:32 for: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> %cond, <4 x i32> undef)
; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %lv
;
; CHECK-SVE-128-LABEL: 'gather_load_4xi32_variable_mask'
-; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:28 CodeSize:20 Lat:44 SizeLat:32 for: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> %cond, <4 x i32> undef)
+; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:28 CodeSize:20 Lat:36 SizeLat:32 for: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> %cond, <4 x i32> undef)
; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %lv
;
; CHECK-SVE-256-LABEL: 'gather_load_4xi32_variable_mask'
-; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:40 CodeSize:40 Lat:160 SizeLat:40 for: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> %cond, <4 x i32> undef)
+; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:40 CodeSize:40 Lat:80 SizeLat:40 for: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> %cond, <4 x i32> undef)
; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %lv
;
; CHECK-SVE-512-LABEL: 'gather_load_4xi32_variable_mask'
-; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:40 CodeSize:40 Lat:160 SizeLat:40 for: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> %cond, <4 x i32> undef)
+; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:40 CodeSize:40 Lat:80 SizeLat:40 for: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> %cond, <4 x i32> undef)
; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %lv
;
%lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i32> undef)
@@ -370,19 +370,19 @@ declare <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr>, i32, <256 x
define void @sve_gather_vls(<256 x i1> %v256i1mask) {
; CHECK-LABEL: 'sve_scatter_vls'
; CHECK-NEON-LABEL: 'sve_gather_vls'
-; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:1792 CodeSize:1280 Lat:2816 SizeLat:2048 for: %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> align 2 undef, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer)
+; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:1792 CodeSize:1280 Lat:2304 SizeLat:2048 for: %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> align 2 undef, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer)
; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; CHECK-SVE-128-LABEL: 'sve_gather_vls'
-; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:1792 CodeSize:1280 Lat:2816 SizeLat:2048 for: %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> align 2 undef, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer)
+; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:1792 CodeSize:1280 Lat:2304 SizeLat:2048 for: %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> align 2 undef, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer)
; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; CHECK-SVE-256-LABEL: 'sve_gather_vls'
-; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:2560 CodeSize:2560 Lat:10240 SizeLat:2560 for: %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> align 2 undef, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer)
+; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:2560 CodeSize:2560 Lat:5120 SizeLat:2560 for: %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> align 2 undef, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer)
; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; CHECK-SVE-512-LABEL: 'sve_gather_vls'
-; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:2560 CodeSize:2560 Lat:10240 SizeLat:2560 for: %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> align 2 undef, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer)
+; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:2560 CodeSize:2560 Lat:5120 SizeLat:2560 for: %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> align 2 undef, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer)
; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
entry:
@@ -394,19 +394,19 @@ declare <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr>, i32, <256
define void @sve_gather_vls_float(<256 x i1> %v256i1mask) {
; CHECK-LABEL: 'sve_gather_vls_float'
; CHECK-NEON-LABEL: 'sve_gather_vls_float'
-; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:1664 CodeSize:1216 Lat:2688 SizeLat:1920 for: %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> align 4 undef, <256 x i1> %v256i1mask, <256 x float> zeroinitializer)
+; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:1664 CodeSize:1216 Lat:2176 SizeLat:1920 for: %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> align 4 undef, <256 x i1> %v256i1mask, <256 x float> zeroinitializer)
; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; CHECK-SVE-128-LABEL: 'sve_gather_vls_float'
-; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:1664 CodeSize:1216 Lat:2688 SizeLat:1920 for: %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> align 4 undef, <256 x i1> %v256i1mask, <256 x float> zeroinitializer)
+; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:1664 CodeSize:1216 Lat:2176 SizeLat:1920 for: %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> align 4 undef, <256 x i1> %v256i1mask, <256 x float> zeroinitializer)
; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; CHECK-SVE-256-LABEL: 'sve_gather_vls_float'
-; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:2560 CodeSize:2560 Lat:10240 SizeLat:2560 for: %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> align 4 undef, <256 x i1> %v256i1mask, <256 x float> zeroinitializer)
+; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:2560 CodeSize:2560 Lat:5120 SizeLat:2560 for: %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> align 4 undef, <256 x i1> %v256i1mask, <256 x float> zeroinitializer)
; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; CHECK-SVE-512-LABEL: 'sve_gather_vls_float'
-; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:2560 CodeSize:2560 Lat:10240 SizeLat:2560 for: %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> align 4 undef, <256 x i1> %v256i1mask, <256 x float> zeroinitializer)
+; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:2560 CodeSize:2560 Lat:5120 SizeLat:2560 for: %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> align 4 undef, <256 x i1> %v256i1mask, <256 x float> zeroinitializer)
; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
entry:
diff --git a/llvm/test/Analysis/CostModel/AArch64/shuffle-load.ll b/llvm/test/Analysis/CostModel/AArch64/shuffle-load.ll
index dc80267360ea6..e64a8a67c5a1b 100644
--- a/llvm/test/Analysis/CostModel/AArch64/shuffle-load.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/shuffle-load.ll
@@ -10,49 +10,49 @@ target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
; and splats as that is how getShuffleCost currently recognizes them.
define void @shuffle() {
; CHECK-LABEL: 'shuffle'
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:4 SizeLat:1 for: %lv2i8 = load <2 x i8>, ptr undef, align 2
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:2 SizeLat:1 for: %lv2i8 = load <2 x i8>, ptr undef, align 2
; CHECK-NEXT: Cost Model: Found costs of 1 for: %sv2i8 = shufflevector <2 x i8> %lv2i8, <2 x i8> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %lv4i8 = load <4 x i8>, ptr undef, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %lv4i8 = load <4 x i8>, ptr undef, align 4
; CHECK-NEXT: Cost Model: Found costs of 1 for: %sv4i8 = shufflevector <4 x i8> %lv4i8, <4 x i8> undef, <4 x i32> zeroinitializer
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %lv8i8 = load <8 x i8>, ptr undef, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %lv8i8 = load <8 x i8>, ptr undef, align 8
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv8i8 = shufflevector <8 x i8> %lv8i8, <8 x i8> undef, <8 x i32> zeroinitializer
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %lv16i8 = load <16 x i8>, ptr undef, align 16
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %lv16i8 = load <16 x i8>, ptr undef, align 16
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv16i8 = shufflevector <16 x i8> %lv16i8, <16 x i8> undef, <16 x i32> zeroinitializer
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:4 SizeLat:1 for: %lv2i16 = load <2 x i16>, ptr undef, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:2 SizeLat:1 for: %lv2i16 = load <2 x i16>, ptr undef, align 4
; CHECK-NEXT: Cost Model: Found costs of 1 for: %sv2i16 = shufflevector <2 x i16> %lv2i16, <2 x i16> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %lv4i16 = load <4 x i16>, ptr undef, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %lv4i16 = load <4 x i16>, ptr undef, align 8
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv4i16 = shufflevector <4 x i16> %lv4i16, <4 x i16> undef, <4 x i32> zeroinitializer
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %lv8i16 = load <8 x i16>, ptr undef, align 16
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %lv8i16 = load <8 x i16>, ptr undef, align 16
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv8i16 = shufflevector <8 x i16> %lv8i16, <8 x i16> undef, <8 x i32> zeroinitializer
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %lv16i16 = load <16 x i16>, ptr undef, align 32
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:3 SizeLat:2 for: %lv16i16 = load <16 x i16>, ptr undef, align 32
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv16i16 = shufflevector <16 x i16> %lv16i16, <16 x i16> undef, <16 x i32> zeroinitializer
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %lv2i32 = load <2 x i32>, ptr undef, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %lv2i32 = load <2 x i32>, ptr undef, align 8
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv2i32 = shufflevector <2 x i32> %lv2i32, <2 x i32> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %lv4i32 = load <4 x i32>, ptr undef, align 16
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %lv4i32 = load <4 x i32>, ptr undef, align 16
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv4i32 = shufflevector <4 x i32> %lv4i32, <4 x i32> undef, <4 x i32> zeroinitializer
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %lv8i32 = load <8 x i32>, ptr undef, align 32
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:3 SizeLat:2 for: %lv8i32 = load <8 x i32>, ptr undef, align 32
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv8i32 = shufflevector <8 x i32> %lv8i32, <8 x i32> undef, <8 x i32> zeroinitializer
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %lv2i64 = load <2 x i64>, ptr undef, align 16
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %lv2i64 = load <2 x i64>, ptr undef, align 16
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv2i64 = shufflevector <2 x i64> %lv2i64, <2 x i64> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %lv4i64 = load <4 x i64>, ptr undef, align 32
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:3 SizeLat:2 for: %lv4i64 = load <4 x i64>, ptr undef, align 32
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv4i64 = shufflevector <4 x i64> %lv4i64, <4 x i64> undef, <4 x i32> zeroinitializer
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %lv2f16 = load <2 x half>, ptr undef, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %lv2f16 = load <2 x half>, ptr undef, align 4
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv2f16 = shufflevector <2 x half> %lv2f16, <2 x half> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %lv4f16 = load <4 x half>, ptr undef, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %lv4f16 = load <4 x half>, ptr undef, align 8
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv4f16 = shufflevector <4 x half> %lv4f16, <4 x half> undef, <4 x i32> zeroinitializer
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %lv8f16 = load <8 x half>, ptr undef, align 16
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %lv8f16 = load <8 x half>, ptr undef, align 16
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv8f16 = shufflevector <8 x half> %lv8f16, <8 x half> undef, <8 x i32> zeroinitializer
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %lv16f16 = load <16 x half>, ptr undef, align 32
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:3 SizeLat:2 for: %lv16f16 = load <16 x half>, ptr undef, align 32
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv16f16 = shufflevector <16 x half> %lv16f16, <16 x half> undef, <16 x i32> zeroinitializer
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %lv2f32 = load <2 x float>, ptr undef, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %lv2f32 = load <2 x float>, ptr undef, align 8
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv2f32 = shufflevector <2 x float> %lv2f32, <2 x float> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %lv4f32 = load <4 x float>, ptr undef, align 16
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %lv4f32 = load <4 x float>, ptr undef, align 16
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv4f32 = shufflevector <4 x float> %lv4f32, <4 x float> undef, <4 x i32> zeroinitializer
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %lv8f32 = load <8 x float>, ptr undef, align 32
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:3 SizeLat:2 for: %lv8f32 = load <8 x float>, ptr undef, align 32
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv8f32 = shufflevector <8 x float> %lv8f32, <8 x float> undef, <8 x i32> zeroinitializer
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %lv2f64 = load <2 x double>, ptr undef, align 16
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %lv2f64 = load <2 x double>, ptr undef, align 16
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv2f64 = shufflevector <2 x double> %lv2f64, <2 x double> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %lv4f64 = load <4 x double>, ptr undef, align 32
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:3 SizeLat:2 for: %lv4f64 = load <4 x double>, ptr undef, align 32
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv4f64 = shufflevector <4 x double> %lv4f64, <4 x double> undef, <4 x i32> zeroinitializer
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
@@ -114,7 +114,7 @@ define void @shuffle() {
define <4 x half> @ld1r_4h_float_shuff(ptr nocapture %x) {
; CHECK-LABEL: 'ld1r_4h_float_shuff'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %tmp = load half, ptr %x, align 2
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %tmp = load half, ptr %x, align 2
; CHECK-NEXT: Cost Model: Found costs of 0 for: %tmp1 = insertelement <4 x half> undef, half %tmp, i32 0
; CHECK-NEXT: Cost Model: Found costs of 1 for: %lane = shufflevector <4 x half> %tmp1, <4 x half> undef, <4 x i32> zeroinitializer
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x half> %lane
@@ -128,7 +128,7 @@ entry:
define <8 x half> @ld1r_8h_float_shuff(ptr nocapture %x) {
; CHECK-LABEL: 'ld1r_8h_float_shuff'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %tmp = load half, ptr %x, align 2
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %tmp = load half, ptr %x, align 2
; CHECK-NEXT: Cost Model: Found costs of 0 for: %tmp1 = insertelement <8 x half> undef, half %tmp, i32 0
; CHECK-NEXT: Cost Model: Found costs of 1 for: %lane = shufflevector <8 x half> %tmp1, <8 x half> undef, <8 x i32> zeroinitializer
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x half> %lane
@@ -142,7 +142,7 @@ entry:
define <2 x float> @ld1r_2s_float_shuff(ptr nocapture %x) {
; CHECK-LABEL: 'ld1r_2s_float_shuff'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %tmp = load float, ptr %x, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %tmp = load float, ptr %x, align 4
; CHECK-NEXT: Cost Model: Found costs of 0 for: %tmp1 = insertelement <2 x float> undef, float %tmp, i32 0
; CHECK-NEXT: Cost Model: Found costs of 1 for: %lane = shufflevector <2 x float> %tmp1, <2 x float> undef, <2 x i32> zeroinitializer
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x float> %lane
@@ -156,7 +156,7 @@ entry:
define <4 x float> @ld1r_4s_float_shuff(ptr nocapture %x) {
; CHECK-LABEL: 'ld1r_4s_float_shuff'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %tmp = load float, ptr %x, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %tmp = load float, ptr %x, align 4
; CHECK-NEXT: Cost Model: Found costs of 0 for: %tmp1 = insertelement <4 x float> undef, float %tmp, i32 0
; CHECK-NEXT: Cost Model: Found costs of 1 for: %lane = shufflevector <4 x float> %tmp1, <4 x float> undef, <4 x i32> zeroinitializer
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x float> %lane
@@ -170,7 +170,7 @@ entry:
define <2 x double> @ld1r_2d_double_shuff(ptr nocapture %x) {
; CHECK-LABEL: 'ld1r_2d_double_shuff'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %tmp = load double, ptr %x, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %tmp = load double, ptr %x, align 4
; CHECK-NEXT: Cost Model: Found costs of 0 for: %tmp1 = insertelement <2 x double> undef, double %tmp, i32 0
; CHECK-NEXT: Cost Model: Found costs of 1 for: %lane = shufflevector <2 x double> %tmp1, <2 x double> undef, <2 x i32> zeroinitializer
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x double> %lane
@@ -186,7 +186,7 @@ entry:
define <8 x i8> @ld1r_8b_int_shuff(ptr nocapture %x) {
; CHECK-LABEL: 'ld1r_8b_int_shuff'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %tmp = load i8, ptr %x, align 2
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %tmp = load i8, ptr %x, align 2
; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:0 Lat:3 SizeLat:3 for: %tmp1 = insertelement <8 x i8> undef, i8 %tmp, i8 0
; CHECK-NEXT: Cost Model: Found costs of 1 for: %lane = shufflevector <8 x i8> %tmp1, <8 x i8> undef, <8 x i32> zeroinitializer
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %lane
@@ -200,7 +200,7 @@ entry:
define <16 x i8> @ld1r_16b_int_shuff(ptr nocapture %x) {
; CHECK-LABEL: 'ld1r_16b_int_shuff'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %tmp = load i8, ptr %x, align 2
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %tmp = load i8, ptr %x, align 2
; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:0 Lat:3 SizeLat:3 for: %tmp1 = insertelement <16 x i8> undef, i8 %tmp, i8 0
; CHECK-NEXT: Cost Model: Found costs of 1 for: %lane = shufflevector <16 x i8> %tmp1, <16 x i8> undef, <16 x i32> zeroinitializer
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i8> %lane
@@ -214,7 +214,7 @@ entry:
define <4 x i16> @ld1r_4h_int_shuff(ptr nocapture %x) {
; CHECK-LABEL: 'ld1r_4h_int_shuff'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %tmp = load i16, ptr %x, align 2
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %tmp = load i16, ptr %x, align 2
; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:0 Lat:3 SizeLat:3 for: %tmp1 = insertelement <4 x i16> undef, i16 %tmp, i16 0
; CHECK-NEXT: Cost Model: Found costs of 1 for: %lane = shufflevector <4 x i16> %tmp1, <4 x i16> undef, <4 x i32> zeroinitializer
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i16> %lane
@@ -228,7 +228,7 @@ entry:
define <8 x i16> @ld1r_8h_int_shuff(ptr nocapture %x) {
; CHECK-LABEL: 'ld1r_8h_int_shuff'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %tmp = load i16, ptr %x, align 2
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %tmp = load i16, ptr %x, align 2
; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:0 Lat:3 SizeLat:3 for: %tmp1 = insertelement <8 x i16> undef, i16 %tmp, i16 0
; CHECK-NEXT: Cost Model: Found costs of 1 for: %lane = shufflevector <8 x i16> %tmp1, <8 x i16> undef, <8 x i32> zeroinitializer
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i16> %lane
@@ -242,7 +242,7 @@ entry:
define <2 x i32> @ld1r_2s_int_shuff(ptr nocapture %x) {
; CHECK-LABEL: 'ld1r_2s_int_shuff'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %tmp = load i32, ptr %x, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %tmp = load i32, ptr %x, align 4
; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:0 Lat:3 SizeLat:3 for: %tmp1 = insertelement <2 x i32> undef, i32 %tmp, i32 0
; CHECK-NEXT: Cost Model: Found costs of 1 for: %lane = shufflevector <2 x i32> %tmp1, <2 x i32> undef, <2 x i32> zeroinitializer
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i32> %lane
@@ -256,7 +256,7 @@ entry:
define <4 x i32> @ld1r_4s_int_shuff(ptr nocapture %x) {
; CHECK-LABEL: 'ld1r_4s_int_shuff'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %tmp = load i32, ptr %x, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %tmp = load i32, ptr %x, align 4
; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:0 Lat:3 SizeLat:3 for: %tmp1 = insertelement <4 x i32> undef, i32 %tmp, i32 0
; CHECK-NEXT: Cost Model: Found costs of 1 for: %lane = shufflevector <4 x i32> %tmp1, <4 x i32> undef, <4 x i32> zeroinitializer
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %lane
@@ -270,7 +270,7 @@ entry:
define <2 x i64> @ld1r_2d_int_shuff(ptr nocapture %x) {
; CHECK-LABEL: 'ld1r_2d_int_shuff'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %tmp = load i64, ptr %x, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %tmp = load i64, ptr %x, align 8
; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:0 Lat:3 SizeLat:3 for: %tmp1 = insertelement <2 x i64> undef, i64 %tmp, i32 0
; CHECK-NEXT: Cost Model: Found costs of 1 for: %lane = shufflevector <2 x i64> %tmp1, <2 x i64> undef, <2 x i32> zeroinitializer
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i64> %lane
@@ -284,52 +284,52 @@ entry:
define void @vld2(ptr %p) {
; CHECK-LABEL: 'vld2'
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %v4i8 = load <4 x i8>, ptr %p, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4i8 = load <4 x i8>, ptr %p, align 4
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i8_0 = shufflevector <4 x i8> %v4i8, <4 x i8> undef, <2 x i32> <i32 0, i32 2>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i8_1 = shufflevector <4 x i8> %v4i8, <4 x i8> undef, <2 x i32> <i32 1, i32 3>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v8i8 = load <8 x i8>, ptr %p, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v8i8 = load <8 x i8>, ptr %p, align 8
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i8_0 = shufflevector <8 x i8> %v8i8, <8 x i8> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i8_1 = shufflevector <8 x i8> %v8i8, <8 x i8> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v16i8 = load <16 x i8>, ptr %p, align 16
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v16i8 = load <16 x i8>, ptr %p, align 16
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v16i8_0 = shufflevector <16 x i8> %v16i8, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v16i8_1 = shufflevector <16 x i8> %v16i8, <16 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %v32i8 = load <32 x i8>, ptr %p, align 32
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:3 SizeLat:2 for: %v32i8 = load <32 x i8>, ptr %p, align 32
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v32i8_0 = shufflevector <32 x i8> %v32i8, <32 x i8> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v32i8_1 = shufflevector <32 x i8> %v32i8, <32 x i8> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v4i16 = load <4 x i16>, ptr %p, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4i16 = load <4 x i16>, ptr %p, align 8
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i16_0 = shufflevector <4 x i16> %v4i16, <4 x i16> undef, <2 x i32> <i32 0, i32 2>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i16_1 = shufflevector <4 x i16> %v4i16, <4 x i16> undef, <2 x i32> <i32 1, i32 3>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v8i16 = load <8 x i16>, ptr %p, align 16
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v8i16 = load <8 x i16>, ptr %p, align 16
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i16_0 = shufflevector <8 x i16> %v8i16, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i16_1 = shufflevector <8 x i16> %v8i16, <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %v16i16 = load <16 x i16>, ptr %p, align 32
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:3 SizeLat:2 for: %v16i16 = load <16 x i16>, ptr %p, align 32
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v16i16_0 = shufflevector <16 x i16> %v16i16, <16 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v16i16_1 = shufflevector <16 x i16> %v16i16, <16 x i16> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; CHECK-NEXT: Cost Model: Found costs of 4 for: %v32i16 = load <32 x i16>, ptr %p, align 64
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:5 SizeLat:4 for: %v32i16 = load <32 x i16>, ptr %p, align 64
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v32i16_0 = shufflevector <32 x i16> %v32i16, <32 x i16> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v32i16_1 = shufflevector <32 x i16> %v32i16, <32 x i16> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v4i32 = load <4 x i32>, ptr %p, align 16
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4i32 = load <4 x i32>, ptr %p, align 16
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i32_0 = shufflevector <4 x i32> %v4i32, <4 x i32> undef, <2 x i32> <i32 0, i32 2>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i32_1 = shufflevector <4 x i32> %v4i32, <4 x i32> undef, <2 x i32> <i32 1, i32 3>
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %v8i32 = load <8 x i32>, ptr %p, align 32
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:3 SizeLat:2 for: %v8i32 = load <8 x i32>, ptr %p, align 32
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i32_0 = shufflevector <8 x i32> %v8i32, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i32_1 = shufflevector <8 x i32> %v8i32, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT: Cost Model: Found costs of 4 for: %v16i32 = load <16 x i32>, ptr %p, align 64
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:5 SizeLat:4 for: %v16i32 = load <16 x i32>, ptr %p, align 64
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v16i32_0 = shufflevector <16 x i32> %v16i32, <16 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v16i32_1 = shufflevector <16 x i32> %v16i32, <16 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:4 SizeLat:8 for: %v32i32 = load <32 x i32>, ptr %p, align 128
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:9 SizeLat:8 for: %v32i32 = load <32 x i32>, ptr %p, align 128
; CHECK-NEXT: Cost Model: Found costs of 4 for: %v32i32_0 = shufflevector <32 x i32> %v32i32, <32 x i32> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
; CHECK-NEXT: Cost Model: Found costs of 4 for: %v32i32_1 = shufflevector <32 x i32> %v32i32, <32 x i32> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %v2i64 = load <4 x i64>, ptr %p, align 32
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:3 SizeLat:2 for: %v2i64 = load <4 x i64>, ptr %p, align 32
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v2i64_0 = shufflevector <4 x i64> %v2i64, <4 x i64> undef, <2 x i32> <i32 0, i32 2>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v2i64_1 = shufflevector <4 x i64> %v2i64, <4 x i64> undef, <2 x i32> <i32 1, i32 3>
-; CHECK-NEXT: Cost Model: Found costs of 4 for: %v4i64 = load <8 x i64>, ptr %p, align 64
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:5 SizeLat:4 for: %v4i64 = load <8 x i64>, ptr %p, align 64
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4i64_0 = shufflevector <8 x i64> %v4i64, <8 x i64> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4i64_1 = shufflevector <8 x i64> %v4i64, <8 x i64> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:4 SizeLat:8 for: %v8i64 = load <16 x i64>, ptr %p, align 128
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:9 SizeLat:8 for: %v8i64 = load <16 x i64>, ptr %p, align 128
; CHECK-NEXT: Cost Model: Found costs of 4 for: %v8i64_0 = shufflevector <16 x i64> %v8i64, <16 x i64> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
; CHECK-NEXT: Cost Model: Found costs of 4 for: %v8i64_1 = shufflevector <16 x i64> %v8i64, <16 x i64> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; CHECK-NEXT: Cost Model: Found costs of RThru:16 CodeSize:16 Lat:4 SizeLat:16 for: %v16i64 = load <32 x i64>, ptr %p, align 256
+; CHECK-NEXT: Cost Model: Found costs of RThru:16 CodeSize:16 Lat:17 SizeLat:16 for: %v16i64 = load <32 x i64>, ptr %p, align 256
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v16i64_0 = shufflevector <32 x i64> %v16i64, <32 x i64> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v16i64_1 = shufflevector <32 x i64> %v16i64, <32 x i64> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
@@ -392,67 +392,67 @@ define void @vld2(ptr %p) {
define void @vld3(ptr %p) {
; CHECK-LABEL: 'vld3'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v2i8 = load <6 x i8>, ptr %p, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2i8 = load <6 x i8>, ptr %p, align 8
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v2i8_0 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> <i32 0, i32 3>
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v2i8_1 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> <i32 1, i32 4>
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v2i8_2 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> <i32 2, i32 5>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v4i8 = load <12 x i8>, ptr %p, align 16
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4i8 = load <12 x i8>, ptr %p, align 16
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v4i8_0 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v4i8_1 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v4i8_2 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %v8i8 = load <24 x i8>, ptr %p, align 32
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:3 SizeLat:2 for: %v8i8 = load <24 x i8>, ptr %p, align 32
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i8_0 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i8_1 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i8_2 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
-; CHECK-NEXT: Cost Model: Found costs of 4 for: %v16i8 = load <48 x i8>, ptr %p, align 64
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:5 SizeLat:4 for: %v16i8 = load <48 x i8>, ptr %p, align 64
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v16i8_0 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v16i8_1 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v16i8_2 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v2i16 = load <6 x i16>, ptr %p, align 16
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2i16 = load <6 x i16>, ptr %p, align 16
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v2i16_0 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> <i32 0, i32 3>
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v2i16_1 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> <i32 1, i32 4>
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v2i16_2 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> <i32 2, i32 5>
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %v4i16 = load <12 x i16>, ptr %p, align 32
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:3 SizeLat:2 for: %v4i16 = load <12 x i16>, ptr %p, align 32
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i16_0 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i16_1 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i16_2 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
-; CHECK-NEXT: Cost Model: Found costs of 4 for: %v8i16 = load <24 x i16>, ptr %p, align 64
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:5 SizeLat:4 for: %v8i16 = load <24 x i16>, ptr %p, align 64
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i16_0 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i16_1 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i16_2 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
-; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:4 SizeLat:8 for: %v16i16 = load <48 x i16>, ptr %p, align 128
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:9 SizeLat:8 for: %v16i16 = load <48 x i16>, ptr %p, align 128
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v16i16_0 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v16i16_1 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46>
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v16i16_2 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47>
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %v2i32 = load <6 x i32>, ptr %p, align 32
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:3 SizeLat:2 for: %v2i32 = load <6 x i32>, ptr %p, align 32
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v2i32_0 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> <i32 0, i32 3>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v2i32_1 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> <i32 1, i32 4>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v2i32_2 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> <i32 2, i32 5>
-; CHECK-NEXT: Cost Model: Found costs of 4 for: %v4i32 = load <12 x i32>, ptr %p, align 64
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:5 SizeLat:4 for: %v4i32 = load <12 x i32>, ptr %p, align 64
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i32_0 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i32_1 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i32_2 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
-; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:4 SizeLat:8 for: %v8i32 = load <24 x i32>, ptr %p, align 128
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:9 SizeLat:8 for: %v8i32 = load <24 x i32>, ptr %p, align 128
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v8i32_0 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v8i32_1 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v8i32_2 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
-; CHECK-NEXT: Cost Model: Found costs of RThru:16 CodeSize:16 Lat:4 SizeLat:16 for: %v16i32 = load <48 x i32>, ptr %p, align 256
+; CHECK-NEXT: Cost Model: Found costs of RThru:16 CodeSize:16 Lat:17 SizeLat:16 for: %v16i32 = load <48 x i32>, ptr %p, align 256
; CHECK-NEXT: Cost Model: Found costs of 4 for: %v16i32_0 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
; CHECK-NEXT: Cost Model: Found costs of 4 for: %v16i32_1 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46>
; CHECK-NEXT: Cost Model: Found costs of 4 for: %v16i32_2 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47>
-; CHECK-NEXT: Cost Model: Found costs of 4 for: %v2i64 = load <6 x i64>, ptr %p, align 64
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:5 SizeLat:4 for: %v2i64 = load <6 x i64>, ptr %p, align 64
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v2i64_0 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> <i32 0, i32 3>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v2i64_1 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> <i32 1, i32 4>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v2i64_2 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> <i32 2, i32 5>
-; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:4 SizeLat:8 for: %v4i64 = load <12 x i64>, ptr %p, align 128
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:9 SizeLat:8 for: %v4i64 = load <12 x i64>, ptr %p, align 128
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4i64_0 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4i64_1 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4i64_2 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
-; CHECK-NEXT: Cost Model: Found costs of RThru:16 CodeSize:16 Lat:4 SizeLat:16 for: %v8i64 = load <24 x i64>, ptr %p, align 256
+; CHECK-NEXT: Cost Model: Found costs of RThru:16 CodeSize:16 Lat:17 SizeLat:16 for: %v8i64 = load <24 x i64>, ptr %p, align 256
; CHECK-NEXT: Cost Model: Found costs of 4 for: %v8i64_0 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
; CHECK-NEXT: Cost Model: Found costs of 4 for: %v8i64_1 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
; CHECK-NEXT: Cost Model: Found costs of 4 for: %v8i64_2 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
-; CHECK-NEXT: Cost Model: Found costs of RThru:32 CodeSize:32 Lat:4 SizeLat:32 for: %v16i64 = load <48 x i64>, ptr %p, align 512
+; CHECK-NEXT: Cost Model: Found costs of RThru:32 CodeSize:32 Lat:33 SizeLat:32 for: %v16i64 = load <48 x i64>, ptr %p, align 512
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v16i64_0 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v16i64_1 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46>
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v16i64_2 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47>
@@ -531,82 +531,82 @@ define void @vld3(ptr %p) {
define void @vld4(ptr %p) {
; CHECK-LABEL: 'vld4'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v2i8 = load <8 x i8>, ptr %p, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2i8 = load <8 x i8>, ptr %p, align 8
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v2i8_0 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 0, i32 4>
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v2i8_1 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 1, i32 5>
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v2i8_2 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 2, i32 6>
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v2i8_3 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 3, i32 7>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v4i8 = load <16 x i8>, ptr %p, align 16
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4i8 = load <16 x i8>, ptr %p, align 16
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v4i8_0 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v4i8_1 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v4i8_2 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v4i8_3 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %v8i8 = load <32 x i8>, ptr %p, align 32
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:3 SizeLat:2 for: %v8i8 = load <32 x i8>, ptr %p, align 32
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i8_0 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i8_1 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i8_2 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i8_3 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
-; CHECK-NEXT: Cost Model: Found costs of 4 for: %v16i8 = load <64 x i8>, ptr %p, align 64
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:5 SizeLat:4 for: %v16i8 = load <64 x i8>, ptr %p, align 64
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v16i8_0 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v16i8_1 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v16i8_2 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v16i8_3 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v2i16 = load <8 x i16>, ptr %p, align 16
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2i16 = load <8 x i16>, ptr %p, align 16
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v2i16_0 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 0, i32 4>
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v2i16_1 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 1, i32 5>
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v2i16_2 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 2, i32 6>
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v2i16_3 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 3, i32 7>
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %v4i16 = load <16 x i16>, ptr %p, align 32
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:3 SizeLat:2 for: %v4i16 = load <16 x i16>, ptr %p, align 32
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i16_0 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i16_1 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i16_2 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i16_3 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CHECK-NEXT: Cost Model: Found costs of 4 for: %v8i16 = load <32 x i16>, ptr %p, align 64
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:5 SizeLat:4 for: %v8i16 = load <32 x i16>, ptr %p, align 64
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i16_0 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i16_1 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i16_2 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i16_3 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
-; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:4 SizeLat:8 for: %v16i16 = load <64 x i16>, ptr %p, align 128
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:9 SizeLat:8 for: %v16i16 = load <64 x i16>, ptr %p, align 128
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v16i16_0 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v16i16_1 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v16i16_2 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v16i16_3 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %v2i32 = load <8 x i32>, ptr %p, align 32
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:3 SizeLat:2 for: %v2i32 = load <8 x i32>, ptr %p, align 32
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v2i32_0 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 0, i32 4>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v2i32_1 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 1, i32 5>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v2i32_2 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 2, i32 6>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v2i32_3 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 3, i32 7>
-; CHECK-NEXT: Cost Model: Found costs of 4 for: %v4i32 = load <16 x i32>, ptr %p, align 64
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:5 SizeLat:4 for: %v4i32 = load <16 x i32>, ptr %p, align 64
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i32_0 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i32_1 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i32_2 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i32_3 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:4 SizeLat:8 for: %v8i32 = load <32 x i32>, ptr %p, align 128
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:9 SizeLat:8 for: %v8i32 = load <32 x i32>, ptr %p, align 128
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v8i32_0 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v8i32_1 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v8i32_2 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v8i32_3 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
-; CHECK-NEXT: Cost Model: Found costs of RThru:16 CodeSize:16 Lat:4 SizeLat:16 for: %v16i32 = load <64 x i32>, ptr %p, align 256
+; CHECK-NEXT: Cost Model: Found costs of RThru:16 CodeSize:16 Lat:17 SizeLat:16 for: %v16i32 = load <64 x i32>, ptr %p, align 256
; CHECK-NEXT: Cost Model: Found costs of 4 for: %v16i32_0 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
; CHECK-NEXT: Cost Model: Found costs of 4 for: %v16i32_1 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
; CHECK-NEXT: Cost Model: Found costs of 4 for: %v16i32_2 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
; CHECK-NEXT: Cost Model: Found costs of 4 for: %v16i32_3 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
-; CHECK-NEXT: Cost Model: Found costs of 4 for: %v2i64 = load <8 x i64>, ptr %p, align 64
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:5 SizeLat:4 for: %v2i64 = load <8 x i64>, ptr %p, align 64
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v2i64_0 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 0, i32 4>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v2i64_1 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 1, i32 5>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v2i64_2 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 2, i32 6>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v2i64_3 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 3, i32 7>
-; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:4 SizeLat:8 for: %v4i64 = load <16 x i64>, ptr %p, align 128
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:9 SizeLat:8 for: %v4i64 = load <16 x i64>, ptr %p, align 128
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4i64_0 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4i64_1 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4i64_2 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4i64_3 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CHECK-NEXT: Cost Model: Found costs of RThru:16 CodeSize:16 Lat:4 SizeLat:16 for: %v8i64 = load <32 x i64>, ptr %p, align 256
+; CHECK-NEXT: Cost Model: Found costs of RThru:16 CodeSize:16 Lat:17 SizeLat:16 for: %v8i64 = load <32 x i64>, ptr %p, align 256
; CHECK-NEXT: Cost Model: Found costs of 4 for: %v8i64_0 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
; CHECK-NEXT: Cost Model: Found costs of 4 for: %v8i64_1 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
; CHECK-NEXT: Cost Model: Found costs of 4 for: %v8i64_2 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
; CHECK-NEXT: Cost Model: Found costs of 4 for: %v8i64_3 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
-; CHECK-NEXT: Cost Model: Found costs of RThru:32 CodeSize:32 Lat:4 SizeLat:32 for: %v16i64 = load <64 x i64>, ptr %p, align 512
+; CHECK-NEXT: Cost Model: Found costs of RThru:32 CodeSize:32 Lat:33 SizeLat:32 for: %v16i64 = load <64 x i64>, ptr %p, align 512
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v16i64_0 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v16i64_1 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v16i64_2 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
diff --git a/llvm/test/Analysis/CostModel/AArch64/store.ll b/llvm/test/Analysis/CostModel/AArch64/store.ll
index 4452a34b37072..d3dd7f8e8dcbf 100644
--- a/llvm/test/Analysis/CostModel/AArch64/store.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/store.ll
@@ -26,8 +26,8 @@ define void @getMemoryOpCost() {
; CHECK-NEXT: Cost Model: Found costs of 1 for: store <8 x half> undef, ptr undef, align 4
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: store <2 x i8> undef, ptr undef, align 2
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <4 x i8> undef, ptr undef, align 4
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:4 SizeLat:1 for: %1 = load <2 x i8>, ptr undef, align 2
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %2 = load <4 x i8>, ptr undef, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:2 SizeLat:1 for: %1 = load <2 x i8>, ptr undef, align 2
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %2 = load <4 x i8>, ptr undef, align 4
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; SLOW_MISALIGNED_128_STORE-LABEL: 'getMemoryOpCost'
@@ -47,8 +47,8 @@ define void @getMemoryOpCost() {
; SLOW_MISALIGNED_128_STORE-NEXT: Cost Model: Found costs of RThru:12 CodeSize:1 Lat:1 SizeLat:1 for: store <8 x half> undef, ptr undef, align 4
; SLOW_MISALIGNED_128_STORE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: store <2 x i8> undef, ptr undef, align 2
; SLOW_MISALIGNED_128_STORE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <4 x i8> undef, ptr undef, align 4
-; SLOW_MISALIGNED_128_STORE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:4 SizeLat:1 for: %1 = load <2 x i8>, ptr undef, align 2
-; SLOW_MISALIGNED_128_STORE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %2 = load <4 x i8>, ptr undef, align 4
+; SLOW_MISALIGNED_128_STORE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:2 SizeLat:1 for: %1 = load <2 x i8>, ptr undef, align 2
+; SLOW_MISALIGNED_128_STORE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %2 = load <4 x i8>, ptr undef, align 4
; SLOW_MISALIGNED_128_STORE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
store <4 x i64> undef, ptr undef
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-cast.ll b/llvm/test/Analysis/CostModel/AArch64/sve-cast.ll
index 7b482f39a228f..e33864ab5c5ad 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-cast.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-cast.ll
@@ -1783,18 +1783,18 @@ define i32 @bitcasts() {
define i32 @load_extends() #0 {
; CHECK-SVE-LABEL: 'load_extends'
-; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadi8 = load i8, ptr undef, align 1
-; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadi16 = load i16, ptr undef, align 2
-; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadi32 = load i32, ptr undef, align 4
-; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:4 SizeLat:1 for: %loadv2i8 = load <2 x i8>, ptr undef, align 2
-; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %loadv4i8 = load <4 x i8>, ptr undef, align 4
-; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv8i8 = load <8 x i8>, ptr undef, align 8
-; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:4 SizeLat:1 for: %loadv2i16 = load <2 x i16>, ptr undef, align 4
-; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv4i16 = load <4 x i16>, ptr undef, align 8
-; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv2i32 = load <2 x i32>, ptr undef, align 8
-; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv4i32 = load <4 x i32>, ptr undef, align 16
-; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadnxv2i32 = load <vscale x 2 x i32>, ptr undef, align 8
-; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadnxv4i32 = load <vscale x 4 x i32>, ptr undef, align 16
+; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadi8 = load i8, ptr undef, align 1
+; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadi16 = load i16, ptr undef, align 2
+; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadi32 = load i32, ptr undef, align 4
+; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:2 SizeLat:1 for: %loadv2i8 = load <2 x i8>, ptr undef, align 2
+; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %loadv4i8 = load <4 x i8>, ptr undef, align 4
+; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadv8i8 = load <8 x i8>, ptr undef, align 8
+; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:2 SizeLat:1 for: %loadv2i16 = load <2 x i16>, ptr undef, align 4
+; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadv4i16 = load <4 x i16>, ptr undef, align 8
+; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadv2i32 = load <2 x i32>, ptr undef, align 8
+; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadv4i32 = load <4 x i32>, ptr undef, align 16
+; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %loadnxv2i32 = load <vscale x 2 x i32>, ptr undef, align 8
+; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %loadnxv4i32 = load <vscale x 4 x i32>, ptr undef, align 16
; CHECK-SVE-NEXT: Cost Model: Found costs of 0 for: %r0 = sext i8 %loadi8 to i16
; CHECK-SVE-NEXT: Cost Model: Found costs of 0 for: %r1 = zext i8 %loadi8 to i16
; CHECK-SVE-NEXT: Cost Model: Found costs of 0 for: %r2 = sext i8 %loadi8 to i32
@@ -1828,18 +1828,18 @@ define i32 @load_extends() #0 {
; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; SVE128-NO-NEON-LABEL: 'load_extends'
-; SVE128-NO-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadi8 = load i8, ptr undef, align 1
-; SVE128-NO-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadi16 = load i16, ptr undef, align 2
-; SVE128-NO-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadi32 = load i32, ptr undef, align 4
-; SVE128-NO-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv2i8 = load <2 x i8>, ptr undef, align 2
-; SVE128-NO-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv4i8 = load <4 x i8>, ptr undef, align 4
-; SVE128-NO-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv8i8 = load <8 x i8>, ptr undef, align 8
-; SVE128-NO-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv2i16 = load <2 x i16>, ptr undef, align 4
-; SVE128-NO-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv4i16 = load <4 x i16>, ptr undef, align 8
-; SVE128-NO-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv2i32 = load <2 x i32>, ptr undef, align 8
-; SVE128-NO-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv4i32 = load <4 x i32>, ptr undef, align 16
-; SVE128-NO-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadnxv2i32 = load <vscale x 2 x i32>, ptr undef, align 8
-; SVE128-NO-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadnxv4i32 = load <vscale x 4 x i32>, ptr undef, align 16
+; SVE128-NO-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadi8 = load i8, ptr undef, align 1
+; SVE128-NO-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadi16 = load i16, ptr undef, align 2
+; SVE128-NO-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadi32 = load i32, ptr undef, align 4
+; SVE128-NO-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %loadv2i8 = load <2 x i8>, ptr undef, align 2
+; SVE128-NO-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %loadv4i8 = load <4 x i8>, ptr undef, align 4
+; SVE128-NO-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %loadv8i8 = load <8 x i8>, ptr undef, align 8
+; SVE128-NO-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %loadv2i16 = load <2 x i16>, ptr undef, align 4
+; SVE128-NO-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %loadv4i16 = load <4 x i16>, ptr undef, align 8
+; SVE128-NO-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %loadv2i32 = load <2 x i32>, ptr undef, align 8
+; SVE128-NO-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %loadv4i32 = load <4 x i32>, ptr undef, align 16
+; SVE128-NO-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %loadnxv2i32 = load <vscale x 2 x i32>, ptr undef, align 8
+; SVE128-NO-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %loadnxv4i32 = load <vscale x 4 x i32>, ptr undef, align 16
; SVE128-NO-NEON-NEXT: Cost Model: Found costs of 0 for: %r0 = sext i8 %loadi8 to i16
; SVE128-NO-NEON-NEXT: Cost Model: Found costs of 0 for: %r1 = zext i8 %loadi8 to i16
; SVE128-NO-NEON-NEXT: Cost Model: Found costs of 0 for: %r2 = sext i8 %loadi8 to i32
@@ -1873,18 +1873,18 @@ define i32 @load_extends() #0 {
; SVE128-NO-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; FIXED-MIN-256-LABEL: 'load_extends'
-; FIXED-MIN-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadi8 = load i8, ptr undef, align 1
-; FIXED-MIN-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadi16 = load i16, ptr undef, align 2
-; FIXED-MIN-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadi32 = load i32, ptr undef, align 4
-; FIXED-MIN-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv2i8 = load <2 x i8>, ptr undef, align 2
-; FIXED-MIN-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv4i8 = load <4 x i8>, ptr undef, align 4
-; FIXED-MIN-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv8i8 = load <8 x i8>, ptr undef, align 8
-; FIXED-MIN-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv2i16 = load <2 x i16>, ptr undef, align 4
-; FIXED-MIN-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv4i16 = load <4 x i16>, ptr undef, align 8
-; FIXED-MIN-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv2i32 = load <2 x i32>, ptr undef, align 8
-; FIXED-MIN-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv4i32 = load <4 x i32>, ptr undef, align 16
-; FIXED-MIN-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadnxv2i32 = load <vscale x 2 x i32>, ptr undef, align 8
-; FIXED-MIN-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadnxv4i32 = load <vscale x 4 x i32>, ptr undef, align 16
+; FIXED-MIN-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadi8 = load i8, ptr undef, align 1
+; FIXED-MIN-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadi16 = load i16, ptr undef, align 2
+; FIXED-MIN-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadi32 = load i32, ptr undef, align 4
+; FIXED-MIN-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadv2i8 = load <2 x i8>, ptr undef, align 2
+; FIXED-MIN-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadv4i8 = load <4 x i8>, ptr undef, align 4
+; FIXED-MIN-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadv8i8 = load <8 x i8>, ptr undef, align 8
+; FIXED-MIN-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadv2i16 = load <2 x i16>, ptr undef, align 4
+; FIXED-MIN-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadv4i16 = load <4 x i16>, ptr undef, align 8
+; FIXED-MIN-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadv2i32 = load <2 x i32>, ptr undef, align 8
+; FIXED-MIN-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadv4i32 = load <4 x i32>, ptr undef, align 16
+; FIXED-MIN-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %loadnxv2i32 = load <vscale x 2 x i32>, ptr undef, align 8
+; FIXED-MIN-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %loadnxv4i32 = load <vscale x 4 x i32>, ptr undef, align 16
; FIXED-MIN-256-NEXT: Cost Model: Found costs of 0 for: %r0 = sext i8 %loadi8 to i16
; FIXED-MIN-256-NEXT: Cost Model: Found costs of 0 for: %r1 = zext i8 %loadi8 to i16
; FIXED-MIN-256-NEXT: Cost Model: Found costs of 0 for: %r2 = sext i8 %loadi8 to i32
@@ -1918,18 +1918,18 @@ define i32 @load_extends() #0 {
; FIXED-MIN-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; FIXED-MIN-2048-LABEL: 'load_extends'
-; FIXED-MIN-2048-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadi8 = load i8, ptr undef, align 1
-; FIXED-MIN-2048-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadi16 = load i16, ptr undef, align 2
-; FIXED-MIN-2048-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadi32 = load i32, ptr undef, align 4
-; FIXED-MIN-2048-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv2i8 = load <2 x i8>, ptr undef, align 2
-; FIXED-MIN-2048-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv4i8 = load <4 x i8>, ptr undef, align 4
-; FIXED-MIN-2048-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv8i8 = load <8 x i8>, ptr undef, align 8
-; FIXED-MIN-2048-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv2i16 = load <2 x i16>, ptr undef, align 4
-; FIXED-MIN-2048-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv4i16 = load <4 x i16>, ptr undef, align 8
-; FIXED-MIN-2048-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv2i32 = load <2 x i32>, ptr undef, align 8
-; FIXED-MIN-2048-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadv4i32 = load <4 x i32>, ptr undef, align 16
-; FIXED-MIN-2048-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadnxv2i32 = load <vscale x 2 x i32>, ptr undef, align 8
-; FIXED-MIN-2048-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %loadnxv4i32 = load <vscale x 4 x i32>, ptr undef, align 16
+; FIXED-MIN-2048-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadi8 = load i8, ptr undef, align 1
+; FIXED-MIN-2048-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadi16 = load i16, ptr undef, align 2
+; FIXED-MIN-2048-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadi32 = load i32, ptr undef, align 4
+; FIXED-MIN-2048-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadv2i8 = load <2 x i8>, ptr undef, align 2
+; FIXED-MIN-2048-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadv4i8 = load <4 x i8>, ptr undef, align 4
+; FIXED-MIN-2048-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadv8i8 = load <8 x i8>, ptr undef, align 8
+; FIXED-MIN-2048-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadv2i16 = load <2 x i16>, ptr undef, align 4
+; FIXED-MIN-2048-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadv4i16 = load <4 x i16>, ptr undef, align 8
+; FIXED-MIN-2048-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadv2i32 = load <2 x i32>, ptr undef, align 8
+; FIXED-MIN-2048-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %loadv4i32 = load <4 x i32>, ptr undef, align 16
+; FIXED-MIN-2048-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %loadnxv2i32 = load <vscale x 2 x i32>, ptr undef, align 8
+; FIXED-MIN-2048-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %loadnxv4i32 = load <vscale x 4 x i32>, ptr undef, align 16
; FIXED-MIN-2048-NEXT: Cost Model: Found costs of 0 for: %r0 = sext i8 %loadi8 to i16
; FIXED-MIN-2048-NEXT: Cost Model: Found costs of 0 for: %r1 = zext i8 %loadi8 to i16
; FIXED-MIN-2048-NEXT: Cost Model: Found costs of 0 for: %r2 = sext i8 %loadi8 to i32
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
index a7bf82a62ea04..063a2bb707870 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
@@ -1108,7 +1108,7 @@ define void @fshl() #0 {
define <vscale x 4 x i32> @masked_gather_nxv4i32(<vscale x 4 x ptr> %ld, <vscale x 4 x i1> %masks, <vscale x 4 x i32> %passthru) {
; CHECK-VSCALE-1-LABEL: 'masked_gather_nxv4i32'
-; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:40 CodeSize:40 Lat:160 SizeLat:40 for: %res = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 %ld, <vscale x 4 x i1> %masks, <vscale x 4 x i32> %passthru)
+; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:40 CodeSize:40 Lat:80 SizeLat:40 for: %res = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 %ld, <vscale x 4 x i1> %masks, <vscale x 4 x i32> %passthru)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <vscale x 4 x i32> %res
;
; CHECK-VSCALE-2-LABEL: 'masked_gather_nxv4i32'
@@ -1125,7 +1125,7 @@ define <vscale x 4 x i32> @masked_gather_nxv4i32(<vscale x 4 x ptr> %ld, <vscale
define <vscale x 8 x i32> @masked_gather_nxv8i32(<vscale x 8 x ptr> %ld, <vscale x 8 x i1> %masks, <vscale x 8 x i32> %passthru) {
; CHECK-VSCALE-1-LABEL: 'masked_gather_nxv8i32'
-; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:80 CodeSize:80 Lat:320 SizeLat:80 for: %res = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0(<vscale x 8 x ptr> align 4 %ld, <vscale x 8 x i1> %masks, <vscale x 8 x i32> %passthru)
+; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:80 CodeSize:80 Lat:160 SizeLat:80 for: %res = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0(<vscale x 8 x ptr> align 4 %ld, <vscale x 8 x i1> %masks, <vscale x 8 x i32> %passthru)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <vscale x 8 x i32> %res
;
; CHECK-VSCALE-2-LABEL: 'masked_gather_nxv8i32'
@@ -1142,7 +1142,7 @@ define <vscale x 8 x i32> @masked_gather_nxv8i32(<vscale x 8 x ptr> %ld, <vscale
define <4 x i32> @masked_gather_v4i32(<4 x ptr> %ld, <4 x i1> %masks, <4 x i32> %passthru) {
; CHECK-VSCALE-1-LABEL: 'masked_gather_v4i32'
-; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:28 CodeSize:20 Lat:44 SizeLat:32 for: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 %ld, <4 x i1> %masks, <4 x i32> %passthru)
+; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:28 CodeSize:20 Lat:36 SizeLat:32 for: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 %ld, <4 x i1> %masks, <4 x i32> %passthru)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %res
;
; CHECK-VSCALE-2-LABEL: 'masked_gather_v4i32'
@@ -1159,11 +1159,11 @@ define <4 x i32> @masked_gather_v4i32(<4 x ptr> %ld, <4 x i1> %masks, <4 x i32>
define <1 x i128> @masked_gather_v1i128(<1 x ptr> %ld, <1 x i1> %masks, <1 x i128> %passthru) {
; CHECK-VSCALE-1-LABEL: 'masked_gather_v1i128'
-; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:8 CodeSize:6 Lat:11 SizeLat:9 for: %res = call <1 x i128> @llvm.masked.gather.v1i128.v1p0(<1 x ptr> align 16 %ld, <1 x i1> %masks, <1 x i128> %passthru)
+; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:8 CodeSize:6 Lat:10 SizeLat:9 for: %res = call <1 x i128> @llvm.masked.gather.v1i128.v1p0(<1 x ptr> align 16 %ld, <1 x i1> %masks, <1 x i128> %passthru)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <1 x i128> %res
;
; CHECK-VSCALE-2-LABEL: 'masked_gather_v1i128'
-; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:8 CodeSize:6 Lat:11 SizeLat:9 for: %res = call <1 x i128> @llvm.masked.gather.v1i128.v1p0(<1 x ptr> align 16 %ld, <1 x i1> %masks, <1 x i128> %passthru)
+; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:8 CodeSize:6 Lat:12 SizeLat:9 for: %res = call <1 x i128> @llvm.masked.gather.v1i128.v1p0(<1 x ptr> align 16 %ld, <1 x i1> %masks, <1 x i128> %passthru)
; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <1 x i128> %res
;
; TYPE_BASED_ONLY-LABEL: 'masked_gather_v1i128'
@@ -1316,7 +1316,7 @@ define void @histogram_nxv16i8(<vscale x 16 x ptr> %buckets, <vscale x 16 x i1>
define void @histogram_v2i64(<2 x ptr> %buckets, <2 x i1> %mask) {
; CHECK-VSCALE-1-LABEL: 'histogram_v2i64'
-; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:10 CodeSize:8 Lat:16 SizeLat:10 for: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask)
+; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:10 CodeSize:8 Lat:12 SizeLat:10 for: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; CHECK-VSCALE-2-LABEL: 'histogram_v2i64'
@@ -1324,7 +1324,7 @@ define void @histogram_v2i64(<2 x ptr> %buckets, <2 x i1> %mask) {
; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; TYPE_BASED_ONLY-LABEL: 'histogram_v2i64'
-; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:10 CodeSize:8 Lat:16 SizeLat:10 for: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask)
+; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:10 CodeSize:8 Lat:12 SizeLat:10 for: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask)
; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask)
@@ -1333,7 +1333,7 @@ define void @histogram_v2i64(<2 x ptr> %buckets, <2 x i1> %mask) {
define void @histogram_v4i32(<4 x ptr> %buckets, <4 x i1> %mask) {
; CHECK-VSCALE-1-LABEL: 'histogram_v4i32'
-; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:20 CodeSize:16 Lat:32 SizeLat:20 for: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
+; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:20 CodeSize:16 Lat:24 SizeLat:20 for: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; CHECK-VSCALE-2-LABEL: 'histogram_v4i32'
@@ -1341,7 +1341,7 @@ define void @histogram_v4i32(<4 x ptr> %buckets, <4 x i1> %mask) {
; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; TYPE_BASED_ONLY-LABEL: 'histogram_v4i32'
-; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:20 CodeSize:16 Lat:32 SizeLat:20 for: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
+; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:20 CodeSize:16 Lat:24 SizeLat:20 for: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
@@ -1350,7 +1350,7 @@ define void @histogram_v4i32(<4 x ptr> %buckets, <4 x i1> %mask) {
define void @histogram_v8i16(<8 x ptr> %buckets, <8 x i1> %mask) {
; CHECK-VSCALE-1-LABEL: 'histogram_v8i16'
-; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:40 CodeSize:32 Lat:64 SizeLat:40 for: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask)
+; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:40 CodeSize:32 Lat:48 SizeLat:40 for: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; CHECK-VSCALE-2-LABEL: 'histogram_v8i16'
@@ -1358,7 +1358,7 @@ define void @histogram_v8i16(<8 x ptr> %buckets, <8 x i1> %mask) {
; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; TYPE_BASED_ONLY-LABEL: 'histogram_v8i16'
-; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:40 CodeSize:32 Lat:64 SizeLat:40 for: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask)
+; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:40 CodeSize:32 Lat:48 SizeLat:40 for: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask)
; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask)
@@ -1367,7 +1367,7 @@ define void @histogram_v8i16(<8 x ptr> %buckets, <8 x i1> %mask) {
define void @histogram_v16i8(<16 x ptr> %buckets, <16 x i1> %mask) {
; CHECK-VSCALE-1-LABEL: 'histogram_v16i8'
-; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:80 CodeSize:64 Lat:128 SizeLat:80 for: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> %buckets, i8 1, <16 x i1> %mask)
+; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:80 CodeSize:64 Lat:96 SizeLat:80 for: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> %buckets, i8 1, <16 x i1> %mask)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; CHECK-VSCALE-2-LABEL: 'histogram_v16i8'
@@ -1375,7 +1375,7 @@ define void @histogram_v16i8(<16 x ptr> %buckets, <16 x i1> %mask) {
; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; TYPE_BASED_ONLY-LABEL: 'histogram_v16i8'
-; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:80 CodeSize:64 Lat:128 SizeLat:80 for: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> %buckets, i8 1, <16 x i1> %mask)
+; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:80 CodeSize:64 Lat:96 SizeLat:80 for: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> %buckets, i8 1, <16 x i1> %mask)
; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
call void @llvm.experimental.vector.histogram.add.v16p0.i64(<16 x ptr> %buckets, i8 1, <16 x i1> %mask)
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-ldst.ll b/llvm/test/Analysis/CostModel/AArch64/sve-ldst.ll
index 1df82b583a449..97dd09d144242 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-ldst.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-ldst.ll
@@ -5,13 +5,13 @@ target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
define void @scalable_loads() {
; CHECK-LABEL: 'scalable_loads'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %res.nxv8i8 = load <vscale x 8 x i8>, ptr undef, align 8
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %res.nxv16i8 = load <vscale x 16 x i8>, ptr undef, align 16
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %res.nxv8i8 = load <vscale x 8 x i8>, ptr undef, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %res.nxv16i8 = load <vscale x 16 x i8>, ptr undef, align 16
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %res.nxv32i8 = load <vscale x 32 x i8>, ptr undef, align 32
-; CHECK-NEXT: Cost Model: Found costs of RThru:Invalid CodeSize:Invalid Lat:4 SizeLat:Invalid for: %res.nxv1i64 = load <vscale x 1 x i64>, ptr undef, align 8
+; CHECK-NEXT: Cost Model: Found costs of Invalid for: %res.nxv1i64 = load <vscale x 1 x i64>, ptr undef, align 8
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %res.nxv32i1 = load <vscale x 32 x i1>, ptr undef, align 4
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %res.nxv16i1 = load <vscale x 16 x i1>, ptr undef, align 2
-; CHECK-NEXT: Cost Model: Found costs of RThru:Invalid CodeSize:Invalid Lat:4 SizeLat:Invalid for: %res.nxv4i1 = load <vscale x 4 x i1>, ptr undef, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %res.nxv16i1 = load <vscale x 16 x i1>, ptr undef, align 2
+; CHECK-NEXT: Cost Model: Found costs of Invalid for: %res.nxv4i1 = load <vscale x 4 x i1>, ptr undef, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%res.nxv8i8 = load <vscale x 8 x i8>, ptr undef
@@ -47,57 +47,57 @@ define void @scalable_stores() {
define void @scalable_ext_loads() {
; CHECK-LABEL: 'scalable_ext_loads'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load.nxv16i8 = load <vscale x 16 x i8>, ptr undef, align 16
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %load.nxv16i8 = load <vscale x 16 x i8>, ptr undef, align 16
; CHECK-NEXT: Cost Model: Found costs of 2 for: %zext.nxv16i8to16 = zext <vscale x 16 x i8> %load.nxv16i8 to <vscale x 16 x i16>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load.nxv8i8 = load <vscale x 8 x i8>, ptr undef, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %load.nxv8i8 = load <vscale x 8 x i8>, ptr undef, align 8
; CHECK-NEXT: Cost Model: Found costs of 0 for: %zext.nxv8i8to16 = zext <vscale x 8 x i8> %load.nxv8i8 to <vscale x 8 x i16>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load.nxv4i8 = load <vscale x 4 x i8>, ptr undef, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %load.nxv4i8 = load <vscale x 4 x i8>, ptr undef, align 4
; CHECK-NEXT: Cost Model: Found costs of 0 for: %zext.nxv4i8to32 = zext <vscale x 4 x i8> %load.nxv4i8 to <vscale x 4 x i32>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load.nxv2i8 = load <vscale x 2 x i8>, ptr undef, align 2
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %load.nxv2i8 = load <vscale x 2 x i8>, ptr undef, align 2
; CHECK-NEXT: Cost Model: Found costs of 0 for: %zext.nxv2i8to64 = zext <vscale x 2 x i8> %load.nxv2i8 to <vscale x 2 x i64>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load.nxv8i16 = load <vscale x 8 x i16>, ptr undef, align 16
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %load.nxv8i16 = load <vscale x 8 x i16>, ptr undef, align 16
; CHECK-NEXT: Cost Model: Found costs of 2 for: %zext.nxv8i16to32 = zext <vscale x 8 x i16> %load.nxv8i16 to <vscale x 8 x i32>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load.nxv4i16 = load <vscale x 4 x i16>, ptr undef, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %load.nxv4i16 = load <vscale x 4 x i16>, ptr undef, align 8
; CHECK-NEXT: Cost Model: Found costs of 0 for: %zext.nxv4i16to32 = zext <vscale x 4 x i16> %load.nxv4i16 to <vscale x 4 x i32>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load.nxv2i16 = load <vscale x 2 x i16>, ptr undef, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %load.nxv2i16 = load <vscale x 2 x i16>, ptr undef, align 4
; CHECK-NEXT: Cost Model: Found costs of 0 for: %zext.nxv2i16to64 = zext <vscale x 2 x i16> %load.nxv2i16 to <vscale x 2 x i64>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load.nxv4i32 = load <vscale x 4 x i32>, ptr undef, align 16
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %load.nxv4i32 = load <vscale x 4 x i32>, ptr undef, align 16
; CHECK-NEXT: Cost Model: Found costs of 2 for: %zext.nxv4i32to64 = zext <vscale x 4 x i32> %load.nxv4i32 to <vscale x 4 x i64>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load.nxv2i32 = load <vscale x 2 x i32>, ptr undef, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %load.nxv2i32 = load <vscale x 2 x i32>, ptr undef, align 8
; CHECK-NEXT: Cost Model: Found costs of 0 for: %zext.nxv2i32to64 = zext <vscale x 2 x i32> %load.nxv2i32 to <vscale x 2 x i64>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load.nxv4i8.2 = load <vscale x 4 x i8>, ptr undef, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %load.nxv4i8.2 = load <vscale x 4 x i8>, ptr undef, align 4
; CHECK-NEXT: Cost Model: Found costs of 1 for: %zext.nxv4i8to64 = zext <vscale x 4 x i8> %load.nxv4i8.2 to <vscale x 4 x i64>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load.nxv4i16.2 = load <vscale x 4 x i16>, ptr undef, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %load.nxv4i16.2 = load <vscale x 4 x i16>, ptr undef, align 8
; CHECK-NEXT: Cost Model: Found costs of 1 for: %zext.nxv4i16to64 = zext <vscale x 4 x i16> %load.nxv4i16.2 to <vscale x 4 x i64>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load.nxv8i8.2 = load <vscale x 8 x i8>, ptr undef, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %load.nxv8i8.2 = load <vscale x 8 x i8>, ptr undef, align 8
; CHECK-NEXT: Cost Model: Found costs of 1 for: %zext.nxv8i8to32 = zext <vscale x 8 x i8> %load.nxv8i8.2 to <vscale x 8 x i32>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load.nxv8i8.3 = load <vscale x 8 x i8>, ptr undef, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %load.nxv8i8.3 = load <vscale x 8 x i8>, ptr undef, align 8
; CHECK-NEXT: Cost Model: Found costs of 3 for: %zext.nxv8i8to64 = zext <vscale x 8 x i8> %load.nxv8i8.3 to <vscale x 8 x i64>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load2.nxv16i8 = load <vscale x 16 x i8>, ptr undef, align 16
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %load2.nxv16i8 = load <vscale x 16 x i8>, ptr undef, align 16
; CHECK-NEXT: Cost Model: Found costs of 2 for: %sext.nxv16i8to16 = sext <vscale x 16 x i8> %load2.nxv16i8 to <vscale x 16 x i16>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load2.nxv8i8 = load <vscale x 8 x i8>, ptr undef, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %load2.nxv8i8 = load <vscale x 8 x i8>, ptr undef, align 8
; CHECK-NEXT: Cost Model: Found costs of 0 for: %sext.nxv8i8to16 = sext <vscale x 8 x i8> %load2.nxv8i8 to <vscale x 8 x i16>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load2.nxv4i8 = load <vscale x 4 x i8>, ptr undef, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %load2.nxv4i8 = load <vscale x 4 x i8>, ptr undef, align 4
; CHECK-NEXT: Cost Model: Found costs of 0 for: %sext.nxv4i8to32 = sext <vscale x 4 x i8> %load2.nxv4i8 to <vscale x 4 x i32>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load2.nxv2i8 = load <vscale x 2 x i8>, ptr undef, align 2
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %load2.nxv2i8 = load <vscale x 2 x i8>, ptr undef, align 2
; CHECK-NEXT: Cost Model: Found costs of 0 for: %sext.nxv2i8to64 = sext <vscale x 2 x i8> %load2.nxv2i8 to <vscale x 2 x i64>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load2.nxv8i16 = load <vscale x 8 x i16>, ptr undef, align 16
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %load2.nxv8i16 = load <vscale x 8 x i16>, ptr undef, align 16
; CHECK-NEXT: Cost Model: Found costs of 2 for: %sext.nxv8i16to32 = sext <vscale x 8 x i16> %load2.nxv8i16 to <vscale x 8 x i32>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load2.nxv4i16 = load <vscale x 4 x i16>, ptr undef, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %load2.nxv4i16 = load <vscale x 4 x i16>, ptr undef, align 8
; CHECK-NEXT: Cost Model: Found costs of 0 for: %sext.nxv4i16to32 = sext <vscale x 4 x i16> %load2.nxv4i16 to <vscale x 4 x i32>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load2.nxv2i16 = load <vscale x 2 x i16>, ptr undef, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %load2.nxv2i16 = load <vscale x 2 x i16>, ptr undef, align 4
; CHECK-NEXT: Cost Model: Found costs of 0 for: %sext.nxv2i16to64 = sext <vscale x 2 x i16> %load2.nxv2i16 to <vscale x 2 x i64>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load2.nxv4i32 = load <vscale x 4 x i32>, ptr undef, align 16
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %load2.nxv4i32 = load <vscale x 4 x i32>, ptr undef, align 16
; CHECK-NEXT: Cost Model: Found costs of 2 for: %sext.nxv4i32to64 = sext <vscale x 4 x i32> %load2.nxv4i32 to <vscale x 4 x i64>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load2.nxv2i32 = load <vscale x 2 x i32>, ptr undef, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %load2.nxv2i32 = load <vscale x 2 x i32>, ptr undef, align 8
; CHECK-NEXT: Cost Model: Found costs of 0 for: %sext.nxv2i32to64 = sext <vscale x 2 x i32> %load2.nxv2i32 to <vscale x 2 x i64>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load2.nxv4i8.2 = load <vscale x 4 x i8>, ptr undef, align 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %load2.nxv4i8.2 = load <vscale x 4 x i8>, ptr undef, align 4
; CHECK-NEXT: Cost Model: Found costs of 1 for: %sext.nxv4i8to64 = sext <vscale x 4 x i8> %load2.nxv4i8.2 to <vscale x 4 x i64>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load2.nxv4i16.2 = load <vscale x 4 x i16>, ptr undef, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %load2.nxv4i16.2 = load <vscale x 4 x i16>, ptr undef, align 8
; CHECK-NEXT: Cost Model: Found costs of 1 for: %sext.nxv4i16to64 = sext <vscale x 4 x i16> %load2.nxv4i16.2 to <vscale x 4 x i64>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load2.nxv8i8.2 = load <vscale x 8 x i8>, ptr undef, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %load2.nxv8i8.2 = load <vscale x 8 x i8>, ptr undef, align 8
; CHECK-NEXT: Cost Model: Found costs of 1 for: %sext.nxv8i8to32 = sext <vscale x 8 x i8> %load2.nxv8i8.2 to <vscale x 8 x i32>
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load2.nxv8i8.3 = load <vscale x 8 x i8>, ptr undef, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %load2.nxv8i8.3 = load <vscale x 8 x i8>, ptr undef, align 8
; CHECK-NEXT: Cost Model: Found costs of 3 for: %sext.nxv8i8to64 = sext <vscale x 8 x i8> %load2.nxv8i8.3 to <vscale x 8 x i64>
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
diff --git a/llvm/test/Analysis/CostModel/AArch64/vec3-ops.ll b/llvm/test/Analysis/CostModel/AArch64/vec3-ops.ll
index f234341b0fd86..0832678a2e8eb 100644
--- a/llvm/test/Analysis/CostModel/AArch64/vec3-ops.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/vec3-ops.ll
@@ -3,7 +3,7 @@
define void @vec3_i32(<3 x i32> %a, <3 x i32> %b, ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec3_i32'
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <3 x i32>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %l = load <3 x i32>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <3 x i32> %l, %b
; CHECK-NEXT: Cost Model: Found costs of 1 for: %cmp = icmp uge <3 x i32> %add, %a
; CHECK-NEXT: Cost Model: Found costs of 1 for: %sub = sub <3 x i32> %add, %a
@@ -22,7 +22,7 @@ define void @vec3_i32(<3 x i32> %a, <3 x i32> %b, ptr %src, ptr %dst) {
define void @vec3_i32_default_alignment(<3 x i32> %a, <3 x i32> %b, ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec3_i32_default_alignment'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <3 x i32>, ptr %src, align 16
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %l = load <3 x i32>, ptr %src, align 16
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <3 x i32> %l, %b
; CHECK-NEXT: Cost Model: Found costs of 1 for: %cmp = icmp uge <3 x i32> %add, %a
; CHECK-NEXT: Cost Model: Found costs of 1 for: %sub = sub <3 x i32> %add, %a
@@ -41,7 +41,7 @@ define void @vec3_i32_default_alignment(<3 x i32> %a, <3 x i32> %b, ptr %src, pt
define void @vec3_i16(<3 x i16> %a, <3 x i16> %b, ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec3_i16'
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <3 x i16>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %l = load <3 x i16>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <3 x i16> %l, %b
; CHECK-NEXT: Cost Model: Found costs of 1 for: %cmp = icmp uge <3 x i16> %add, %a
; CHECK-NEXT: Cost Model: Found costs of 1 for: %sub = sub <3 x i16> %add, %a
@@ -60,7 +60,7 @@ define void @vec3_i16(<3 x i16> %a, <3 x i16> %b, ptr %src, ptr %dst) {
define void @vec7_i16(ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec7_i16'
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <7 x i16>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:2 SizeLat:1 for: %l = load <7 x i16>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <7 x i16> %l, %l
; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <7 x i16> %add, ptr %dst, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
@@ -73,7 +73,7 @@ define void @vec7_i16(ptr %src, ptr %dst) {
define void @vec6_i16(ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec6_i16'
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <6 x i16>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %l = load <6 x i16>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <6 x i16> %l, %l
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <6 x i16> %add, ptr %dst, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
@@ -86,7 +86,7 @@ define void @vec6_i16(ptr %src, ptr %dst) {
define void @vec5_i16(ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec5_i16'
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <5 x i16>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %l = load <5 x i16>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <5 x i16> %l, %l
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <5 x i16> %add, ptr %dst, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
@@ -99,7 +99,7 @@ define void @vec5_i16(ptr %src, ptr %dst) {
define void @vec3_i16_zext_i32(<3 x i32> %a, <3 x i32> %b, ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec3_i16_zext_i32'
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <3 x i16>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %l = load <3 x i16>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %l.ext = zext <3 x i16> %l to <3 x i32>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <3 x i32> %l.ext, %b
; CHECK-NEXT: Cost Model: Found costs of 1 for: %sub = sub <3 x i32> %add, %a
@@ -118,7 +118,7 @@ define void @vec3_i16_zext_i32(<3 x i32> %a, <3 x i32> %b, ptr %src, ptr %dst) {
define void @vec3_i8(<3 x i8> %a, <3 x i8> %b, ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec3_i8'
-; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <3 x i8>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:1 Lat:2 SizeLat:1 for: %l = load <3 x i8>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <3 x i8> %l, %b
; CHECK-NEXT: Cost Model: Found costs of 1 for: %cmp = icmp uge <3 x i8> %add, %a
; CHECK-NEXT: Cost Model: Found costs of 1 for: %sub = sub <3 x i8> %add, %a
@@ -137,7 +137,7 @@ define void @vec3_i8(<3 x i8> %a, <3 x i8> %b, ptr %src, ptr %dst) {
define void @vec3_i8_zext_i32(<3 x i32> %a, <3 x i32> %b, ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec3_i8_zext_i32'
-; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <3 x i8>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:1 Lat:2 SizeLat:1 for: %l = load <3 x i8>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %l.ext = zext <3 x i8> %l to <3 x i32>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <3 x i32> %l.ext, %b
; CHECK-NEXT: Cost Model: Found costs of 1 for: %sub = sub <3 x i32> %add, %a
@@ -156,7 +156,7 @@ define void @vec3_i8_zext_i32(<3 x i32> %a, <3 x i32> %b, ptr %src, ptr %dst) {
define void @vec3_i8_sext_i32(<3 x i32> %a, <3 x i32> %b, ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec3_i8_sext_i32'
-; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <3 x i8>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:1 Lat:2 SizeLat:1 for: %l = load <3 x i8>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %l.ext = sext <3 x i8> %l to <3 x i32>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <3 x i32> %l.ext, %b
; CHECK-NEXT: Cost Model: Found costs of 1 for: %sub = sub <3 x i32> %add, %a
@@ -175,7 +175,7 @@ define void @vec3_i8_sext_i32(<3 x i32> %a, <3 x i32> %b, ptr %src, ptr %dst) {
define void @vec3_i30(<3 x i30> %a, <3 x i30> %b, ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec3_i30'
-; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <3 x i30>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:1 Lat:2 SizeLat:1 for: %l = load <3 x i30>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <3 x i30> %l, %b
; CHECK-NEXT: Cost Model: Found costs of 1 for: %sub = sub <3 x i30> %add, %a
; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: store <3 x i30> %sub, ptr %dst, align 1
@@ -190,7 +190,7 @@ define void @vec3_i30(<3 x i30> %a, <3 x i30> %b, ptr %src, ptr %dst) {
define void @vec3_float(<3 x float> %a, <3 x float> %b, ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec3_float'
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <3 x float>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %l = load <3 x float>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %add = fadd <3 x float> %l, %b
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %sub = fsub <3 x float> %add, %a
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <3 x float> %sub, ptr %dst, align 1
@@ -205,7 +205,7 @@ define void @vec3_float(<3 x float> %a, <3 x float> %b, ptr %src, ptr %dst) {
define void @vec3_half(<3 x half> %a, <3 x half> %b, ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec3_half'
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <3 x half>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %l = load <3 x half>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %add = fadd <3 x half> %l, %b
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %sub = fsub <3 x half> %add, %a
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <3 x half> %sub, ptr %dst, align 1
@@ -220,7 +220,7 @@ define void @vec3_half(<3 x half> %a, <3 x half> %b, ptr %src, ptr %dst) {
define void @vec15_i8(ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec15_i8'
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <15 x i8>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:2 SizeLat:1 for: %l = load <15 x i8>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <15 x i8> %l, %l
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: store <15 x i8> %add, ptr %dst, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
@@ -233,7 +233,7 @@ define void @vec15_i8(ptr %src, ptr %dst) {
define void @vec14_i8(ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec14_i8'
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <14 x i8>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:2 SizeLat:1 for: %l = load <14 x i8>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <14 x i8> %l, %l
; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <14 x i8> %add, ptr %dst, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
@@ -246,7 +246,7 @@ define void @vec14_i8(ptr %src, ptr %dst) {
define void @vec13_i8(ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec13_i8'
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <13 x i8>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:2 SizeLat:1 for: %l = load <13 x i8>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <13 x i8> %l, %l
; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <13 x i8> %add, ptr %dst, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
@@ -259,7 +259,7 @@ define void @vec13_i8(ptr %src, ptr %dst) {
define void @vec12_i8(ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec12_i8'
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <12 x i8>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %l = load <12 x i8>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <12 x i8> %l, %l
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <12 x i8> %add, ptr %dst, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
@@ -272,7 +272,7 @@ define void @vec12_i8(ptr %src, ptr %dst) {
define void @vec11_i8(ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec11_i8'
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <11 x i8>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:2 SizeLat:1 for: %l = load <11 x i8>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <11 x i8> %l, %l
; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <11 x i8> %add, ptr %dst, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
@@ -285,7 +285,7 @@ define void @vec11_i8(ptr %src, ptr %dst) {
define void @vec7_i8(<7 x i8> %a, <7 x i8> %b, ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec7_i8'
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <7 x i8>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:2 SizeLat:1 for: %l = load <7 x i8>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <7 x i8> %l, %b
; CHECK-NEXT: Cost Model: Found costs of 1 for: %sub = sub <7 x i8> %add, %a
; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: store <7 x i8> %sub, ptr %dst, align 1
@@ -300,7 +300,7 @@ define void @vec7_i8(<7 x i8> %a, <7 x i8> %b, ptr %src, ptr %dst) {
define void @vec6_i8(ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec6_i8'
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <6 x i8>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %l = load <6 x i8>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <6 x i8> %l, %l
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <6 x i8> %add, ptr %dst, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
@@ -313,7 +313,7 @@ define void @vec6_i8(ptr %src, ptr %dst) {
define void @vec5_i8(ptr %src, ptr %dst) {
; CHECK-LABEL: 'vec5_i8'
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <5 x i8>, ptr %src, align 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %l = load <5 x i8>, ptr %src, align 1
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add = add <5 x i8> %l, %l
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <5 x i8> %add, ptr %dst, align 1
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
diff --git a/llvm/test/CodeGen/AArch64/selectopt-cast.ll b/llvm/test/CodeGen/AArch64/selectopt-cast.ll
index 2ea9f94e8e4bc..b2c4868c9e989 100644
--- a/llvm/test/CodeGen/AArch64/selectopt-cast.ll
+++ b/llvm/test/CodeGen/AArch64/selectopt-cast.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals smart
-; RUN: opt -select-optimize -mtriple=arm64-apple-macosx -S %s | FileCheck %s
-; RUN: opt -passes='require<profile-summary>,function(select-optimize)' -mtriple=arm64-apple-macosx -S %s | FileCheck %s
+; RUN: opt -select-optimize -mtriple=arm64-apple-macosx -mcpu=cortex-a710 -S %s | FileCheck %s
+; RUN: opt -passes='require<profile-summary>,function(select-optimize)' -mtriple=arm64-apple-macosx -mcpu=cortex-a710 -S %s | FileCheck %s
define void @test_add_zext(ptr %dst, ptr %src, i64 %j.start, i64 %p, i64 %i.start) {
; CHECK-LABEL: @test_add_zext(
diff --git a/llvm/test/DebugInfo/AArch64/select-optimize-trailing-dbg-records.ll b/llvm/test/DebugInfo/AArch64/select-optimize-trailing-dbg-records.ll
index 610315dd8b31f..8a21cd89b7fc7 100644
--- a/llvm/test/DebugInfo/AArch64/select-optimize-trailing-dbg-records.ll
+++ b/llvm/test/DebugInfo/AArch64/select-optimize-trailing-dbg-records.ll
@@ -1,4 +1,4 @@
-; RUN: opt %s -passes='require<profile-summary>,function(select-optimize)' -o - -S \
+; RUN: opt %s -passes='require<profile-summary>,function(select-optimize)' -mcpu=cortex-a710 -o - -S \
; RUN: | FileCheck %s
;; Check that the dbg.value is moved into the start of the end-block of the
More information about the llvm-commits
mailing list