[llvm] [AArch64] Adjust costs for masked CompressStore and ExpandLoad (PR #226950)

Thu Oct 1 01:20:55 PDT 2026

https://github.com/Stylie777 updated https://github.com/llvm/llvm-project/pull/226950

>From d9fb183ccfe1a56041b15aba8fcea9d3f035f504 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Fri, 25 Sep 2026 13:52:35 +0100
Subject: [PATCH 1/4] [AArch64] Adjust costs for masked CompressStore and
 ExpandLoad

Currently, costs for the `llvm.expand.compressstore` and
`llvm.masked.expandload` are costs simply at 2 x the type legalization
cost. However, this does not properly represent the instructions that
are generated for these intrinsics.

The costs have been adjusted to better represent the cost of using
these intrinsics with SVE Instructions.
---
 .../AArch64/AArch64TargetTransformInfo.cpp    |  10 +-
 .../CostModel/AArch64/masked_compress_load.ll | 170 +++++++-------
 .../CostModel/AArch64/masked_expand_load.ll   | 216 +++++++++---------
 3 files changed, 201 insertions(+), 195 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 581f37d4c17bce2..fe849df4e448872 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5563,7 +5563,13 @@ AArch64TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
       return InstructionCost::getInvalid();
 
     // Operation will be split into expand of masked.load
-    MemOpCost *= 2;
+    // Something like:
+    // cntp x8, p0, p0.s
+    // whilelo p1.s, xzr, x8
+    // ld1w { z1.s }, p1/z, [x0]
+    // expand z1.s, p0, z1.s
+    // mov z0.s, p0/m, z1.s1
+    MemOpCost *= 4;
   }
 
   if (MICA.getID() == Intrinsic::masked_compressstore) {
@@ -5576,7 +5582,7 @@ AArch64TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
     //  cntp     x8, p1, p0.s
     //  whilelo  p0.s, xzr, x8
     //  st1w     { z0.s }, p0, [x0]
-    MemOpCost *= 2;
+    MemOpCost *= 4;
   }
 
   // If we need to split the memory operation, we will also need to split the
diff --git a/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll b/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll
index fe7357097200455..941eee99a943399 100644
--- a/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll
@@ -54,42 +54,42 @@ define void @fixed() {
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i8.p0(<2 x i8> poison, ptr poison, <2 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4i8.p0(<4 x i8> poison, ptr poison, <4 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v8i8.p0(<8 x i8> poison, ptr poison, <8 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v16i8.p0(<16 x i8> poison, ptr poison, <16 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v16i8.p0(<16 x i8> poison, ptr poison, <16 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i16.p0(<2 x i16> poison, ptr poison, <2 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4i16.p0(<4 x i16> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v8i16.p0(<8 x i16> poison, ptr poison, <8 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v8i16.p0(<8 x i16> poison, ptr poison, <8 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i32.p0(<2 x i32> poison, ptr poison, <2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v2i64.p0(<2 x i64> poison, ptr poison, <2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> poison, ptr poison, <4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v2i64.p0(<2 x i64> poison, ptr poison, <2 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2f16.p0(<2 x half> poison, ptr poison, <2 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4f16.p0(<4 x half> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v8f16.p0(<8 x half> poison, ptr poison, <8 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v8f16.p0(<8 x half> poison, ptr poison, <8 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2f32.p0(<2 x float> poison, ptr poison, <2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v4f32.p0(<4 x float> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v2f64.p0(<2 x double> poison, ptr poison, <2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 8 for: call void @llvm.masked.compressstore.v4i64.p0(<4 x i64> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.v32f16.p0(<32 x half> poison, ptr poison, <32 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v4f32.p0(<4 x float> poison, ptr poison, <4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v2f64.p0(<2 x double> poison, ptr poison, <2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.v4i64.p0(<4 x i64> poison, ptr poison, <4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 32 for: call void @llvm.masked.compressstore.v32f16.p0(<32 x half> poison, ptr poison, <32 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-8-64-SVE256-LABEL: 'fixed'
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i8.p0(<2 x i8> poison, ptr poison, <2 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4i8.p0(<4 x i8> poison, ptr poison, <4 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v8i8.p0(<8 x i8> poison, ptr poison, <8 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v16i8.p0(<16 x i8> poison, ptr poison, <16 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v16i8.p0(<16 x i8> poison, ptr poison, <16 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i16.p0(<2 x i16> poison, ptr poison, <2 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4i16.p0(<4 x i16> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v8i16.p0(<8 x i16> poison, ptr poison, <8 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v8i16.p0(<8 x i16> poison, ptr poison, <8 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i32.p0(<2 x i32> poison, ptr poison, <2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v2i64.p0(<2 x i64> poison, ptr poison, <2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> poison, ptr poison, <4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v2i64.p0(<2 x i64> poison, ptr poison, <2 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2f16.p0(<2 x half> poison, ptr poison, <2 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4f16.p0(<4 x half> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v8f16.p0(<8 x half> poison, ptr poison, <8 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v8f16.p0(<8 x half> poison, ptr poison, <8 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2f32.p0(<2 x float> poison, ptr poison, <2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v4f32.p0(<4 x float> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v2f64.p0(<2 x double> poison, ptr poison, <2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v4i64.p0(<4 x i64> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 8 for: call void @llvm.masked.compressstore.v32f16.p0(<32 x half> poison, ptr poison, <32 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v4f32.p0(<4 x float> poison, ptr poison, <4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v2f64.p0(<2 x double> poison, ptr poison, <2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v4i64.p0(<4 x i64> poison, ptr poison, <4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.v32f16.p0(<32 x half> poison, ptr poison, <32 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-UNAVAILABLE-LABEL: 'fixed'
@@ -143,91 +143,91 @@ entry:
 
 define void @scalable() {
 ; CHECK-32-64-LABEL: 'scalable'
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
 ; CHECK-32-64-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
 ; CHECK-32-64-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
 ; CHECK-32-64-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
 ; CHECK-32-64-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
 ; CHECK-32-64-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv1i64.p0(<vscale x 1 x i64> poison, ptr poison, <vscale x 1 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 8 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
 ; CHECK-32-64-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
 ; CHECK-32-64-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-8-64-LABEL: 'scalable'
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
 ; CHECK-8-64-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv1i64.p0(<vscale x 1 x i64> poison, ptr poison, <vscale x 1 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 8 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 32 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
 ; CHECK-8-64-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-8-64-STREAMING-ONLY-LABEL: 'scalable'
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv1i64.p0(<vscale x 1 x i64> poison, ptr poison, <vscale x 1 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 8 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 32 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-8-64-SVE256-LABEL: 'scalable'
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv1i64.p0(<vscale x 1 x i64> poison, ptr poison, <vscale x 1 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 8 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 32 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-UNAVAILABLE-LABEL: 'scalable'
diff --git a/llvm/test/Analysis/CostModel/AArch64/masked_expand_load.ll b/llvm/test/Analysis/CostModel/AArch64/masked_expand_load.ll
index 342b4b31abbd327..1a477a52acd15bd 100644
--- a/llvm/test/Analysis/CostModel/AArch64/masked_expand_load.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/masked_expand_load.ll
@@ -29,24 +29,24 @@ define void @fixed() {
 ; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; SVE2p2-SME2p2-STREAMING-LABEL: 'fixed'
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %v2i8 = call <2 x i8> @llvm.masked.expandload.v2i8.p0(ptr poison, <2 x i1> poison, <2 x i8> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %v4i8 = call <4 x i8> @llvm.masked.expandload.v4i8.p0(ptr poison, <4 x i1> poison, <4 x i8> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %v8i8 = call <8 x i8> @llvm.masked.expandload.v8i8.p0(ptr poison, <8 x i1> poison, <8 x i8> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %v16i8 = call <16 x i8> @llvm.masked.expandload.v16i8.p0(ptr poison, <16 x i1> poison, <16 x i8> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %v2i16 = call <2 x i16> @llvm.masked.expandload.v2i16.p0(ptr poison, <2 x i1> poison, <2 x i16> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %v4i16 = call <4 x i16> @llvm.masked.expandload.v4i16.p0(ptr poison, <4 x i1> poison, <4 x i16> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %v8i16 = call <8 x i16> @llvm.masked.expandload.v8i16.p0(ptr poison, <8 x i1> poison, <8 x i16> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %v2i32 = call <2 x i32> @llvm.masked.expandload.v2i32.p0(ptr poison, <2 x i1> poison, <2 x i32> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %v4i32 = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr poison, <4 x i1> poison, <4 x i32> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %v2i64 = call <2 x i64> @llvm.masked.expandload.v2i64.p0(ptr poison, <2 x i1> poison, <2 x i64> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %v2f16 = call <2 x half> @llvm.masked.expandload.v2f16.p0(ptr poison, <2 x i1> poison, <2 x half> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %v4f16 = call <4 x half> @llvm.masked.expandload.v4f16.p0(ptr poison, <4 x i1> poison, <4 x half> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %v8f16 = call <8 x half> @llvm.masked.expandload.v8f16.p0(ptr poison, <8 x i1> poison, <8 x half> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %v2f32 = call <2 x float> @llvm.masked.expandload.v2f32.p0(ptr poison, <2 x i1> poison, <2 x float> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %v4f32 = call <4 x float> @llvm.masked.expandload.v4f32.p0(ptr poison, <4 x i1> poison, <4 x float> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %v2f64 = call <2 x double> @llvm.masked.expandload.v2f64.p0(ptr poison, <2 x i1> poison, <2 x double> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 8 for: %v4i64 = call <4 x i64> @llvm.masked.expandload.v4i64.p0(ptr poison, <4 x i1> poison, <4 x i64> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 16 for: %v32f16 = call <32 x half> @llvm.masked.expandload.v32f16.p0(ptr poison, <32 x i1> poison, <32 x half> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v2i8 = call <2 x i8> @llvm.masked.expandload.v2i8.p0(ptr poison, <2 x i1> poison, <2 x i8> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v4i8 = call <4 x i8> @llvm.masked.expandload.v4i8.p0(ptr poison, <4 x i1> poison, <4 x i8> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v8i8 = call <8 x i8> @llvm.masked.expandload.v8i8.p0(ptr poison, <8 x i1> poison, <8 x i8> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v16i8 = call <16 x i8> @llvm.masked.expandload.v16i8.p0(ptr poison, <16 x i1> poison, <16 x i8> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v2i16 = call <2 x i16> @llvm.masked.expandload.v2i16.p0(ptr poison, <2 x i1> poison, <2 x i16> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v4i16 = call <4 x i16> @llvm.masked.expandload.v4i16.p0(ptr poison, <4 x i1> poison, <4 x i16> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v8i16 = call <8 x i16> @llvm.masked.expandload.v8i16.p0(ptr poison, <8 x i1> poison, <8 x i16> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v2i32 = call <2 x i32> @llvm.masked.expandload.v2i32.p0(ptr poison, <2 x i1> poison, <2 x i32> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v4i32 = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr poison, <4 x i1> poison, <4 x i32> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v2i64 = call <2 x i64> @llvm.masked.expandload.v2i64.p0(ptr poison, <2 x i1> poison, <2 x i64> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v2f16 = call <2 x half> @llvm.masked.expandload.v2f16.p0(ptr poison, <2 x i1> poison, <2 x half> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v4f16 = call <4 x half> @llvm.masked.expandload.v4f16.p0(ptr poison, <4 x i1> poison, <4 x half> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v8f16 = call <8 x half> @llvm.masked.expandload.v8f16.p0(ptr poison, <8 x i1> poison, <8 x half> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v2f32 = call <2 x float> @llvm.masked.expandload.v2f32.p0(ptr poison, <2 x i1> poison, <2 x float> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v4f32 = call <4 x float> @llvm.masked.expandload.v4f32.p0(ptr poison, <4 x i1> poison, <4 x float> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v2f64 = call <2 x double> @llvm.masked.expandload.v2f64.p0(ptr poison, <2 x i1> poison, <2 x double> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 16 for: %v4i64 = call <4 x i64> @llvm.masked.expandload.v4i64.p0(ptr poison, <4 x i1> poison, <4 x i64> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 32 for: %v32f16 = call <32 x half> @llvm.masked.expandload.v32f16.p0(ptr poison, <32 x i1> poison, <32 x half> poison)
 ; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; SVE2p2-SME2p2-LABEL: 'fixed'
@@ -92,24 +92,24 @@ define void @fixed() {
 ; SVE-ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; SVE2p2-SME2p2-SVE256-LABEL: 'fixed'
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %v2i8 = call <2 x i8> @llvm.masked.expandload.v2i8.p0(ptr poison, <2 x i1> poison, <2 x i8> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %v4i8 = call <4 x i8> @llvm.masked.expandload.v4i8.p0(ptr poison, <4 x i1> poison, <4 x i8> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %v8i8 = call <8 x i8> @llvm.masked.expandload.v8i8.p0(ptr poison, <8 x i1> poison, <8 x i8> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %v16i8 = call <16 x i8> @llvm.masked.expandload.v16i8.p0(ptr poison, <16 x i1> poison, <16 x i8> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %v2i16 = call <2 x i16> @llvm.masked.expandload.v2i16.p0(ptr poison, <2 x i1> poison, <2 x i16> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %v4i16 = call <4 x i16> @llvm.masked.expandload.v4i16.p0(ptr poison, <4 x i1> poison, <4 x i16> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %v8i16 = call <8 x i16> @llvm.masked.expandload.v8i16.p0(ptr poison, <8 x i1> poison, <8 x i16> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %v2i32 = call <2 x i32> @llvm.masked.expandload.v2i32.p0(ptr poison, <2 x i1> poison, <2 x i32> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %v4i32 = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr poison, <4 x i1> poison, <4 x i32> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %v2i64 = call <2 x i64> @llvm.masked.expandload.v2i64.p0(ptr poison, <2 x i1> poison, <2 x i64> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %v2f16 = call <2 x half> @llvm.masked.expandload.v2f16.p0(ptr poison, <2 x i1> poison, <2 x half> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %v4f16 = call <4 x half> @llvm.masked.expandload.v4f16.p0(ptr poison, <4 x i1> poison, <4 x half> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %v8f16 = call <8 x half> @llvm.masked.expandload.v8f16.p0(ptr poison, <8 x i1> poison, <8 x half> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %v2f32 = call <2 x float> @llvm.masked.expandload.v2f32.p0(ptr poison, <2 x i1> poison, <2 x float> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %v4f32 = call <4 x float> @llvm.masked.expandload.v4f32.p0(ptr poison, <4 x i1> poison, <4 x float> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %v2f64 = call <2 x double> @llvm.masked.expandload.v2f64.p0(ptr poison, <2 x i1> poison, <2 x double> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %v4i64 = call <4 x i64> @llvm.masked.expandload.v4i64.p0(ptr poison, <4 x i1> poison, <4 x i64> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 8 for: %v32f16 = call <32 x half> @llvm.masked.expandload.v32f16.p0(ptr poison, <32 x i1> poison, <32 x half> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v2i8 = call <2 x i8> @llvm.masked.expandload.v2i8.p0(ptr poison, <2 x i1> poison, <2 x i8> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v4i8 = call <4 x i8> @llvm.masked.expandload.v4i8.p0(ptr poison, <4 x i1> poison, <4 x i8> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v8i8 = call <8 x i8> @llvm.masked.expandload.v8i8.p0(ptr poison, <8 x i1> poison, <8 x i8> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v16i8 = call <16 x i8> @llvm.masked.expandload.v16i8.p0(ptr poison, <16 x i1> poison, <16 x i8> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v2i16 = call <2 x i16> @llvm.masked.expandload.v2i16.p0(ptr poison, <2 x i1> poison, <2 x i16> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v4i16 = call <4 x i16> @llvm.masked.expandload.v4i16.p0(ptr poison, <4 x i1> poison, <4 x i16> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v8i16 = call <8 x i16> @llvm.masked.expandload.v8i16.p0(ptr poison, <8 x i1> poison, <8 x i16> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v2i32 = call <2 x i32> @llvm.masked.expandload.v2i32.p0(ptr poison, <2 x i1> poison, <2 x i32> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v4i32 = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr poison, <4 x i1> poison, <4 x i32> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v2i64 = call <2 x i64> @llvm.masked.expandload.v2i64.p0(ptr poison, <2 x i1> poison, <2 x i64> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v2f16 = call <2 x half> @llvm.masked.expandload.v2f16.p0(ptr poison, <2 x i1> poison, <2 x half> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v4f16 = call <4 x half> @llvm.masked.expandload.v4f16.p0(ptr poison, <4 x i1> poison, <4 x half> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v8f16 = call <8 x half> @llvm.masked.expandload.v8f16.p0(ptr poison, <8 x i1> poison, <8 x half> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v2f32 = call <2 x float> @llvm.masked.expandload.v2f32.p0(ptr poison, <2 x i1> poison, <2 x float> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v4f32 = call <4 x float> @llvm.masked.expandload.v4f32.p0(ptr poison, <4 x i1> poison, <4 x float> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v2f64 = call <2 x double> @llvm.masked.expandload.v2f64.p0(ptr poison, <2 x i1> poison, <2 x double> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v4i64 = call <4 x i64> @llvm.masked.expandload.v4i64.p0(ptr poison, <4 x i1> poison, <4 x i64> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 16 for: %v32f16 = call <32 x half> @llvm.masked.expandload.v32f16.p0(ptr poison, <32 x i1> poison, <32 x half> poison)
 ; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 entry:
@@ -142,69 +142,69 @@ entry:
 
 define void @scalable() {
 ; SVE2p2-SME2p2-NON-STREAMING-LABEL: 'scalable'
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv2i8 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv4i8 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv8i8 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv16i8 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8.p0(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv2i16 = call <vscale x 2 x i16> @llvm.masked.expandload.nxv2i16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i16> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv4i16 = call <vscale x 4 x i16> @llvm.masked.expandload.nxv4i16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i16> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.masked.expandload.nxv8i16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i16> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv2i32 = call <vscale x 2 x i32> @llvm.masked.expandload.nxv2i32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i32> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.masked.expandload.nxv4i32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i32> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv2f16 = call <vscale x 2 x half> @llvm.masked.expandload.nxv2f16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x half> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv4f16 = call <vscale x 4 x half> @llvm.masked.expandload.nxv4f16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x half> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv8f16 = call <vscale x 8 x half> @llvm.masked.expandload.nxv8f16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x half> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.expandload.nxv2f32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x float> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2i8 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4i8 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv8i8 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv16i8 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8.p0(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2i16 = call <vscale x 2 x i16> @llvm.masked.expandload.nxv2i16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i16> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4i16 = call <vscale x 4 x i16> @llvm.masked.expandload.nxv4i16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i16> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.masked.expandload.nxv8i16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i16> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2i32 = call <vscale x 2 x i32> @llvm.masked.expandload.nxv2i32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i32> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.masked.expandload.nxv4i32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i32> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2f16 = call <vscale x 2 x half> @llvm.masked.expandload.nxv2f16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x half> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4f16 = call <vscale x 4 x half> @llvm.masked.expandload.nxv4f16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x half> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv8f16 = call <vscale x 8 x half> @llvm.masked.expandload.nxv8f16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x half> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.expandload.nxv2f32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x float> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
 ; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of Invalid for: %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.expandload.nxv1i64.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x i64> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 8 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 16 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 16 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 32 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
 ; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; SVE2p2-SME2p2-STREAMING-LABEL: 'scalable'
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv2i8 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv4i8 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv8i8 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv16i8 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8.p0(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv2i16 = call <vscale x 2 x i16> @llvm.masked.expandload.nxv2i16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i16> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv4i16 = call <vscale x 4 x i16> @llvm.masked.expandload.nxv4i16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i16> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.masked.expandload.nxv8i16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i16> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv2i32 = call <vscale x 2 x i32> @llvm.masked.expandload.nxv2i32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i32> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.masked.expandload.nxv4i32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i32> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv2f16 = call <vscale x 2 x half> @llvm.masked.expandload.nxv2f16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x half> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv4f16 = call <vscale x 4 x half> @llvm.masked.expandload.nxv4f16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x half> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv8f16 = call <vscale x 8 x half> @llvm.masked.expandload.nxv8f16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x half> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.expandload.nxv2f32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x float> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 2 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2i8 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4i8 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv8i8 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv16i8 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8.p0(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2i16 = call <vscale x 2 x i16> @llvm.masked.expandload.nxv2i16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i16> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4i16 = call <vscale x 4 x i16> @llvm.masked.expandload.nxv4i16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i16> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.masked.expandload.nxv8i16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i16> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2i32 = call <vscale x 2 x i32> @llvm.masked.expandload.nxv2i32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i32> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.masked.expandload.nxv4i32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i32> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2f16 = call <vscale x 2 x half> @llvm.masked.expandload.nxv2f16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x half> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4f16 = call <vscale x 4 x half> @llvm.masked.expandload.nxv4f16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x half> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv8f16 = call <vscale x 8 x half> @llvm.masked.expandload.nxv8f16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x half> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.expandload.nxv2f32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x float> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
 ; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of Invalid for: %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.expandload.nxv1i64.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x i64> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 8 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 16 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 16 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 32 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
 ; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; SVE2p2-SME2p2-LABEL: 'scalable'
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 2 for: %nxv2i8 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 2 for: %nxv4i8 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 2 for: %nxv8i8 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 2 for: %nxv16i8 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8.p0(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 2 for: %nxv2i16 = call <vscale x 2 x i16> @llvm.masked.expandload.nxv2i16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i16> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 2 for: %nxv4i16 = call <vscale x 4 x i16> @llvm.masked.expandload.nxv4i16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i16> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 2 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.masked.expandload.nxv8i16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i16> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 2 for: %nxv2i32 = call <vscale x 2 x i32> @llvm.masked.expandload.nxv2i32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i32> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 2 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.masked.expandload.nxv4i32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i32> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 2 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 2 for: %nxv2f16 = call <vscale x 2 x half> @llvm.masked.expandload.nxv2f16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x half> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 2 for: %nxv4f16 = call <vscale x 4 x half> @llvm.masked.expandload.nxv4f16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x half> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 2 for: %nxv8f16 = call <vscale x 8 x half> @llvm.masked.expandload.nxv8f16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x half> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 2 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.expandload.nxv2f32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x float> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 2 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 2 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv2i8 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv4i8 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv8i8 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv16i8 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8.p0(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv2i16 = call <vscale x 2 x i16> @llvm.masked.expandload.nxv2i16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i16> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv4i16 = call <vscale x 4 x i16> @llvm.masked.expandload.nxv4i16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i16> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.masked.expandload.nxv8i16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i16> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv2i32 = call <vscale x 2 x i32> @llvm.masked.expandload.nxv2i32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i32> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.masked.expandload.nxv4i32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i32> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv2f16 = call <vscale x 2 x half> @llvm.masked.expandload.nxv2f16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x half> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv4f16 = call <vscale x 4 x half> @llvm.masked.expandload.nxv4f16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x half> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv8f16 = call <vscale x 8 x half> @llvm.masked.expandload.nxv8f16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x half> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.expandload.nxv2f32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x float> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
 ; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of Invalid for: %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.expandload.nxv1i64.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x i64> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 8 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 16 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 16 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 32 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
 ; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; SVE-ONLY-LABEL: 'scalable'
@@ -230,25 +230,25 @@ define void @scalable() {
 ; SVE-ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; SVE2p2-SME2p2-SVE256-LABEL: 'scalable'
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %nxv2i8 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %nxv4i8 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %nxv8i8 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %nxv16i8 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8.p0(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %nxv2i16 = call <vscale x 2 x i16> @llvm.masked.expandload.nxv2i16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i16> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %nxv4i16 = call <vscale x 4 x i16> @llvm.masked.expandload.nxv4i16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i16> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.masked.expandload.nxv8i16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i16> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %nxv2i32 = call <vscale x 2 x i32> @llvm.masked.expandload.nxv2i32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i32> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.masked.expandload.nxv4i32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i32> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %nxv2f16 = call <vscale x 2 x half> @llvm.masked.expandload.nxv2f16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x half> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %nxv4f16 = call <vscale x 4 x half> @llvm.masked.expandload.nxv4f16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x half> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %nxv8f16 = call <vscale x 8 x half> @llvm.masked.expandload.nxv8f16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x half> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.expandload.nxv2f32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x float> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 2 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv2i8 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv4i8 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv8i8 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv16i8 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8.p0(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv2i16 = call <vscale x 2 x i16> @llvm.masked.expandload.nxv2i16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i16> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv4i16 = call <vscale x 4 x i16> @llvm.masked.expandload.nxv4i16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i16> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.masked.expandload.nxv8i16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i16> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv2i32 = call <vscale x 2 x i32> @llvm.masked.expandload.nxv2i32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i32> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.masked.expandload.nxv4i32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i32> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv2f16 = call <vscale x 2 x half> @llvm.masked.expandload.nxv2f16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x half> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv4f16 = call <vscale x 4 x half> @llvm.masked.expandload.nxv4f16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x half> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv8f16 = call <vscale x 8 x half> @llvm.masked.expandload.nxv8f16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x half> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.expandload.nxv2f32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x float> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
 ; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of Invalid for: %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.expandload.nxv1i64.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x i64> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 8 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 16 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 16 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 32 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
 ; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 entry:

>From 5834a6d37c7363bfe0061c937996c5242cc1ea35 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Mon, 28 Sep 2026 13:51:10 +0100
Subject: [PATCH 2/4] review comments

---
 .../AArch64/AArch64TargetTransformInfo.cpp    |  13 +-
 .../CostModel/AArch64/masked_compress_load.ll | 170 +++++++-------
 .../CostModel/AArch64/masked_expand_load.ll   | 216 +++++++++---------
 3 files changed, 199 insertions(+), 200 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index fe849df4e448872..4b06c8bd7718081 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5564,12 +5564,11 @@ AArch64TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
 
     // Operation will be split into expand of masked.load
     // Something like:
-    // cntp x8, p0, p0.s
-    // whilelo p1.s, xzr, x8
-    // ld1w { z1.s }, p1/z, [x0]
-    // expand z1.s, p0, z1.s
-    // mov z0.s, p0/m, z1.s1
-    MemOpCost *= 4;
+    // cntp     x8, p0, p0.s
+    // whilelo  p1.s, xzr, x8
+    // ld1w     { z1.s }, p1/z, [x0]
+    // expand   z1.s, p0, z1.s
+    MemOpCost += 4;
   }
 
   if (MICA.getID() == Intrinsic::masked_compressstore) {
@@ -5582,7 +5581,7 @@ AArch64TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
     //  cntp     x8, p1, p0.s
     //  whilelo  p0.s, xzr, x8
     //  st1w     { z0.s }, p0, [x0]
-    MemOpCost *= 4;
+    MemOpCost += 4;
   }
 
   // If we need to split the memory operation, we will also need to split the
diff --git a/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll b/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll
index 941eee99a943399..9cf6ac44edbe4e4 100644
--- a/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll
@@ -54,42 +54,42 @@ define void @fixed() {
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i8.p0(<2 x i8> poison, ptr poison, <2 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4i8.p0(<4 x i8> poison, ptr poison, <4 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v8i8.p0(<8 x i8> poison, ptr poison, <8 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v16i8.p0(<16 x i8> poison, ptr poison, <16 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v16i8.p0(<16 x i8> poison, ptr poison, <16 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i16.p0(<2 x i16> poison, ptr poison, <2 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4i16.p0(<4 x i16> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v8i16.p0(<8 x i16> poison, ptr poison, <8 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v8i16.p0(<8 x i16> poison, ptr poison, <8 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i32.p0(<2 x i32> poison, ptr poison, <2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v2i64.p0(<2 x i64> poison, ptr poison, <2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> poison, ptr poison, <4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v2i64.p0(<2 x i64> poison, ptr poison, <2 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2f16.p0(<2 x half> poison, ptr poison, <2 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4f16.p0(<4 x half> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v8f16.p0(<8 x half> poison, ptr poison, <8 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v8f16.p0(<8 x half> poison, ptr poison, <8 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2f32.p0(<2 x float> poison, ptr poison, <2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v4f32.p0(<4 x float> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v2f64.p0(<2 x double> poison, ptr poison, <2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.v4i64.p0(<4 x i64> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 32 for: call void @llvm.masked.compressstore.v32f16.p0(<32 x half> poison, ptr poison, <32 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v4f32.p0(<4 x float> poison, ptr poison, <4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v2f64.p0(<2 x double> poison, ptr poison, <2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 12 for: call void @llvm.masked.compressstore.v4i64.p0(<4 x i64> poison, ptr poison, <4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.v32f16.p0(<32 x half> poison, ptr poison, <32 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-8-64-SVE256-LABEL: 'fixed'
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i8.p0(<2 x i8> poison, ptr poison, <2 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4i8.p0(<4 x i8> poison, ptr poison, <4 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v8i8.p0(<8 x i8> poison, ptr poison, <8 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v16i8.p0(<16 x i8> poison, ptr poison, <16 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v16i8.p0(<16 x i8> poison, ptr poison, <16 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i16.p0(<2 x i16> poison, ptr poison, <2 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4i16.p0(<4 x i16> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v8i16.p0(<8 x i16> poison, ptr poison, <8 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v8i16.p0(<8 x i16> poison, ptr poison, <8 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i32.p0(<2 x i32> poison, ptr poison, <2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v2i64.p0(<2 x i64> poison, ptr poison, <2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> poison, ptr poison, <4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v2i64.p0(<2 x i64> poison, ptr poison, <2 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2f16.p0(<2 x half> poison, ptr poison, <2 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4f16.p0(<4 x half> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v8f16.p0(<8 x half> poison, ptr poison, <8 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v8f16.p0(<8 x half> poison, ptr poison, <8 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2f32.p0(<2 x float> poison, ptr poison, <2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v4f32.p0(<4 x float> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v2f64.p0(<2 x double> poison, ptr poison, <2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v4i64.p0(<4 x i64> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.v32f16.p0(<32 x half> poison, ptr poison, <32 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v4f32.p0(<4 x float> poison, ptr poison, <4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v2f64.p0(<2 x double> poison, ptr poison, <2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v4i64.p0(<4 x i64> poison, ptr poison, <4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 12 for: call void @llvm.masked.compressstore.v32f16.p0(<32 x half> poison, ptr poison, <32 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-UNAVAILABLE-LABEL: 'fixed'
@@ -143,91 +143,91 @@ entry:
 
 define void @scalable() {
 ; CHECK-32-64-LABEL: 'scalable'
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
 ; CHECK-32-64-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
 ; CHECK-32-64-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
 ; CHECK-32-64-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
 ; CHECK-32-64-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
 ; CHECK-32-64-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv1i64.p0(<vscale x 1 x i64> poison, ptr poison, <vscale x 1 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 12 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
 ; CHECK-32-64-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
 ; CHECK-32-64-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-8-64-LABEL: 'scalable'
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
 ; CHECK-8-64-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv1i64.p0(<vscale x 1 x i64> poison, ptr poison, <vscale x 1 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 32 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 12 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
 ; CHECK-8-64-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-8-64-STREAMING-ONLY-LABEL: 'scalable'
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv1i64.p0(<vscale x 1 x i64> poison, ptr poison, <vscale x 1 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 32 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 12 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-8-64-SVE256-LABEL: 'scalable'
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv1i64.p0(<vscale x 1 x i64> poison, ptr poison, <vscale x 1 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 32 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 12 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-UNAVAILABLE-LABEL: 'scalable'
diff --git a/llvm/test/Analysis/CostModel/AArch64/masked_expand_load.ll b/llvm/test/Analysis/CostModel/AArch64/masked_expand_load.ll
index 1a477a52acd15bd..905f6dbfc0f1829 100644
--- a/llvm/test/Analysis/CostModel/AArch64/masked_expand_load.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/masked_expand_load.ll
@@ -29,24 +29,24 @@ define void @fixed() {
 ; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; SVE2p2-SME2p2-STREAMING-LABEL: 'fixed'
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v2i8 = call <2 x i8> @llvm.masked.expandload.v2i8.p0(ptr poison, <2 x i1> poison, <2 x i8> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v4i8 = call <4 x i8> @llvm.masked.expandload.v4i8.p0(ptr poison, <4 x i1> poison, <4 x i8> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v8i8 = call <8 x i8> @llvm.masked.expandload.v8i8.p0(ptr poison, <8 x i1> poison, <8 x i8> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v16i8 = call <16 x i8> @llvm.masked.expandload.v16i8.p0(ptr poison, <16 x i1> poison, <16 x i8> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v2i16 = call <2 x i16> @llvm.masked.expandload.v2i16.p0(ptr poison, <2 x i1> poison, <2 x i16> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v4i16 = call <4 x i16> @llvm.masked.expandload.v4i16.p0(ptr poison, <4 x i1> poison, <4 x i16> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v8i16 = call <8 x i16> @llvm.masked.expandload.v8i16.p0(ptr poison, <8 x i1> poison, <8 x i16> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v2i32 = call <2 x i32> @llvm.masked.expandload.v2i32.p0(ptr poison, <2 x i1> poison, <2 x i32> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v4i32 = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr poison, <4 x i1> poison, <4 x i32> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v2i64 = call <2 x i64> @llvm.masked.expandload.v2i64.p0(ptr poison, <2 x i1> poison, <2 x i64> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v2f16 = call <2 x half> @llvm.masked.expandload.v2f16.p0(ptr poison, <2 x i1> poison, <2 x half> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v4f16 = call <4 x half> @llvm.masked.expandload.v4f16.p0(ptr poison, <4 x i1> poison, <4 x half> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v8f16 = call <8 x half> @llvm.masked.expandload.v8f16.p0(ptr poison, <8 x i1> poison, <8 x half> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v2f32 = call <2 x float> @llvm.masked.expandload.v2f32.p0(ptr poison, <2 x i1> poison, <2 x float> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v4f32 = call <4 x float> @llvm.masked.expandload.v4f32.p0(ptr poison, <4 x i1> poison, <4 x float> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v2f64 = call <2 x double> @llvm.masked.expandload.v2f64.p0(ptr poison, <2 x i1> poison, <2 x double> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 16 for: %v4i64 = call <4 x i64> @llvm.masked.expandload.v4i64.p0(ptr poison, <4 x i1> poison, <4 x i64> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 32 for: %v32f16 = call <32 x half> @llvm.masked.expandload.v32f16.p0(ptr poison, <32 x i1> poison, <32 x half> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v2i8 = call <2 x i8> @llvm.masked.expandload.v2i8.p0(ptr poison, <2 x i1> poison, <2 x i8> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v4i8 = call <4 x i8> @llvm.masked.expandload.v4i8.p0(ptr poison, <4 x i1> poison, <4 x i8> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v8i8 = call <8 x i8> @llvm.masked.expandload.v8i8.p0(ptr poison, <8 x i1> poison, <8 x i8> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v16i8 = call <16 x i8> @llvm.masked.expandload.v16i8.p0(ptr poison, <16 x i1> poison, <16 x i8> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v2i16 = call <2 x i16> @llvm.masked.expandload.v2i16.p0(ptr poison, <2 x i1> poison, <2 x i16> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v4i16 = call <4 x i16> @llvm.masked.expandload.v4i16.p0(ptr poison, <4 x i1> poison, <4 x i16> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v8i16 = call <8 x i16> @llvm.masked.expandload.v8i16.p0(ptr poison, <8 x i1> poison, <8 x i16> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v2i32 = call <2 x i32> @llvm.masked.expandload.v2i32.p0(ptr poison, <2 x i1> poison, <2 x i32> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v4i32 = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr poison, <4 x i1> poison, <4 x i32> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v2i64 = call <2 x i64> @llvm.masked.expandload.v2i64.p0(ptr poison, <2 x i1> poison, <2 x i64> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v2f16 = call <2 x half> @llvm.masked.expandload.v2f16.p0(ptr poison, <2 x i1> poison, <2 x half> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v4f16 = call <4 x half> @llvm.masked.expandload.v4f16.p0(ptr poison, <4 x i1> poison, <4 x half> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v8f16 = call <8 x half> @llvm.masked.expandload.v8f16.p0(ptr poison, <8 x i1> poison, <8 x half> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v2f32 = call <2 x float> @llvm.masked.expandload.v2f32.p0(ptr poison, <2 x i1> poison, <2 x float> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v4f32 = call <4 x float> @llvm.masked.expandload.v4f32.p0(ptr poison, <4 x i1> poison, <4 x float> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v2f64 = call <2 x double> @llvm.masked.expandload.v2f64.p0(ptr poison, <2 x i1> poison, <2 x double> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 12 for: %v4i64 = call <4 x i64> @llvm.masked.expandload.v4i64.p0(ptr poison, <4 x i1> poison, <4 x i64> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 16 for: %v32f16 = call <32 x half> @llvm.masked.expandload.v32f16.p0(ptr poison, <32 x i1> poison, <32 x half> poison)
 ; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; SVE2p2-SME2p2-LABEL: 'fixed'
@@ -92,24 +92,24 @@ define void @fixed() {
 ; SVE-ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; SVE2p2-SME2p2-SVE256-LABEL: 'fixed'
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v2i8 = call <2 x i8> @llvm.masked.expandload.v2i8.p0(ptr poison, <2 x i1> poison, <2 x i8> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v4i8 = call <4 x i8> @llvm.masked.expandload.v4i8.p0(ptr poison, <4 x i1> poison, <4 x i8> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v8i8 = call <8 x i8> @llvm.masked.expandload.v8i8.p0(ptr poison, <8 x i1> poison, <8 x i8> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v16i8 = call <16 x i8> @llvm.masked.expandload.v16i8.p0(ptr poison, <16 x i1> poison, <16 x i8> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v2i16 = call <2 x i16> @llvm.masked.expandload.v2i16.p0(ptr poison, <2 x i1> poison, <2 x i16> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v4i16 = call <4 x i16> @llvm.masked.expandload.v4i16.p0(ptr poison, <4 x i1> poison, <4 x i16> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v8i16 = call <8 x i16> @llvm.masked.expandload.v8i16.p0(ptr poison, <8 x i1> poison, <8 x i16> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v2i32 = call <2 x i32> @llvm.masked.expandload.v2i32.p0(ptr poison, <2 x i1> poison, <2 x i32> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v4i32 = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr poison, <4 x i1> poison, <4 x i32> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v2i64 = call <2 x i64> @llvm.masked.expandload.v2i64.p0(ptr poison, <2 x i1> poison, <2 x i64> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v2f16 = call <2 x half> @llvm.masked.expandload.v2f16.p0(ptr poison, <2 x i1> poison, <2 x half> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v4f16 = call <4 x half> @llvm.masked.expandload.v4f16.p0(ptr poison, <4 x i1> poison, <4 x half> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v8f16 = call <8 x half> @llvm.masked.expandload.v8f16.p0(ptr poison, <8 x i1> poison, <8 x half> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v2f32 = call <2 x float> @llvm.masked.expandload.v2f32.p0(ptr poison, <2 x i1> poison, <2 x float> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v4f32 = call <4 x float> @llvm.masked.expandload.v4f32.p0(ptr poison, <4 x i1> poison, <4 x float> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v2f64 = call <2 x double> @llvm.masked.expandload.v2f64.p0(ptr poison, <2 x i1> poison, <2 x double> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v4i64 = call <4 x i64> @llvm.masked.expandload.v4i64.p0(ptr poison, <4 x i1> poison, <4 x i64> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 16 for: %v32f16 = call <32 x half> @llvm.masked.expandload.v32f16.p0(ptr poison, <32 x i1> poison, <32 x half> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v2i8 = call <2 x i8> @llvm.masked.expandload.v2i8.p0(ptr poison, <2 x i1> poison, <2 x i8> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v4i8 = call <4 x i8> @llvm.masked.expandload.v4i8.p0(ptr poison, <4 x i1> poison, <4 x i8> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v8i8 = call <8 x i8> @llvm.masked.expandload.v8i8.p0(ptr poison, <8 x i1> poison, <8 x i8> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v16i8 = call <16 x i8> @llvm.masked.expandload.v16i8.p0(ptr poison, <16 x i1> poison, <16 x i8> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v2i16 = call <2 x i16> @llvm.masked.expandload.v2i16.p0(ptr poison, <2 x i1> poison, <2 x i16> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v4i16 = call <4 x i16> @llvm.masked.expandload.v4i16.p0(ptr poison, <4 x i1> poison, <4 x i16> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v8i16 = call <8 x i16> @llvm.masked.expandload.v8i16.p0(ptr poison, <8 x i1> poison, <8 x i16> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v2i32 = call <2 x i32> @llvm.masked.expandload.v2i32.p0(ptr poison, <2 x i1> poison, <2 x i32> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v4i32 = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr poison, <4 x i1> poison, <4 x i32> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v2i64 = call <2 x i64> @llvm.masked.expandload.v2i64.p0(ptr poison, <2 x i1> poison, <2 x i64> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v2f16 = call <2 x half> @llvm.masked.expandload.v2f16.p0(ptr poison, <2 x i1> poison, <2 x half> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v4f16 = call <4 x half> @llvm.masked.expandload.v4f16.p0(ptr poison, <4 x i1> poison, <4 x half> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v8f16 = call <8 x half> @llvm.masked.expandload.v8f16.p0(ptr poison, <8 x i1> poison, <8 x half> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v2f32 = call <2 x float> @llvm.masked.expandload.v2f32.p0(ptr poison, <2 x i1> poison, <2 x float> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v4f32 = call <4 x float> @llvm.masked.expandload.v4f32.p0(ptr poison, <4 x i1> poison, <4 x float> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v2f64 = call <2 x double> @llvm.masked.expandload.v2f64.p0(ptr poison, <2 x i1> poison, <2 x double> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v4i64 = call <4 x i64> @llvm.masked.expandload.v4i64.p0(ptr poison, <4 x i1> poison, <4 x i64> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 12 for: %v32f16 = call <32 x half> @llvm.masked.expandload.v32f16.p0(ptr poison, <32 x i1> poison, <32 x half> poison)
 ; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 entry:
@@ -142,69 +142,69 @@ entry:
 
 define void @scalable() {
 ; SVE2p2-SME2p2-NON-STREAMING-LABEL: 'scalable'
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2i8 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4i8 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv8i8 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv16i8 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8.p0(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2i16 = call <vscale x 2 x i16> @llvm.masked.expandload.nxv2i16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i16> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4i16 = call <vscale x 4 x i16> @llvm.masked.expandload.nxv4i16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i16> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.masked.expandload.nxv8i16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i16> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2i32 = call <vscale x 2 x i32> @llvm.masked.expandload.nxv2i32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i32> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.masked.expandload.nxv4i32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i32> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2f16 = call <vscale x 2 x half> @llvm.masked.expandload.nxv2f16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x half> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4f16 = call <vscale x 4 x half> @llvm.masked.expandload.nxv4f16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x half> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv8f16 = call <vscale x 8 x half> @llvm.masked.expandload.nxv8f16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x half> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.expandload.nxv2f32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x float> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2i8 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv4i8 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv8i8 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv16i8 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8.p0(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2i16 = call <vscale x 2 x i16> @llvm.masked.expandload.nxv2i16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i16> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv4i16 = call <vscale x 4 x i16> @llvm.masked.expandload.nxv4i16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i16> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.masked.expandload.nxv8i16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i16> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2i32 = call <vscale x 2 x i32> @llvm.masked.expandload.nxv2i32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i32> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.masked.expandload.nxv4i32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i32> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2f16 = call <vscale x 2 x half> @llvm.masked.expandload.nxv2f16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x half> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv4f16 = call <vscale x 4 x half> @llvm.masked.expandload.nxv4f16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x half> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv8f16 = call <vscale x 8 x half> @llvm.masked.expandload.nxv8f16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x half> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.expandload.nxv2f32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x float> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
 ; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of Invalid for: %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.expandload.nxv1i64.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x i64> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 16 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 32 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 12 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 16 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
 ; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; SVE2p2-SME2p2-STREAMING-LABEL: 'scalable'
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2i8 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4i8 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv8i8 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv16i8 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8.p0(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2i16 = call <vscale x 2 x i16> @llvm.masked.expandload.nxv2i16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i16> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4i16 = call <vscale x 4 x i16> @llvm.masked.expandload.nxv4i16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i16> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.masked.expandload.nxv8i16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i16> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2i32 = call <vscale x 2 x i32> @llvm.masked.expandload.nxv2i32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i32> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.masked.expandload.nxv4i32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i32> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2f16 = call <vscale x 2 x half> @llvm.masked.expandload.nxv2f16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x half> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4f16 = call <vscale x 4 x half> @llvm.masked.expandload.nxv4f16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x half> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv8f16 = call <vscale x 8 x half> @llvm.masked.expandload.nxv8f16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x half> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.expandload.nxv2f32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x float> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2i8 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv4i8 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv8i8 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv16i8 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8.p0(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2i16 = call <vscale x 2 x i16> @llvm.masked.expandload.nxv2i16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i16> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv4i16 = call <vscale x 4 x i16> @llvm.masked.expandload.nxv4i16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i16> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.masked.expandload.nxv8i16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i16> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2i32 = call <vscale x 2 x i32> @llvm.masked.expandload.nxv2i32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i32> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.masked.expandload.nxv4i32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i32> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2f16 = call <vscale x 2 x half> @llvm.masked.expandload.nxv2f16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x half> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv4f16 = call <vscale x 4 x half> @llvm.masked.expandload.nxv4f16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x half> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv8f16 = call <vscale x 8 x half> @llvm.masked.expandload.nxv8f16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x half> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.expandload.nxv2f32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x float> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
 ; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of Invalid for: %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.expandload.nxv1i64.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x i64> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 16 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 32 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 12 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 16 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
 ; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; SVE2p2-SME2p2-LABEL: 'scalable'
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv2i8 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv4i8 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv8i8 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv16i8 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8.p0(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv2i16 = call <vscale x 2 x i16> @llvm.masked.expandload.nxv2i16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i16> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv4i16 = call <vscale x 4 x i16> @llvm.masked.expandload.nxv4i16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i16> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.masked.expandload.nxv8i16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i16> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv2i32 = call <vscale x 2 x i32> @llvm.masked.expandload.nxv2i32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i32> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.masked.expandload.nxv4i32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i32> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv2f16 = call <vscale x 2 x half> @llvm.masked.expandload.nxv2f16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x half> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv4f16 = call <vscale x 4 x half> @llvm.masked.expandload.nxv4f16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x half> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv8f16 = call <vscale x 8 x half> @llvm.masked.expandload.nxv8f16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x half> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.expandload.nxv2f32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x float> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv2i8 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv4i8 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv8i8 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv16i8 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8.p0(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv2i16 = call <vscale x 2 x i16> @llvm.masked.expandload.nxv2i16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i16> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv4i16 = call <vscale x 4 x i16> @llvm.masked.expandload.nxv4i16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i16> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.masked.expandload.nxv8i16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i16> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv2i32 = call <vscale x 2 x i32> @llvm.masked.expandload.nxv2i32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i32> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.masked.expandload.nxv4i32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i32> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv2f16 = call <vscale x 2 x half> @llvm.masked.expandload.nxv2f16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x half> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv4f16 = call <vscale x 4 x half> @llvm.masked.expandload.nxv4f16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x half> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv8f16 = call <vscale x 8 x half> @llvm.masked.expandload.nxv8f16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x half> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.expandload.nxv2f32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x float> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
 ; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of Invalid for: %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.expandload.nxv1i64.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x i64> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 16 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 32 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 12 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 16 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
 ; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; SVE-ONLY-LABEL: 'scalable'
@@ -230,25 +230,25 @@ define void @scalable() {
 ; SVE-ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; SVE2p2-SME2p2-SVE256-LABEL: 'scalable'
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv2i8 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv4i8 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv8i8 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv16i8 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8.p0(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv2i16 = call <vscale x 2 x i16> @llvm.masked.expandload.nxv2i16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i16> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv4i16 = call <vscale x 4 x i16> @llvm.masked.expandload.nxv4i16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i16> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.masked.expandload.nxv8i16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i16> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv2i32 = call <vscale x 2 x i32> @llvm.masked.expandload.nxv2i32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i32> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.masked.expandload.nxv4i32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i32> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv2f16 = call <vscale x 2 x half> @llvm.masked.expandload.nxv2f16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x half> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv4f16 = call <vscale x 4 x half> @llvm.masked.expandload.nxv4f16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x half> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv8f16 = call <vscale x 8 x half> @llvm.masked.expandload.nxv8f16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x half> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.expandload.nxv2f32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x float> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv2i8 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv4i8 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv8i8 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv16i8 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8.p0(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv2i16 = call <vscale x 2 x i16> @llvm.masked.expandload.nxv2i16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i16> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv4i16 = call <vscale x 4 x i16> @llvm.masked.expandload.nxv4i16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i16> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.masked.expandload.nxv8i16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i16> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv2i32 = call <vscale x 2 x i32> @llvm.masked.expandload.nxv2i32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i32> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.masked.expandload.nxv4i32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i32> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv2f16 = call <vscale x 2 x half> @llvm.masked.expandload.nxv2f16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x half> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv4f16 = call <vscale x 4 x half> @llvm.masked.expandload.nxv4f16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x half> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv8f16 = call <vscale x 8 x half> @llvm.masked.expandload.nxv8f16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x half> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.expandload.nxv2f32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x float> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
 ; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of Invalid for: %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.expandload.nxv1i64.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x i64> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 16 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 32 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 12 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 16 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
 ; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 entry:

>From b5c14f7e2f30902e0a77a3dfca1b7148e48cf28a Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Wed, 30 Sep 2026 13:59:57 +0100
Subject: [PATCH 3/4] Adjust costs to not double count ld1/st1 instructions

---
 .../AArch64/AArch64TargetTransformInfo.cpp    |   5 +-
 .../CostModel/AArch64/masked_compress_load.ll | 170 +++++++-------
 .../CostModel/AArch64/masked_expand_load.ll   | 216 +++++++++---------
 3 files changed, 195 insertions(+), 196 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 4b06c8bd7718081..7108560e0573c9b 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5568,7 +5568,7 @@ AArch64TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
     // whilelo  p1.s, xzr, x8
     // ld1w     { z1.s }, p1/z, [x0]
     // expand   z1.s, p0, z1.s
-    MemOpCost += 4;
+    MemOpCost += 3;
   }
 
   if (MICA.getID() == Intrinsic::masked_compressstore) {
@@ -5576,12 +5576,11 @@ AArch64TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
       return InstructionCost::getInvalid();
 
     // A compress store lowers to something like:
-    //  ptrue    p1.s
     //  compact  z0.s, p0, z0.s
     //  cntp     x8, p1, p0.s
     //  whilelo  p0.s, xzr, x8
     //  st1w     { z0.s }, p0, [x0]
-    MemOpCost += 4;
+    MemOpCost += 3;
   }
 
   // If we need to split the memory operation, we will also need to split the
diff --git a/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll b/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll
index 9cf6ac44edbe4e4..93a3781e25cd52c 100644
--- a/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll
@@ -54,42 +54,42 @@ define void @fixed() {
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i8.p0(<2 x i8> poison, ptr poison, <2 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4i8.p0(<4 x i8> poison, ptr poison, <4 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v8i8.p0(<8 x i8> poison, ptr poison, <8 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v16i8.p0(<16 x i8> poison, ptr poison, <16 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v16i8.p0(<16 x i8> poison, ptr poison, <16 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i16.p0(<2 x i16> poison, ptr poison, <2 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4i16.p0(<4 x i16> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v8i16.p0(<8 x i16> poison, ptr poison, <8 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v8i16.p0(<8 x i16> poison, ptr poison, <8 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i32.p0(<2 x i32> poison, ptr poison, <2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v2i64.p0(<2 x i64> poison, ptr poison, <2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> poison, ptr poison, <4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v2i64.p0(<2 x i64> poison, ptr poison, <2 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2f16.p0(<2 x half> poison, ptr poison, <2 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4f16.p0(<4 x half> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v8f16.p0(<8 x half> poison, ptr poison, <8 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v8f16.p0(<8 x half> poison, ptr poison, <8 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2f32.p0(<2 x float> poison, ptr poison, <2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v4f32.p0(<4 x float> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v2f64.p0(<2 x double> poison, ptr poison, <2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 12 for: call void @llvm.masked.compressstore.v4i64.p0(<4 x i64> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.v32f16.p0(<32 x half> poison, ptr poison, <32 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v4f32.p0(<4 x float> poison, ptr poison, <4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v2f64.p0(<2 x double> poison, ptr poison, <2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 10 for: call void @llvm.masked.compressstore.v4i64.p0(<4 x i64> poison, ptr poison, <4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 14 for: call void @llvm.masked.compressstore.v32f16.p0(<32 x half> poison, ptr poison, <32 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-8-64-SVE256-LABEL: 'fixed'
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i8.p0(<2 x i8> poison, ptr poison, <2 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4i8.p0(<4 x i8> poison, ptr poison, <4 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v8i8.p0(<8 x i8> poison, ptr poison, <8 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v16i8.p0(<16 x i8> poison, ptr poison, <16 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v16i8.p0(<16 x i8> poison, ptr poison, <16 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i16.p0(<2 x i16> poison, ptr poison, <2 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4i16.p0(<4 x i16> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v8i16.p0(<8 x i16> poison, ptr poison, <8 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v8i16.p0(<8 x i16> poison, ptr poison, <8 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i32.p0(<2 x i32> poison, ptr poison, <2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v2i64.p0(<2 x i64> poison, ptr poison, <2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> poison, ptr poison, <4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v2i64.p0(<2 x i64> poison, ptr poison, <2 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2f16.p0(<2 x half> poison, ptr poison, <2 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4f16.p0(<4 x half> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v8f16.p0(<8 x half> poison, ptr poison, <8 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v8f16.p0(<8 x half> poison, ptr poison, <8 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2f32.p0(<2 x float> poison, ptr poison, <2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v4f32.p0(<4 x float> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v2f64.p0(<2 x double> poison, ptr poison, <2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.v4i64.p0(<4 x i64> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 12 for: call void @llvm.masked.compressstore.v32f16.p0(<32 x half> poison, ptr poison, <32 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v4f32.p0(<4 x float> poison, ptr poison, <4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v2f64.p0(<2 x double> poison, ptr poison, <2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v4i64.p0(<4 x i64> poison, ptr poison, <4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 10 for: call void @llvm.masked.compressstore.v32f16.p0(<32 x half> poison, ptr poison, <32 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-UNAVAILABLE-LABEL: 'fixed'
@@ -143,91 +143,91 @@ entry:
 
 define void @scalable() {
 ; CHECK-32-64-LABEL: 'scalable'
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
 ; CHECK-32-64-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
 ; CHECK-32-64-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
 ; CHECK-32-64-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
 ; CHECK-32-64-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
 ; CHECK-32-64-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv1i64.p0(<vscale x 1 x i64> poison, ptr poison, <vscale x 1 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 12 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 10 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
 ; CHECK-32-64-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
 ; CHECK-32-64-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-8-64-LABEL: 'scalable'
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
 ; CHECK-8-64-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv1i64.p0(<vscale x 1 x i64> poison, ptr poison, <vscale x 1 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 12 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 10 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 14 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
 ; CHECK-8-64-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-8-64-STREAMING-ONLY-LABEL: 'scalable'
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv1i64.p0(<vscale x 1 x i64> poison, ptr poison, <vscale x 1 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 12 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 10 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 14 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-8-64-SVE256-LABEL: 'scalable'
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 5 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv1i64.p0(<vscale x 1 x i64> poison, ptr poison, <vscale x 1 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 12 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 10 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 14 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-UNAVAILABLE-LABEL: 'scalable'
diff --git a/llvm/test/Analysis/CostModel/AArch64/masked_expand_load.ll b/llvm/test/Analysis/CostModel/AArch64/masked_expand_load.ll
index 905f6dbfc0f1829..75cc89c85aa9f68 100644
--- a/llvm/test/Analysis/CostModel/AArch64/masked_expand_load.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/masked_expand_load.ll
@@ -29,24 +29,24 @@ define void @fixed() {
 ; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; SVE2p2-SME2p2-STREAMING-LABEL: 'fixed'
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v2i8 = call <2 x i8> @llvm.masked.expandload.v2i8.p0(ptr poison, <2 x i1> poison, <2 x i8> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v4i8 = call <4 x i8> @llvm.masked.expandload.v4i8.p0(ptr poison, <4 x i1> poison, <4 x i8> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v8i8 = call <8 x i8> @llvm.masked.expandload.v8i8.p0(ptr poison, <8 x i1> poison, <8 x i8> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v16i8 = call <16 x i8> @llvm.masked.expandload.v16i8.p0(ptr poison, <16 x i1> poison, <16 x i8> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v2i16 = call <2 x i16> @llvm.masked.expandload.v2i16.p0(ptr poison, <2 x i1> poison, <2 x i16> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v4i16 = call <4 x i16> @llvm.masked.expandload.v4i16.p0(ptr poison, <4 x i1> poison, <4 x i16> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v8i16 = call <8 x i16> @llvm.masked.expandload.v8i16.p0(ptr poison, <8 x i1> poison, <8 x i16> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v2i32 = call <2 x i32> @llvm.masked.expandload.v2i32.p0(ptr poison, <2 x i1> poison, <2 x i32> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v4i32 = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr poison, <4 x i1> poison, <4 x i32> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v2i64 = call <2 x i64> @llvm.masked.expandload.v2i64.p0(ptr poison, <2 x i1> poison, <2 x i64> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v2f16 = call <2 x half> @llvm.masked.expandload.v2f16.p0(ptr poison, <2 x i1> poison, <2 x half> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v4f16 = call <4 x half> @llvm.masked.expandload.v4f16.p0(ptr poison, <4 x i1> poison, <4 x half> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v8f16 = call <8 x half> @llvm.masked.expandload.v8f16.p0(ptr poison, <8 x i1> poison, <8 x half> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v2f32 = call <2 x float> @llvm.masked.expandload.v2f32.p0(ptr poison, <2 x i1> poison, <2 x float> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v4f32 = call <4 x float> @llvm.masked.expandload.v4f32.p0(ptr poison, <4 x i1> poison, <4 x float> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %v2f64 = call <2 x double> @llvm.masked.expandload.v2f64.p0(ptr poison, <2 x i1> poison, <2 x double> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 12 for: %v4i64 = call <4 x i64> @llvm.masked.expandload.v4i64.p0(ptr poison, <4 x i1> poison, <4 x i64> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 16 for: %v32f16 = call <32 x half> @llvm.masked.expandload.v32f16.p0(ptr poison, <32 x i1> poison, <32 x half> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v2i8 = call <2 x i8> @llvm.masked.expandload.v2i8.p0(ptr poison, <2 x i1> poison, <2 x i8> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v4i8 = call <4 x i8> @llvm.masked.expandload.v4i8.p0(ptr poison, <4 x i1> poison, <4 x i8> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v8i8 = call <8 x i8> @llvm.masked.expandload.v8i8.p0(ptr poison, <8 x i1> poison, <8 x i8> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v16i8 = call <16 x i8> @llvm.masked.expandload.v16i8.p0(ptr poison, <16 x i1> poison, <16 x i8> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v2i16 = call <2 x i16> @llvm.masked.expandload.v2i16.p0(ptr poison, <2 x i1> poison, <2 x i16> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v4i16 = call <4 x i16> @llvm.masked.expandload.v4i16.p0(ptr poison, <4 x i1> poison, <4 x i16> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v8i16 = call <8 x i16> @llvm.masked.expandload.v8i16.p0(ptr poison, <8 x i1> poison, <8 x i16> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v2i32 = call <2 x i32> @llvm.masked.expandload.v2i32.p0(ptr poison, <2 x i1> poison, <2 x i32> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v4i32 = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr poison, <4 x i1> poison, <4 x i32> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v2i64 = call <2 x i64> @llvm.masked.expandload.v2i64.p0(ptr poison, <2 x i1> poison, <2 x i64> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v2f16 = call <2 x half> @llvm.masked.expandload.v2f16.p0(ptr poison, <2 x i1> poison, <2 x half> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v4f16 = call <4 x half> @llvm.masked.expandload.v4f16.p0(ptr poison, <4 x i1> poison, <4 x half> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v8f16 = call <8 x half> @llvm.masked.expandload.v8f16.p0(ptr poison, <8 x i1> poison, <8 x half> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v2f32 = call <2 x float> @llvm.masked.expandload.v2f32.p0(ptr poison, <2 x i1> poison, <2 x float> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v4f32 = call <4 x float> @llvm.masked.expandload.v4f32.p0(ptr poison, <4 x i1> poison, <4 x float> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v2f64 = call <2 x double> @llvm.masked.expandload.v2f64.p0(ptr poison, <2 x i1> poison, <2 x double> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 10 for: %v4i64 = call <4 x i64> @llvm.masked.expandload.v4i64.p0(ptr poison, <4 x i1> poison, <4 x i64> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 14 for: %v32f16 = call <32 x half> @llvm.masked.expandload.v32f16.p0(ptr poison, <32 x i1> poison, <32 x half> poison)
 ; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; SVE2p2-SME2p2-LABEL: 'fixed'
@@ -92,24 +92,24 @@ define void @fixed() {
 ; SVE-ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; SVE2p2-SME2p2-SVE256-LABEL: 'fixed'
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v2i8 = call <2 x i8> @llvm.masked.expandload.v2i8.p0(ptr poison, <2 x i1> poison, <2 x i8> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v4i8 = call <4 x i8> @llvm.masked.expandload.v4i8.p0(ptr poison, <4 x i1> poison, <4 x i8> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v8i8 = call <8 x i8> @llvm.masked.expandload.v8i8.p0(ptr poison, <8 x i1> poison, <8 x i8> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v16i8 = call <16 x i8> @llvm.masked.expandload.v16i8.p0(ptr poison, <16 x i1> poison, <16 x i8> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v2i16 = call <2 x i16> @llvm.masked.expandload.v2i16.p0(ptr poison, <2 x i1> poison, <2 x i16> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v4i16 = call <4 x i16> @llvm.masked.expandload.v4i16.p0(ptr poison, <4 x i1> poison, <4 x i16> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v8i16 = call <8 x i16> @llvm.masked.expandload.v8i16.p0(ptr poison, <8 x i1> poison, <8 x i16> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v2i32 = call <2 x i32> @llvm.masked.expandload.v2i32.p0(ptr poison, <2 x i1> poison, <2 x i32> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v4i32 = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr poison, <4 x i1> poison, <4 x i32> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v2i64 = call <2 x i64> @llvm.masked.expandload.v2i64.p0(ptr poison, <2 x i1> poison, <2 x i64> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v2f16 = call <2 x half> @llvm.masked.expandload.v2f16.p0(ptr poison, <2 x i1> poison, <2 x half> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v4f16 = call <4 x half> @llvm.masked.expandload.v4f16.p0(ptr poison, <4 x i1> poison, <4 x half> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v8f16 = call <8 x half> @llvm.masked.expandload.v8f16.p0(ptr poison, <8 x i1> poison, <8 x half> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v2f32 = call <2 x float> @llvm.masked.expandload.v2f32.p0(ptr poison, <2 x i1> poison, <2 x float> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v4f32 = call <4 x float> @llvm.masked.expandload.v4f32.p0(ptr poison, <4 x i1> poison, <4 x float> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v2f64 = call <2 x double> @llvm.masked.expandload.v2f64.p0(ptr poison, <2 x i1> poison, <2 x double> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %v4i64 = call <4 x i64> @llvm.masked.expandload.v4i64.p0(ptr poison, <4 x i1> poison, <4 x i64> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 12 for: %v32f16 = call <32 x half> @llvm.masked.expandload.v32f16.p0(ptr poison, <32 x i1> poison, <32 x half> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v2i8 = call <2 x i8> @llvm.masked.expandload.v2i8.p0(ptr poison, <2 x i1> poison, <2 x i8> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v4i8 = call <4 x i8> @llvm.masked.expandload.v4i8.p0(ptr poison, <4 x i1> poison, <4 x i8> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v8i8 = call <8 x i8> @llvm.masked.expandload.v8i8.p0(ptr poison, <8 x i1> poison, <8 x i8> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v16i8 = call <16 x i8> @llvm.masked.expandload.v16i8.p0(ptr poison, <16 x i1> poison, <16 x i8> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v2i16 = call <2 x i16> @llvm.masked.expandload.v2i16.p0(ptr poison, <2 x i1> poison, <2 x i16> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v4i16 = call <4 x i16> @llvm.masked.expandload.v4i16.p0(ptr poison, <4 x i1> poison, <4 x i16> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v8i16 = call <8 x i16> @llvm.masked.expandload.v8i16.p0(ptr poison, <8 x i1> poison, <8 x i16> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v2i32 = call <2 x i32> @llvm.masked.expandload.v2i32.p0(ptr poison, <2 x i1> poison, <2 x i32> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v4i32 = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr poison, <4 x i1> poison, <4 x i32> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v2i64 = call <2 x i64> @llvm.masked.expandload.v2i64.p0(ptr poison, <2 x i1> poison, <2 x i64> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v2f16 = call <2 x half> @llvm.masked.expandload.v2f16.p0(ptr poison, <2 x i1> poison, <2 x half> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v4f16 = call <4 x half> @llvm.masked.expandload.v4f16.p0(ptr poison, <4 x i1> poison, <4 x half> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v8f16 = call <8 x half> @llvm.masked.expandload.v8f16.p0(ptr poison, <8 x i1> poison, <8 x half> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v2f32 = call <2 x float> @llvm.masked.expandload.v2f32.p0(ptr poison, <2 x i1> poison, <2 x float> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v4f32 = call <4 x float> @llvm.masked.expandload.v4f32.p0(ptr poison, <4 x i1> poison, <4 x float> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v2f64 = call <2 x double> @llvm.masked.expandload.v2f64.p0(ptr poison, <2 x i1> poison, <2 x double> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v4i64 = call <4 x i64> @llvm.masked.expandload.v4i64.p0(ptr poison, <4 x i1> poison, <4 x i64> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 10 for: %v32f16 = call <32 x half> @llvm.masked.expandload.v32f16.p0(ptr poison, <32 x i1> poison, <32 x half> poison)
 ; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 entry:
@@ -142,69 +142,69 @@ entry:
 
 define void @scalable() {
 ; SVE2p2-SME2p2-NON-STREAMING-LABEL: 'scalable'
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2i8 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv4i8 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv8i8 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv16i8 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8.p0(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2i16 = call <vscale x 2 x i16> @llvm.masked.expandload.nxv2i16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i16> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv4i16 = call <vscale x 4 x i16> @llvm.masked.expandload.nxv4i16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i16> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.masked.expandload.nxv8i16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i16> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2i32 = call <vscale x 2 x i32> @llvm.masked.expandload.nxv2i32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i32> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.masked.expandload.nxv4i32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i32> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2f16 = call <vscale x 2 x half> @llvm.masked.expandload.nxv2f16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x half> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv4f16 = call <vscale x 4 x half> @llvm.masked.expandload.nxv4f16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x half> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv8f16 = call <vscale x 8 x half> @llvm.masked.expandload.nxv8f16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x half> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.expandload.nxv2f32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x float> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2i8 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4i8 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv8i8 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv16i8 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8.p0(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2i16 = call <vscale x 2 x i16> @llvm.masked.expandload.nxv2i16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i16> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4i16 = call <vscale x 4 x i16> @llvm.masked.expandload.nxv4i16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i16> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.masked.expandload.nxv8i16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i16> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2i32 = call <vscale x 2 x i32> @llvm.masked.expandload.nxv2i32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i32> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.masked.expandload.nxv4i32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i32> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2f16 = call <vscale x 2 x half> @llvm.masked.expandload.nxv2f16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x half> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4f16 = call <vscale x 4 x half> @llvm.masked.expandload.nxv4f16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x half> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv8f16 = call <vscale x 8 x half> @llvm.masked.expandload.nxv8f16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x half> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.expandload.nxv2f32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x float> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
 ; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of Invalid for: %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.expandload.nxv1i64.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x i64> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 12 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 16 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 10 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 14 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
 ; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; SVE2p2-SME2p2-STREAMING-LABEL: 'scalable'
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2i8 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv4i8 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv8i8 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv16i8 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8.p0(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2i16 = call <vscale x 2 x i16> @llvm.masked.expandload.nxv2i16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i16> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv4i16 = call <vscale x 4 x i16> @llvm.masked.expandload.nxv4i16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i16> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.masked.expandload.nxv8i16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i16> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2i32 = call <vscale x 2 x i32> @llvm.masked.expandload.nxv2i32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i32> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.masked.expandload.nxv4i32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i32> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2f16 = call <vscale x 2 x half> @llvm.masked.expandload.nxv2f16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x half> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv4f16 = call <vscale x 4 x half> @llvm.masked.expandload.nxv4f16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x half> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv8f16 = call <vscale x 8 x half> @llvm.masked.expandload.nxv8f16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x half> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.expandload.nxv2f32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x float> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 5 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2i8 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4i8 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv8i8 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv16i8 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8.p0(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2i16 = call <vscale x 2 x i16> @llvm.masked.expandload.nxv2i16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i16> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4i16 = call <vscale x 4 x i16> @llvm.masked.expandload.nxv4i16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i16> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.masked.expandload.nxv8i16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i16> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2i32 = call <vscale x 2 x i32> @llvm.masked.expandload.nxv2i32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i32> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.masked.expandload.nxv4i32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i32> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2f16 = call <vscale x 2 x half> @llvm.masked.expandload.nxv2f16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x half> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4f16 = call <vscale x 4 x half> @llvm.masked.expandload.nxv4f16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x half> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv8f16 = call <vscale x 8 x half> @llvm.masked.expandload.nxv8f16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x half> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.expandload.nxv2f32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x float> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
 ; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of Invalid for: %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.expandload.nxv1i64.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x i64> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 12 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 16 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 10 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 14 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
 ; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; SVE2p2-SME2p2-LABEL: 'scalable'
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv2i8 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv4i8 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv8i8 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv16i8 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8.p0(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv2i16 = call <vscale x 2 x i16> @llvm.masked.expandload.nxv2i16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i16> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv4i16 = call <vscale x 4 x i16> @llvm.masked.expandload.nxv4i16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i16> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.masked.expandload.nxv8i16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i16> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv2i32 = call <vscale x 2 x i32> @llvm.masked.expandload.nxv2i32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i32> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.masked.expandload.nxv4i32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i32> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv2f16 = call <vscale x 2 x half> @llvm.masked.expandload.nxv2f16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x half> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv4f16 = call <vscale x 4 x half> @llvm.masked.expandload.nxv4f16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x half> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv8f16 = call <vscale x 8 x half> @llvm.masked.expandload.nxv8f16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x half> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.expandload.nxv2f32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x float> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 5 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv2i8 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv4i8 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv8i8 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv16i8 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8.p0(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv2i16 = call <vscale x 2 x i16> @llvm.masked.expandload.nxv2i16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i16> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv4i16 = call <vscale x 4 x i16> @llvm.masked.expandload.nxv4i16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i16> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.masked.expandload.nxv8i16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i16> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv2i32 = call <vscale x 2 x i32> @llvm.masked.expandload.nxv2i32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i32> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.masked.expandload.nxv4i32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i32> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv2f16 = call <vscale x 2 x half> @llvm.masked.expandload.nxv2f16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x half> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv4f16 = call <vscale x 4 x half> @llvm.masked.expandload.nxv4f16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x half> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv8f16 = call <vscale x 8 x half> @llvm.masked.expandload.nxv8f16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x half> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.expandload.nxv2f32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x float> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
 ; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of Invalid for: %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.expandload.nxv1i64.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x i64> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 12 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 16 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 10 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 14 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
 ; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; SVE-ONLY-LABEL: 'scalable'
@@ -230,25 +230,25 @@ define void @scalable() {
 ; SVE-ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; SVE2p2-SME2p2-SVE256-LABEL: 'scalable'
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv2i8 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv4i8 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv8i8 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv16i8 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8.p0(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv2i16 = call <vscale x 2 x i16> @llvm.masked.expandload.nxv2i16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i16> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv4i16 = call <vscale x 4 x i16> @llvm.masked.expandload.nxv4i16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i16> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.masked.expandload.nxv8i16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i16> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv2i32 = call <vscale x 2 x i32> @llvm.masked.expandload.nxv2i32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i32> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.masked.expandload.nxv4i32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i32> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv2f16 = call <vscale x 2 x half> @llvm.masked.expandload.nxv2f16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x half> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv4f16 = call <vscale x 4 x half> @llvm.masked.expandload.nxv4f16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x half> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv8f16 = call <vscale x 8 x half> @llvm.masked.expandload.nxv8f16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x half> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.expandload.nxv2f32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x float> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 5 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv2i8 = call <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i8> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv4i8 = call <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i8> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv8i8 = call <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i8> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv16i8 = call <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8.p0(ptr poison, <vscale x 16 x i1> poison, <vscale x 16 x i8> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv2i16 = call <vscale x 2 x i16> @llvm.masked.expandload.nxv2i16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i16> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv4i16 = call <vscale x 4 x i16> @llvm.masked.expandload.nxv4i16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i16> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.masked.expandload.nxv8i16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x i16> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv2i32 = call <vscale x 2 x i32> @llvm.masked.expandload.nxv2i32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i32> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.masked.expandload.nxv4i32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i32> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x i64> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv2f16 = call <vscale x 2 x half> @llvm.masked.expandload.nxv2f16.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x half> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv4f16 = call <vscale x 4 x half> @llvm.masked.expandload.nxv4f16.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x half> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv8f16 = call <vscale x 8 x half> @llvm.masked.expandload.nxv8f16.p0(ptr poison, <vscale x 8 x i1> poison, <vscale x 8 x half> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.expandload.nxv2f32.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x float> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
 ; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of Invalid for: %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.expandload.nxv1i64.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x i64> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 12 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 16 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 10 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 14 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
 ; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 entry:

>From c9fa8254221b837d644f76e21bf6dc97c49cd29b Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Wed, 30 Sep 2026 14:21:43 +0100
Subject: [PATCH 4/4] Revert to Multiplication

LT.first is just the type legalization cost, so we need to multiply
by 4 to handle splitting of types where necessary.
---
 .../AArch64/AArch64TargetTransformInfo.cpp    |  4 ++--
 .../CostModel/AArch64/masked_compress_load.ll | 20 ++++++++---------
 .../CostModel/AArch64/masked_expand_load.ll   | 22 +++++++++----------
 3 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 7108560e0573c9b..8345498ebba9eda 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5568,7 +5568,7 @@ AArch64TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
     // whilelo  p1.s, xzr, x8
     // ld1w     { z1.s }, p1/z, [x0]
     // expand   z1.s, p0, z1.s
-    MemOpCost += 3;
+    MemOpCost *= 4;
   }
 
   if (MICA.getID() == Intrinsic::masked_compressstore) {
@@ -5580,7 +5580,7 @@ AArch64TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
     //  cntp     x8, p1, p0.s
     //  whilelo  p0.s, xzr, x8
     //  st1w     { z0.s }, p0, [x0]
-    MemOpCost += 3;
+    MemOpCost *= 4;
   }
 
   // If we need to split the memory operation, we will also need to split the
diff --git a/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll b/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll
index 93a3781e25cd52c..941eee99a943399 100644
--- a/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll
@@ -67,8 +67,8 @@ define void @fixed() {
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2f32.p0(<2 x float> poison, ptr poison, <2 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v4f32.p0(<4 x float> poison, ptr poison, <4 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v2f64.p0(<2 x double> poison, ptr poison, <2 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 10 for: call void @llvm.masked.compressstore.v4i64.p0(<4 x i64> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 14 for: call void @llvm.masked.compressstore.v32f16.p0(<32 x half> poison, ptr poison, <32 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.v4i64.p0(<4 x i64> poison, ptr poison, <4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 32 for: call void @llvm.masked.compressstore.v32f16.p0(<32 x half> poison, ptr poison, <32 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-8-64-SVE256-LABEL: 'fixed'
@@ -89,7 +89,7 @@ define void @fixed() {
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v4f32.p0(<4 x float> poison, ptr poison, <4 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v2f64.p0(<2 x double> poison, ptr poison, <2 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.v4i64.p0(<4 x i64> poison, ptr poison, <4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 10 for: call void @llvm.masked.compressstore.v32f16.p0(<32 x half> poison, ptr poison, <32 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.v32f16.p0(<32 x half> poison, ptr poison, <32 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-UNAVAILABLE-LABEL: 'fixed'
@@ -160,7 +160,7 @@ define void @scalable() {
 ; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
 ; CHECK-32-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
 ; CHECK-32-64-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv1i64.p0(<vscale x 1 x i64> poison, ptr poison, <vscale x 1 x i1> poison)
-; CHECK-32-64-NEXT:  Cost Model: Found costs of 10 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-32-64-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
 ; CHECK-32-64-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
 ; CHECK-32-64-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
@@ -182,8 +182,8 @@ define void @scalable() {
 ; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
 ; CHECK-8-64-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
 ; CHECK-8-64-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv1i64.p0(<vscale x 1 x i64> poison, ptr poison, <vscale x 1 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 10 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-NEXT:  Cost Model: Found costs of 14 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-NEXT:  Cost Model: Found costs of 32 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
 ; CHECK-8-64-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-8-64-STREAMING-ONLY-LABEL: 'scalable'
@@ -204,8 +204,8 @@ define void @scalable() {
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv1i64.p0(<vscale x 1 x i64> poison, ptr poison, <vscale x 1 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 10 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 14 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of 32 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
 ; CHECK-8-64-STREAMING-ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-8-64-SVE256-LABEL: 'scalable'
@@ -226,8 +226,8 @@ define void @scalable() {
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 4 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv1i64.p0(<vscale x 1 x i64> poison, ptr poison, <vscale x 1 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 10 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
-; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 14 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
+; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of 32 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
 ; CHECK-8-64-SVE256-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-UNAVAILABLE-LABEL: 'scalable'
diff --git a/llvm/test/Analysis/CostModel/AArch64/masked_expand_load.ll b/llvm/test/Analysis/CostModel/AArch64/masked_expand_load.ll
index 75cc89c85aa9f68..1a477a52acd15bd 100644
--- a/llvm/test/Analysis/CostModel/AArch64/masked_expand_load.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/masked_expand_load.ll
@@ -45,8 +45,8 @@ define void @fixed() {
 ; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v2f32 = call <2 x float> @llvm.masked.expandload.v2f32.p0(ptr poison, <2 x i1> poison, <2 x float> poison)
 ; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v4f32 = call <4 x float> @llvm.masked.expandload.v4f32.p0(ptr poison, <4 x i1> poison, <4 x float> poison)
 ; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %v2f64 = call <2 x double> @llvm.masked.expandload.v2f64.p0(ptr poison, <2 x i1> poison, <2 x double> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 10 for: %v4i64 = call <4 x i64> @llvm.masked.expandload.v4i64.p0(ptr poison, <4 x i1> poison, <4 x i64> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 14 for: %v32f16 = call <32 x half> @llvm.masked.expandload.v32f16.p0(ptr poison, <32 x i1> poison, <32 x half> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 16 for: %v4i64 = call <4 x i64> @llvm.masked.expandload.v4i64.p0(ptr poison, <4 x i1> poison, <4 x i64> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 32 for: %v32f16 = call <32 x half> @llvm.masked.expandload.v32f16.p0(ptr poison, <32 x i1> poison, <32 x half> poison)
 ; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; SVE2p2-SME2p2-LABEL: 'fixed'
@@ -109,7 +109,7 @@ define void @fixed() {
 ; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v4f32 = call <4 x float> @llvm.masked.expandload.v4f32.p0(ptr poison, <4 x i1> poison, <4 x float> poison)
 ; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v2f64 = call <2 x double> @llvm.masked.expandload.v2f64.p0(ptr poison, <2 x i1> poison, <2 x double> poison)
 ; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %v4i64 = call <4 x i64> @llvm.masked.expandload.v4i64.p0(ptr poison, <4 x i1> poison, <4 x i64> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 10 for: %v32f16 = call <32 x half> @llvm.masked.expandload.v32f16.p0(ptr poison, <32 x i1> poison, <32 x half> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 16 for: %v32f16 = call <32 x half> @llvm.masked.expandload.v32f16.p0(ptr poison, <32 x i1> poison, <32 x half> poison)
 ; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 entry:
@@ -159,8 +159,8 @@ define void @scalable() {
 ; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
 ; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
 ; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of Invalid for: %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.expandload.nxv1i64.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x i64> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 10 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 14 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 16 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of 32 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
 ; SVE2p2-SME2p2-NON-STREAMING-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; SVE2p2-SME2p2-STREAMING-LABEL: 'scalable'
@@ -181,8 +181,8 @@ define void @scalable() {
 ; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
 ; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 4 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
 ; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of Invalid for: %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.expandload.nxv1i64.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x i64> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 10 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 14 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 16 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of 32 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
 ; SVE2p2-SME2p2-STREAMING-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; SVE2p2-SME2p2-LABEL: 'scalable'
@@ -203,8 +203,8 @@ define void @scalable() {
 ; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
 ; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 4 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
 ; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of Invalid for: %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.expandload.nxv1i64.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x i64> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 10 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
-; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 14 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 16 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
+; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of 32 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
 ; SVE2p2-SME2p2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; SVE-ONLY-LABEL: 'scalable'
@@ -247,8 +247,8 @@ define void @scalable() {
 ; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.expandload.nxv4f32.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x float> poison)
 ; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 4 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.expandload.nxv2f64.p0(ptr poison, <vscale x 2 x i1> poison, <vscale x 2 x double> poison)
 ; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of Invalid for: %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.expandload.nxv1i64.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x i64> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 10 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
-; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 14 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 16 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64.p0(ptr poison, <vscale x 4 x i1> poison, <vscale x 4 x i64> poison)
+; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of 32 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.expandload.nxv32f16.p0(ptr poison, <vscale x 32 x i1> poison, <vscale x 32 x half> poison)
 ; SVE2p2-SME2p2-SVE256-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 entry:

