[llvm-branch-commits] [llvm] [AArch64][CostModel] Consider some nxv1 operations as legal (PR #214471)
Gaëtan Bossu via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Aug 6 05:41:48 PDT 2026
https://github.com/gbossu updated https://github.com/llvm/llvm-project/pull/214471
>From 77e7110974b853e0096429ff718cdbc7577da592 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Tue, 7 Jul 2026 13:48:07 +0000
Subject: [PATCH 1/2] [AArch64][CostModel] Consider some nxv1 operations as
legal
This is allowing some operations on vscale x 1 types, namely:
- load/store
- masked load/store
- arithmetic instructions like add/sub/mul
For those, there is already codegen coverage. See e.g.
- llvm/test/CodeGen/AArch64/sve-int-arith.ll
- llvm/test/CodeGen/AArch64/sve-load-store-legalisation.ll
- llvm/test/CodeGen/AArch64/sve-masked-gather.ll
- llvm/test/CodeGen/AArch64/sve-masked-scatter.ll
---
.../AArch64/AArch64TargetTransformInfo.cpp | 50 +++++++++------
.../Analysis/CostModel/AArch64/masked_ldst.ll | 63 +++++++------------
.../Analysis/CostModel/AArch64/sve-arith.ll | 6 +-
.../Analysis/CostModel/AArch64/sve-ldst.ll | 8 +--
.../force-scalable-vectorization-always.ll | 2 +-
.../LoopVectorize/AArch64/invalid-costs.ll | 3 +-
.../LoopVectorize/AArch64/scalable-alloca.ll | 1 -
.../LoopVectorize/AArch64/scalable-call.ll | 6 --
.../scalable-vectorization-cost-tuning.ll | 6 +-
9 files changed, 65 insertions(+), 80 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index a380cca93d84d..bc2b2a78cad48 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -613,9 +613,12 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
// of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
// it. This change will be removed when code-generation for these types is
// sufficiently reliable.
+ // Only allow masked ld/st to pass through to getMemIntrinsicInstrCost().
auto *RetTy = ICA.getReturnType();
if (auto *VTy = dyn_cast<ScalableVectorType>(RetTy))
- if (VTy->getElementCount() == ElementCount::getScalable(1))
+ if (VTy->getElementCount() == ElementCount::getScalable(1) &&
+ !is_contained({Intrinsic::masked_load, Intrinsic::masked_store},
+ ICA.getID()))
return InstructionCost::getInvalid();
switch (ICA.getID()) {
@@ -4774,15 +4777,15 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
// The code-generator is currently not able to handle scalable vectors
// of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
- // it. This change will be removed when code-generation for these types is
- // sufficiently reliable.
+ // it until all instructions are vetted.
+ int ISD = TLI->InstructionOpcodeToISD(Opcode);
if (auto *VTy = dyn_cast<ScalableVectorType>(Ty))
if (VTy->getElementCount() == ElementCount::getScalable(1))
- return InstructionCost::getInvalid();
+ if (!is_contained({ISD::ADD, ISD::SUB, ISD::MUL}, ISD))
+ return InstructionCost::getInvalid();
// Legalize the type.
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
- int ISD = TLI->InstructionOpcodeToISD(Opcode);
// TODO: Handle more cost kinds for floating point operations.
if (ISD == ISD::FADD || ISD == ISD::FSUB || ISD == ISD::FMUL ||
@@ -5380,12 +5383,13 @@ AArch64TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
if (VT->getElementType()->isIntegerTy(1))
return InstructionCost::getInvalid();
- // The code-generator is currently not able to handle scalable vectors
- // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
- // it. This change will be removed when code-generation for these types is
- // sufficiently reliable.
+ // <vscale x 1 x eltty> operations require mask adaptation.
+ // Allow it for normal masked ld/st
if (VT->getElementCount() == ElementCount::getScalable(1))
- return InstructionCost::getInvalid();
+ return is_contained({Intrinsic::masked_load, Intrinsic::masked_store},
+ MICA.getID())
+ ? LT.first + 1
+ : InstructionCost::getInvalid();
InstructionCost MemOpCost = LT.first;
if (MICA.getID() == Intrinsic::masked_expandload) {
@@ -5507,17 +5511,23 @@ InstructionCost AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty,
if (!LT.first.isValid())
return InstructionCost::getInvalid();
- // The code-generator is currently not able to handle scalable vectors
- // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
- // it. This change will be removed when code-generation for these types is
- // sufficiently reliable.
- // We also only support full register predicate loads and stores.
- if (auto *VTy = dyn_cast<ScalableVectorType>(Ty))
- if (VTy->getElementCount() == ElementCount::getScalable(1) ||
- (VTy->getElementType()->isIntegerTy(1) &&
- !VTy->getElementCount().isKnownMultipleOf(
- ElementCount::getScalable(16))))
+ if (auto *VTy = dyn_cast<ScalableVectorType>(Ty)) {
+ // <vscale x 1 x eltty> operations require crafting a new mask.
+ if (VTy->getElementCount() == ElementCount::getScalable(1)) {
+ Intrinsic::ID IID = Opcode == Instruction::Load ? Intrinsic::masked_load
+ : Intrinsic::masked_store;
+ return getMaskedMemoryOpCost(
+ MemIntrinsicCostAttributes(IID, Ty, Alignment, AddressSpace),
+ CostKind) +
+ 1;
+ }
+
+ // We only support full register predicate loads and stores.
+ if (VTy->getElementType()->isIntegerTy(1) &&
+ !VTy->getElementCount().isKnownMultipleOf(
+ ElementCount::getScalable(16)))
return InstructionCost::getInvalid();
+ }
// TODO: consider latency as well for TCK_SizeAndLatency.
if (CostKind == TTI::TCK_CodeSize || CostKind == TTI::TCK_SizeAndLatency)
diff --git a/llvm/test/Analysis/CostModel/AArch64/masked_ldst.ll b/llvm/test/Analysis/CostModel/AArch64/masked_ldst.ll
index c1ee229ae8e2c..f9d4944d0384b 100644
--- a/llvm/test/Analysis/CostModel/AArch64/masked_ldst.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/masked_ldst.ll
@@ -72,7 +72,7 @@ define void @scalable() {
; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.load.nxv2f32.p0(ptr align 8 undef, <vscale x 2 x i1> undef, <vscale x 2 x float> undef)
; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 8 undef, <vscale x 4 x i1> undef, <vscale x 4 x float> undef)
; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 undef, <vscale x 2 x i1> undef, <vscale x 2 x double> undef)
-; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.load.nxv1i64.p0(ptr align 8 undef, <vscale x 1 x i1> undef, <vscale x 1 x i64> undef)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.load.nxv1i64.p0(ptr align 8 undef, <vscale x 1 x i1> undef, <vscale x 1 x i64> undef)
; CHECK-NEXT: Cost Model: Found costs of 4 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.load.nxv4i64.p0(ptr align 8 undef, <vscale x 4 x i1> undef, <vscale x 4 x i64> undef)
; CHECK-NEXT: Cost Model: Found costs of 8 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.load.nxv32f16.p0(ptr align 8 undef, <vscale x 32 x i1> undef, <vscale x 32 x half> undef)
; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nxv4i1 = call <vscale x 4 x i1> @llvm.masked.load.nxv4i1.p0(ptr align 16 undef, <vscale x 4 x i1> undef, <vscale x 4 x i1> undef)
@@ -250,44 +250,25 @@ define void @scalable_ext_loads() {
ret void
}
+define void @scalable_nxv1() {
+; CHECK-LABEL: 'scalable_nxv1'
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1i8 = call <vscale x 1 x i8> @llvm.masked.load.nxv1i8.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x i8> undef)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1i16 = call <vscale x 1 x i16> @llvm.masked.load.nxv1i16.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x i16> undef)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1i32 = call <vscale x 1 x i32> @llvm.masked.load.nxv1i32.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x i32> undef)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.load.nxv1i64.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x i64> undef)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1f16 = call <vscale x 1 x half> @llvm.masked.load.nxv1f16.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x half> undef)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1f32 = call <vscale x 1 x float> @llvm.masked.load.nxv1f32.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x float> undef)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1f64 = call <vscale x 1 x double> @llvm.masked.load.nxv1f64.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x double> undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+entry:
+ %nxv1i8 = call <vscale x 1 x i8> @llvm.masked.load.nxv1i8.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x i8> undef)
+ %nxv1i16 = call <vscale x 1 x i16> @llvm.masked.load.nxv1i16.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x i16> undef)
+ %nxv1i32 = call <vscale x 1 x i32> @llvm.masked.load.nxv1i32.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x i32> undef)
+ %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.load.nxv1i64.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x i64> undef)
+ %nxv1f16 = call <vscale x 1 x half> @llvm.masked.load.nxv1f16.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x half> undef)
+ %nxv1f32 = call <vscale x 1 x float> @llvm.masked.load.nxv1f32.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x float> undef)
+ %nxv1f64 = call <vscale x 1 x double> @llvm.masked.load.nxv1f64.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x double> undef)
-declare <2 x i8> @llvm.masked.load.v2i8.p0(ptr, i32, <2 x i1>, <2 x i8>)
-declare <4 x i8> @llvm.masked.load.v4i8.p0(ptr, i32, <4 x i1>, <4 x i8>)
-declare <8 x i8> @llvm.masked.load.v8i8.p0(ptr, i32, <8 x i1>, <8 x i8>)
-declare <16 x i8> @llvm.masked.load.v16i8.p0(ptr, i32, <16 x i1>, <16 x i8>)
-declare <2 x i16> @llvm.masked.load.v2i16.p0(ptr, i32, <2 x i1>, <2 x i16>)
-declare <4 x i16> @llvm.masked.load.v4i16.p0(ptr, i32, <4 x i1>, <4 x i16>)
-declare <8 x i16> @llvm.masked.load.v8i16.p0(ptr, i32, <8 x i1>, <8 x i16>)
-declare <2 x i32> @llvm.masked.load.v2i32.p0(ptr, i32, <2 x i1>, <2 x i32>)
-declare <4 x i32> @llvm.masked.load.v4i32.p0(ptr, i32, <4 x i1>, <4 x i32>)
-declare <2 x i64> @llvm.masked.load.v2i64.p0(ptr, i32, <2 x i1>, <2 x i64>)
-declare <4 x i64> @llvm.masked.load.v4i64.p0(ptr, i32, <4 x i1>, <4 x i64>)
-declare <2 x half> @llvm.masked.load.v2f16.p0(ptr, i32, <2 x i1>, <2 x half>)
-declare <4 x half> @llvm.masked.load.v4f16.p0(ptr, i32, <4 x i1>, <4 x half>)
-declare <8 x half> @llvm.masked.load.v8f16.p0(ptr, i32, <8 x i1>, <8 x half>)
-declare <32 x half> @llvm.masked.load.v32f16.p0(ptr, i32, <32 x i1>, <32 x half>)
-declare <2 x float> @llvm.masked.load.v2f32.p0(ptr, i32, <2 x i1>, <2 x float>)
-declare <4 x float> @llvm.masked.load.v4f32.p0(ptr, i32, <4 x i1>, <4 x float>)
-declare <2 x double> @llvm.masked.load.v2f64.p0(ptr, i32, <2 x i1>, <2 x double>)
-declare <vscale x 4 x i1> @llvm.masked.load.nxv4i1.p0(ptr, i32, <vscale x 4 x i1>, <vscale x 4 x i1>)
-
-
-declare <vscale x 2 x i8> @llvm.masked.load.nxv2i8.p0(ptr, i32, <vscale x 2 x i1>, <vscale x 2 x i8>)
-declare <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr, i32, <vscale x 4 x i1>, <vscale x 4 x i8>)
-declare <vscale x 8 x i8> @llvm.masked.load.nxv8i8.p0(ptr, i32, <vscale x 8 x i1>, <vscale x 8 x i8>)
-declare <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr, i32, <vscale x 16 x i1>, <vscale x 16 x i8>)
-declare <vscale x 2 x i16> @llvm.masked.load.nxv2i16.p0(ptr, i32, <vscale x 2 x i1>, <vscale x 2 x i16>)
-declare <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr, i32, <vscale x 4 x i1>, <vscale x 4 x i16>)
-declare <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr, i32, <vscale x 8 x i1>, <vscale x 8 x i16>)
-declare <vscale x 2 x i32> @llvm.masked.load.nxv2i32.p0(ptr, i32, <vscale x 2 x i1>, <vscale x 2 x i32>)
-declare <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr, i32, <vscale x 4 x i1>, <vscale x 4 x i32>)
-declare <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)
-declare <vscale x 4 x i64> @llvm.masked.load.nxv4i64.p0(ptr, i32, <vscale x 4 x i1>, <vscale x 4 x i64>)
-declare <vscale x 1 x i64> @llvm.masked.load.nxv1i64.p0(ptr, i32, <vscale x 1 x i1>, <vscale x 1 x i64>)
-declare <vscale x 2 x half> @llvm.masked.load.nxv2f16.p0(ptr, i32, <vscale x 2 x i1>, <vscale x 2 x half>)
-declare <vscale x 4 x half> @llvm.masked.load.nxv4f16.p0(ptr, i32, <vscale x 4 x i1>, <vscale x 4 x half>)
-declare <vscale x 8 x half> @llvm.masked.load.nxv8f16.p0(ptr, i32, <vscale x 8 x i1>, <vscale x 8 x half>)
-declare <vscale x 32 x half> @llvm.masked.load.nxv32f16.p0(ptr, i32, <vscale x 32 x i1>, <vscale x 32 x half>)
-declare <vscale x 2 x float> @llvm.masked.load.nxv2f32.p0(ptr, i32, <vscale x 2 x i1>, <vscale x 2 x float>)
-declare <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr, i32, <vscale x 4 x i1>, <vscale x 4 x float>)
-declare <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr, i32, <vscale x 2 x i1>, <vscale x 2 x double>)
+ ret void
+}
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-arith.ll b/llvm/test/Analysis/CostModel/AArch64/sve-arith.ll
index 0a9823afdeff5..ecca4fea2e520 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-arith.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-arith.ll
@@ -9,7 +9,7 @@ define void @scalable_add() #0 {
; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv8i16 = add <vscale x 8 x i16> undef, undef
; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv4i32 = add <vscale x 4 x i32> undef, undef
; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv2i64 = add <vscale x 2 x i64> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nxv1i64 = add <vscale x 1 x i64> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv1i64 = add <vscale x 1 x i64> undef, undef
; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nxv2i128 = add <vscale x 2 x i128> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
@@ -30,7 +30,7 @@ define void @scalable_sub() #0 {
; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv8i16 = sub <vscale x 8 x i16> undef, undef
; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv4i32 = sub <vscale x 4 x i32> undef, undef
; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv2i64 = sub <vscale x 2 x i64> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nxv1i64 = sub <vscale x 1 x i64> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv1i64 = sub <vscale x 1 x i64> undef, undef
; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nxv2i128 = sub <vscale x 2 x i128> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
@@ -51,7 +51,7 @@ define void @scalable_mul() #0 {
; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv8i16 = mul <vscale x 8 x i16> undef, undef
; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv4i32 = mul <vscale x 4 x i32> undef, undef
; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv2i64 = mul <vscale x 2 x i64> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nxv1i64 = mul <vscale x 1 x i64> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv1i64 = mul <vscale x 1 x i64> undef, undef
; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nxv2i128 = mul <vscale x 2 x i128> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-ldst.ll b/llvm/test/Analysis/CostModel/AArch64/sve-ldst.ll
index b52fddaf8e776..e6e029981156b 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-ldst.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-ldst.ll
@@ -9,7 +9,7 @@ define void @scalable_loads() {
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %res.nxv8i8 = load <vscale x 8 x i8>, ptr undef, align 8
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %res.nxv16i8 = load <vscale x 16 x i8>, ptr undef, align 16
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %res.nxv32i8 = load <vscale x 32 x i8>, ptr undef, align 32
-; CHECK-NEXT: Cost Model: Found costs of Invalid for: %res.nxv1i64 = load <vscale x 1 x i64>, ptr undef, align 8
+; CHECK-NEXT: Cost Model: Found costs of 3 for: %res.nxv1i64 = load <vscale x 1 x i64>, ptr undef, align 8
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:5 SizeLat:2 for: %res.nxv32i1 = load <vscale x 32 x i1>, ptr undef, align 4
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %res.nxv16i1 = load <vscale x 16 x i1>, ptr undef, align 2
; CHECK-NEXT: Cost Model: Found costs of Invalid for: %res.nxv4i1 = load <vscale x 4 x i1>, ptr undef, align 1
@@ -19,7 +19,7 @@ define void @scalable_loads() {
; A510-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %res.nxv8i8 = load <vscale x 8 x i8>, ptr undef, align 8
; A510-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %res.nxv16i8 = load <vscale x 16 x i8>, ptr undef, align 16
; A510-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:3 SizeLat:2 for: %res.nxv32i8 = load <vscale x 32 x i8>, ptr undef, align 32
-; A510-NEXT: Cost Model: Found costs of Invalid for: %res.nxv1i64 = load <vscale x 1 x i64>, ptr undef, align 8
+; A510-NEXT: Cost Model: Found costs of 3 for: %res.nxv1i64 = load <vscale x 1 x i64>, ptr undef, align 8
; A510-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:3 SizeLat:2 for: %res.nxv32i1 = load <vscale x 32 x i1>, ptr undef, align 4
; A510-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %res.nxv16i1 = load <vscale x 16 x i1>, ptr undef, align 2
; A510-NEXT: Cost Model: Found costs of Invalid for: %res.nxv4i1 = load <vscale x 4 x i1>, ptr undef, align 1
@@ -40,7 +40,7 @@ define void @scalable_stores() {
; CHECK-NEXT: Cost Model: Found costs of 1 for: store <vscale x 8 x i8> undef, ptr undef, align 8
; CHECK-NEXT: Cost Model: Found costs of 1 for: store <vscale x 16 x i8> undef, ptr undef, align 16
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <vscale x 32 x i8> undef, ptr undef, align 32
-; CHECK-NEXT: Cost Model: Found costs of Invalid for: store <vscale x 1 x i64> undef, ptr undef, align 8
+; CHECK-NEXT: Cost Model: Found costs of 3 for: store <vscale x 1 x i64> undef, ptr undef, align 8
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <vscale x 32 x i1> undef, ptr undef, align 4
; CHECK-NEXT: Cost Model: Found costs of 1 for: store <vscale x 16 x i1> undef, ptr undef, align 2
; CHECK-NEXT: Cost Model: Found costs of Invalid for: store <vscale x 4 x i1> undef, ptr undef, align 1
@@ -50,7 +50,7 @@ define void @scalable_stores() {
; A510-NEXT: Cost Model: Found costs of 1 for: store <vscale x 8 x i8> undef, ptr undef, align 8
; A510-NEXT: Cost Model: Found costs of 1 for: store <vscale x 16 x i8> undef, ptr undef, align 16
; A510-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <vscale x 32 x i8> undef, ptr undef, align 32
-; A510-NEXT: Cost Model: Found costs of Invalid for: store <vscale x 1 x i64> undef, ptr undef, align 8
+; A510-NEXT: Cost Model: Found costs of 3 for: store <vscale x 1 x i64> undef, ptr undef, align 8
; A510-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <vscale x 32 x i1> undef, ptr undef, align 4
; A510-NEXT: Cost Model: Found costs of 1 for: store <vscale x 16 x i1> undef, ptr undef, align 2
; A510-NEXT: Cost Model: Found costs of Invalid for: store <vscale x 4 x i1> undef, ptr undef, align 1
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/force-scalable-vectorization-always.ll b/llvm/test/Transforms/LoopVectorize/AArch64/force-scalable-vectorization-always.ll
index e2ab17e6cfe8f..ad7bf169db96d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/force-scalable-vectorization-always.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/force-scalable-vectorization-always.ll
@@ -11,7 +11,7 @@ define i32 @cost_prefers_fixed_width_vf_but_force_scalable_vf(ptr noalias %dst,
; CHECK: Cost for VF 2: 12 (Estimated cost per lane: 6)
; CHECK: Cost for VF 4: 8 (Estimated cost per lane: 2)
; CHECK: Cost for VF 8: 10 (Estimated cost per lane: 1.25)
-; CHECK: Cost for VF vscale x 1: Invalid (Estimated cost per lane: Invalid)
+; CHECK: Cost for VF vscale x 1: 11 (Estimated cost per lane: 11)
; CHECK: Cost for VF vscale x 2: 9 (Estimated cost per lane: 4.5)
; CHECK: Cost for VF vscale x 4: 8 (Estimated cost per lane: 2)
; CHECK: LV: Selecting VF: vscale x 4.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/invalid-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/invalid-costs.ll
index 37b6362c12927..ac1f598e1a85c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/invalid-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/invalid-costs.ll
@@ -4,7 +4,8 @@
target triple = "arm64-apple-macosx"
-; REMARKS: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): load
+; REMARKS: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): ashr
+; REMARKS: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): call to llvm.masked.sdiv
; Test case for https://github.com/llvm/llvm-project/issues/160792.
define void @replicate_sdiv_conditional(ptr noalias %a, ptr noalias %b, ptr noalias %c) #0 {
; CHECK-LABEL: define void @replicate_sdiv_conditional(
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-alloca.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-alloca.ll
index 0ef03c58be97a..57f407cf99be5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-alloca.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-alloca.ll
@@ -3,7 +3,6 @@
; CHECK-REMARKS: UserVF ignored because of invalid costs.
; CHECK-REMARKS: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): alloca
-; CHECK-REMARKS: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): store
define void @alloca(ptr %vla, i64 %N) {
; CHECK-LABEL: @alloca(
; CHECK-NOT: <vscale x
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll
index 0459b98f2eb1d..f5e32e4646df3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll
@@ -101,9 +101,7 @@ for.end:
}
; CHECK-REMARKS: UserVF ignored because of invalid costs.
-; CHECK-REMARKS-NEXT: t.c:3:10: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): load
; CHECK-REMARKS-NEXT: t.c:3:20: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin
-; CHECK-REMARKS-NEXT: t.c:3:30: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): store
define void @vec_sin_no_mapping(ptr noalias nocapture %dst, ptr noalias nocapture readonly %src, i64 %n) {
; CHECK: @vec_sin_no_mapping
; CHECK: call fast <2 x float> @llvm.sin.v2f32
@@ -127,11 +125,9 @@ for.cond.cleanup:
}
; CHECK-REMARKS: UserVF ignored because of invalid costs.
-; CHECK-REMARKS-NEXT: t.c:3:10: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): load
; CHECK-REMARKS-NEXT: t.c:3:30: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): fadd
; CHECK-REMARKS-NEXT: t.c:3:30: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin
; CHECK-REMARKS-NEXT: t.c:3:20: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin
-; CHECK-REMARKS-NEXT: t.c:3:40: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): store
define void @vec_sin_no_mapping_ite(ptr noalias nocapture %dst, ptr noalias nocapture readonly %src, i64 %n) {
; CHECK: @vec_sin_no_mapping_ite
; CHECK-NOT: <vscale x
@@ -165,9 +161,7 @@ for.cond.cleanup:
}
; CHECK-REMARKS: UserVF ignored because of invalid costs.
-; CHECK-REMARKS-NEXT: t.c:3:10: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): load
; CHECK-REMARKS-NEXT: t.c:3:20: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin
-; CHECK-REMARKS-NEXT: t.c:3:30: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): store
define void @vec_sin_fixed_mapping(ptr noalias nocapture %dst, ptr noalias nocapture readonly %src, i64 %n) {
; CHECK: @vec_sin_fixed_mapping
; CHECK: call fast <2 x float> @llvm.sin.v2f32
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-vectorization-cost-tuning.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-vectorization-cost-tuning.ll
index 33b8f22bd048e..8d496944b9d85 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-vectorization-cost-tuning.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-vectorization-cost-tuning.ll
@@ -26,7 +26,7 @@
define void @test0(ptr %a, ptr %b, ptr %c) #0 {
; VSCALEFORTUNING1-LABEL: 'test0'
-; VSCALEFORTUNING1: Cost for VF vscale x 1: Invalid (Estimated cost per lane: Invalid)
+; VSCALEFORTUNING1: Cost for VF vscale x 1: 10 (Estimated cost per lane: 10)
; VSCALEFORTUNING1: Cost for VF vscale x 2: 10 (Estimated cost per lane: 5)
; VSCALEFORTUNING1: Cost for VF vscale x 4: 10 (Estimated cost per lane: 2.5)
; VSCALEFORTUNING1: Cost for VF vscale x 8: 10 (Estimated cost per lane: 1.25)
@@ -34,7 +34,7 @@ define void @test0(ptr %a, ptr %b, ptr %c) #0 {
; VSCALEFORTUNING1: LV: Selecting VF: vscale x 16.
;
; VSCALEFORTUNING2-LABEL: 'test0'
-; VSCALEFORTUNING2: Cost for VF vscale x 1: Invalid (Estimated cost per lane: Invalid)
+; VSCALEFORTUNING2: Cost for VF vscale x 1: 10 (Estimated cost per lane: 5)
; VSCALEFORTUNING2: Cost for VF vscale x 2: 10 (Estimated cost per lane: 2.5)
; VSCALEFORTUNING2: Cost for VF vscale x 4: 10 (Estimated cost per lane: 1.25)
; VSCALEFORTUNING2: Cost for VF vscale x 8: 10 (Estimated cost per lane: 0.625)
@@ -42,7 +42,7 @@ define void @test0(ptr %a, ptr %b, ptr %c) #0 {
; VSCALEFORTUNING2: LV: Selecting VF: vscale x 16.
;
; VSCALEFORTUNING1-PREFER-FIXED-LABEL: 'test0'
-; VSCALEFORTUNING1-PREFER-FIXED: Cost for VF vscale x 1: Invalid (Estimated cost per lane: Invalid)
+; VSCALEFORTUNING1-PREFER-FIXED: Cost for VF vscale x 1: 10 (Estimated cost per lane: 10)
; VSCALEFORTUNING1-PREFER-FIXED: Cost for VF vscale x 2: 10 (Estimated cost per lane: 5)
; VSCALEFORTUNING1-PREFER-FIXED: Cost for VF vscale x 4: 10 (Estimated cost per lane: 2.5)
; VSCALEFORTUNING1-PREFER-FIXED: Cost for VF vscale x 8: 10 (Estimated cost per lane: 1.25)
>From 439fe9a96aecc17f6119eb37c767876f20aa0ce4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Thu, 6 Aug 2026 12:41:05 +0000
Subject: [PATCH 2/2] Avoid new undef in IR
---
.../Analysis/CostModel/AArch64/masked_ldst.ll | 28 +++++++++----------
1 file changed, 14 insertions(+), 14 deletions(-)
diff --git a/llvm/test/Analysis/CostModel/AArch64/masked_ldst.ll b/llvm/test/Analysis/CostModel/AArch64/masked_ldst.ll
index f9d4944d0384b..7aeba01cf321e 100644
--- a/llvm/test/Analysis/CostModel/AArch64/masked_ldst.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/masked_ldst.ll
@@ -252,23 +252,23 @@ define void @scalable_ext_loads() {
define void @scalable_nxv1() {
; CHECK-LABEL: 'scalable_nxv1'
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1i8 = call <vscale x 1 x i8> @llvm.masked.load.nxv1i8.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x i8> undef)
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1i16 = call <vscale x 1 x i16> @llvm.masked.load.nxv1i16.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x i16> undef)
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1i32 = call <vscale x 1 x i32> @llvm.masked.load.nxv1i32.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x i32> undef)
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.load.nxv1i64.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x i64> undef)
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1f16 = call <vscale x 1 x half> @llvm.masked.load.nxv1f16.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x half> undef)
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1f32 = call <vscale x 1 x float> @llvm.masked.load.nxv1f32.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x float> undef)
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1f64 = call <vscale x 1 x double> @llvm.masked.load.nxv1f64.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x double> undef)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1i8 = call <vscale x 1 x i8> @llvm.masked.load.nxv1i8.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x i8> poison)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1i16 = call <vscale x 1 x i16> @llvm.masked.load.nxv1i16.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x i16> poison)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1i32 = call <vscale x 1 x i32> @llvm.masked.load.nxv1i32.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x i32> poison)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.load.nxv1i64.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x i64> poison)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1f16 = call <vscale x 1 x half> @llvm.masked.load.nxv1f16.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x half> poison)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1f32 = call <vscale x 1 x float> @llvm.masked.load.nxv1f32.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x float> poison)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1f64 = call <vscale x 1 x double> @llvm.masked.load.nxv1f64.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x double> poison)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
entry:
- %nxv1i8 = call <vscale x 1 x i8> @llvm.masked.load.nxv1i8.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x i8> undef)
- %nxv1i16 = call <vscale x 1 x i16> @llvm.masked.load.nxv1i16.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x i16> undef)
- %nxv1i32 = call <vscale x 1 x i32> @llvm.masked.load.nxv1i32.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x i32> undef)
- %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.load.nxv1i64.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x i64> undef)
- %nxv1f16 = call <vscale x 1 x half> @llvm.masked.load.nxv1f16.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x half> undef)
- %nxv1f32 = call <vscale x 1 x float> @llvm.masked.load.nxv1f32.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x float> undef)
- %nxv1f64 = call <vscale x 1 x double> @llvm.masked.load.nxv1f64.p0(ptr undef, <vscale x 1 x i1> undef, <vscale x 1 x double> undef)
+ %nxv1i8 = call <vscale x 1 x i8> @llvm.masked.load.nxv1i8.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x i8> poison)
+ %nxv1i16 = call <vscale x 1 x i16> @llvm.masked.load.nxv1i16.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x i16> poison)
+ %nxv1i32 = call <vscale x 1 x i32> @llvm.masked.load.nxv1i32.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x i32> poison)
+ %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.load.nxv1i64.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x i64> poison)
+ %nxv1f16 = call <vscale x 1 x half> @llvm.masked.load.nxv1f16.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x half> poison)
+ %nxv1f32 = call <vscale x 1 x float> @llvm.masked.load.nxv1f32.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x float> poison)
+ %nxv1f64 = call <vscale x 1 x double> @llvm.masked.load.nxv1f64.p0(ptr poison, <vscale x 1 x i1> poison, <vscale x 1 x double> poison)
ret void
}
More information about the llvm-branch-commits
mailing list