[llvm] [AArch64] Add ext mask shuffle costs (PR #226136)
David Green via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 24 05:01:58 PDT 2026
https://github.com/davemgreen created https://github.com/llvm/llvm-project/pull/226136
These were not being handled by the existing shuffle mask costs, leading to higher fallback costs being used. Common the two isEXTMask/isSingletonEXTMask methods between GISel and SDAG into one and make use of it in the costmodel.
>From cc8eaef291a8f684bbd57cbd0e3169084eaf4eef Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Wed, 23 Sep 2026 17:29:06 +0100
Subject: [PATCH 1/2] [AArch64][GlobalISel] Common GISel and SDAG isEXTMask
methods. NFC
We had two of these methods, and want another in the cost model. Make the two
existing version in GISel and SDAG equal.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 84 ++-------------
.../Target/AArch64/AArch64PerfectShuffle.h | 73 +++++++++++++
.../GISel/AArch64PostLegalizerLowering.cpp | 102 +++---------------
3 files changed, 95 insertions(+), 164 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index d90030ee5519e..4479de9b2a5c1 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -15186,37 +15186,6 @@ SDValue AArch64TargetLowering::ReconstructShuffle(SDValue Op,
return V;
}
-// check if an EXT instruction can handle the shuffle mask when the
-// vector sources of the shuffle are the same.
-static bool isSingletonEXTMask(ArrayRef<int> M, EVT VT, unsigned &Imm) {
- unsigned NumElts = VT.getVectorNumElements();
-
- // Assume that the first shuffle index is not UNDEF. Fail if it is.
- if (M[0] < 0)
- return false;
-
- Imm = M[0];
-
- // If this is a VEXT shuffle, the immediate value is the index of the first
- // element. The other shuffle indices must be the successive elements after
- // the first one.
- unsigned ExpectedElt = Imm;
- for (unsigned i = 1; i < NumElts; ++i) {
- // Increment the expected index. If it wraps around, just follow it
- // back to index zero and keep going.
- ++ExpectedElt;
- if (ExpectedElt == NumElts)
- ExpectedElt = 0;
-
- if (M[i] < 0)
- continue; // ignore UNDEF indices
- if (ExpectedElt != static_cast<unsigned>(M[i]))
- return false;
- }
-
- return true;
-}
-
// Detect patterns of a0,a1,a2,a3,b0,b1,b2,b3,c0,c1,c2,c3,d0,d1,d2,d3 from
// v4i32s. This is really a truncate, which we can construct out of (legal)
// concats and truncate nodes.
@@ -15342,48 +15311,6 @@ static bool isWideDUPMask(ArrayRef<int> M, EVT VT, unsigned BlockSize,
return true;
}
-// check if an EXT instruction can handle the shuffle mask when the
-// vector sources of the shuffle are different.
-static bool isEXTMask(ArrayRef<int> M, EVT VT, bool &ReverseEXT,
- unsigned &Imm) {
- // Look for the first non-undef element.
- const int *FirstRealElt = find_if(M, [](int Elt) { return Elt >= 0; });
-
- // Benefit from APInt to handle overflow when calculating expected element.
- unsigned NumElts = VT.getVectorNumElements();
- unsigned MaskBits = APInt(32, NumElts * 2).logBase2();
- APInt ExpectedElt = APInt(MaskBits, *FirstRealElt + 1, /*isSigned=*/false,
- /*implicitTrunc=*/true);
- // The following shuffle indices must be the successive elements after the
- // first real element.
- bool FoundWrongElt = std::any_of(FirstRealElt + 1, M.end(), [&](int Elt) {
- return Elt != ExpectedElt++ && Elt >= 0;
- });
- if (FoundWrongElt)
- return false;
-
- // The index of an EXT is the first element if it is not UNDEF.
- // Watch out for the beginning UNDEFs. The EXT index should be the expected
- // value of the first element. E.g.
- // <-1, -1, 3, ...> is treated as <1, 2, 3, ...>.
- // <-1, -1, 0, 1, ...> is treated as <2*NumElts-2, 2*NumElts-1, 0, 1, ...>.
- // ExpectedElt is the last mask index plus 1.
- Imm = ExpectedElt.getZExtValue();
-
- // There are two difference cases requiring to reverse input vectors.
- // For example, for vector <4 x i32> we have the following cases,
- // Case 1: shufflevector(<4 x i32>,<4 x i32>,<-1, -1, -1, 0>)
- // Case 2: shufflevector(<4 x i32>,<4 x i32>,<-1, -1, 7, 0>)
- // For both cases, we finally use mask <5, 6, 7, 0>, which requires
- // to reverse two input vectors.
- if (Imm < NumElts)
- ReverseEXT = true;
- else
- Imm -= NumElts;
-
- return true;
-}
-
/// Flag slide shuffle patterns where one operand is zeros.
/// Left slide: shufflevector %v, zeros, <1,2,3,...> -> ushr
/// Right slide: shufflevector zeros, %v, <N-1,N,N+1,...> -> shl
@@ -16248,13 +16175,14 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
bool ReverseEXT = false;
unsigned Imm;
- if (isEXTMask(ShuffleMask, VT, ReverseEXT, Imm)) {
+ if (isEXTMask(ShuffleMask, NumElts, ReverseEXT, Imm)) {
if (ReverseEXT)
std::swap(V1, V2);
Imm *= getExtFactor(V1);
return DAG.getNode(AArch64ISD::EXT, DL, V1.getValueType(), V1, V2,
DAG.getConstant(Imm, DL, MVT::i32));
- } else if (V2->isUndef() && isSingletonEXTMask(ShuffleMask, VT, Imm)) {
+ }
+ if (V2->isUndef() && isSingletonEXTMask(ShuffleMask, NumElts, Imm)) {
Imm *= getExtFactor(V1);
return DAG.getNode(AArch64ISD::EXT, DL, V1.getValueType(), V1, V1,
DAG.getConstant(Imm, DL, MVT::i32));
@@ -18036,8 +17964,8 @@ bool AArch64TargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
isREVMask(M, EltSize, NumElts, 64) ||
isREVMask(M, EltSize, NumElts, 32) ||
isREVMask(M, EltSize, NumElts, 16) ||
- isEXTMask(M, VT, DummyBool, DummyUnsigned) ||
- isSingletonEXTMask(M, VT, DummyUnsigned) ||
+ isEXTMask(M, NumElts, DummyBool, DummyUnsigned) ||
+ isSingletonEXTMask(M, NumElts, DummyUnsigned) ||
isTRNMask(M, NumElts, DummyUnsigned, DummyUnsigned) ||
isUZPMask(M, NumElts, DummyUnsigned) ||
isZIPMask(M, NumElts, DummyUnsigned, DummyUnsigned) ||
@@ -35857,7 +35785,7 @@ SDValue AArch64TargetLowering::LowerFixedLengthVECTOR_SHUFFLEToSVE(
bool ReverseEXT = false;
unsigned Imm;
- if (isEXTMask(ShuffleMask, VT, ReverseEXT, Imm) &&
+ if (isEXTMask(ShuffleMask, VT.getVectorNumElements(), ReverseEXT, Imm) &&
Imm == VT.getVectorNumElements() - 1) {
if (ReverseEXT)
std::swap(Op1, Op2);
diff --git a/llvm/lib/Target/AArch64/AArch64PerfectShuffle.h b/llvm/lib/Target/AArch64/AArch64PerfectShuffle.h
index 00d9edbbc02ea..c508b74e6e210 100644
--- a/llvm/lib/Target/AArch64/AArch64PerfectShuffle.h
+++ b/llvm/lib/Target/AArch64/AArch64PerfectShuffle.h
@@ -14,6 +14,7 @@
#ifndef LLVM_LIB_TARGET_AARCH64_AARCH64PERFECTSHUFFLE_H
#define LLVM_LIB_TARGET_AARCH64_AARCH64PERFECTSHUFFLE_H
+#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
@@ -264,6 +265,78 @@ inline bool isREVMask(ArrayRef<int> M, unsigned EltSize, unsigned NumElts,
return true;
}
+/// Check if an EXT instruction can handle the shuffle mask when the
+/// vector sources of the shuffle are different.
+inline bool isEXTMask(ArrayRef<int> M, unsigned NumElts, bool &ReverseEXT,
+ unsigned &Imm) {
+ // Look for the first non-undef element.
+ const int *FirstRealElt = find_if(M, [](int Elt) { return Elt >= 0; });
+ if (FirstRealElt == M.end())
+ return false;
+
+ // Benefit from APInt to handle overflow when calculating expected element.
+ unsigned MaskBits = APInt(32, NumElts * 2).logBase2();
+ APInt ExpectedElt = APInt(MaskBits, *FirstRealElt + 1, /*isSigned=*/false,
+ /*implicitTrunc=*/true);
+ // The following shuffle indices must be the successive elements after the
+ // first real element.
+ if (std::any_of(FirstRealElt + 1, M.end(),
+ [&](int Elt) { return Elt != ExpectedElt++ && Elt >= 0; }))
+ return false;
+
+ // The index of an EXT is the first element if it is not UNDEF.
+ // Watch out for the beginning UNDEFs. The EXT index should be the expected
+ // value of the first element. E.g.
+ // <-1, -1, 3, ...> is treated as <1, 2, 3, ...>.
+ // <-1, -1, 0, 1, ...> is treated as <2*NumElts-2, 2*NumElts-1, 0, 1, ...>.
+ // ExpectedElt is the last mask index plus 1.
+ Imm = ExpectedElt.getZExtValue();
+
+ // There are two difference cases requiring to reverse input vectors.
+ // For example, for vector <4 x i32> we have the following cases,
+ // Case 1: shufflevector(<4 x i32>, <4 x i32>, <-1, -1, -1, 0>)
+ // Case 2: shufflevector(<4 x i32>, <4 x i32>, <-1, -1, 7, 0>)
+ // For both cases, we finally use mask <5, 6, 7, 0>, which requires
+ // to reverse two input vectors.
+ ReverseEXT = false;
+ if (Imm < NumElts)
+ ReverseEXT = true;
+ else
+ Imm -= NumElts;
+
+ return true;
+}
+
+/// check if an EXT instruction can handle the shuffle mask when the
+/// vector sources of the shuffle are the same. i.e <2,3,0,1>
+inline bool isSingletonEXTMask(ArrayRef<int> M, unsigned NumElts,
+ unsigned &Imm) {
+ // Assume that the first shuffle index is not UNDEF. Fail if it is.
+ if (M[0] < 0)
+ return false;
+
+ Imm = M[0];
+
+ // If this is a VEXT shuffle, the immediate value is the index of the first
+ // element. The other shuffle indices must be the successive elements after
+ // the first one.
+ unsigned ExpectedElt = Imm;
+ for (unsigned I = 1; I < NumElts; ++I) {
+ // Increment the expected index. If it wraps around, just follow it
+ // back to index zero and keep going.
+ ++ExpectedElt;
+ if (ExpectedElt == NumElts)
+ ExpectedElt = 0;
+
+ if (M[I] < 0)
+ continue; // Ignore UNDEF indices.
+ if (ExpectedElt != static_cast<unsigned>(M[I]))
+ return false;
+ }
+
+ return true;
+}
+
/// isDUPQMask - matches a splat of equivalent lanes within segments of a given
/// number of elements.
inline std::optional<unsigned> isDUPQMask(ArrayRef<int> Mask, unsigned Segments,
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
index f6f6008003af6..092f6bf336914 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
@@ -77,48 +77,6 @@ struct ShuffleVectorPseudo {
ShuffleVectorPseudo() = default;
};
-/// Check if a G_EXT instruction can handle a shuffle mask \p M when the vector
-/// sources of the shuffle are different.
-std::optional<std::pair<bool, uint64_t>> getExtMask(ArrayRef<int> M,
- unsigned NumElts) {
- // Look for the first non-undef element.
- auto FirstRealElt = find_if(M, [](int Elt) { return Elt >= 0; });
- if (FirstRealElt == M.end())
- return std::nullopt;
-
- // Use APInt to handle overflow when calculating expected element.
- unsigned MaskBits = APInt(32, NumElts * 2).logBase2();
- APInt ExpectedElt = APInt(MaskBits, *FirstRealElt + 1, false, true);
-
- // The following shuffle indices must be the successive elements after the
- // first real element.
- if (any_of(
- make_range(std::next(FirstRealElt), M.end()),
- [&ExpectedElt](int Elt) { return Elt != ExpectedElt++ && Elt >= 0; }))
- return std::nullopt;
-
- // The index of an EXT is the first element if it is not UNDEF.
- // Watch out for the beginning UNDEFs. The EXT index should be the expected
- // value of the first element. E.g.
- // <-1, -1, 3, ...> is treated as <1, 2, 3, ...>.
- // <-1, -1, 0, 1, ...> is treated as <2*NumElts-2, 2*NumElts-1, 0, 1, ...>.
- // ExpectedElt is the last mask index plus 1.
- uint64_t Imm = ExpectedElt.getZExtValue();
- bool ReverseExt = false;
-
- // There are two difference cases requiring to reverse input vectors.
- // For example, for vector <4 x i32> we have the following cases,
- // Case 1: shufflevector(<4 x i32>,<4 x i32>,<-1, -1, -1, 0>)
- // Case 2: shufflevector(<4 x i32>,<4 x i32>,<-1, -1, 7, 0>)
- // For both cases, we finally use mask <5, 6, 7, 0>, which requires
- // to reverse two input vectors.
- if (Imm < NumElts)
- ReverseExt = true;
- else
- Imm -= NumElts;
- return std::make_pair(ReverseExt, Imm);
-}
-
/// Helper function for matchINS.
///
/// \returns a value when \p M is an ins mask for \p NumInputElements.
@@ -326,35 +284,6 @@ bool matchDup(MachineInstr &MI, MachineRegisterInfo &MRI,
return false;
}
-// Check if an EXT instruction can handle the shuffle mask when the vector
-// sources of the shuffle are the same.
-bool isSingletonExtMask(ArrayRef<int> M, LLT Ty) {
- unsigned NumElts = Ty.getNumElements();
-
- // Assume that the first shuffle index is not UNDEF. Fail if it is.
- if (M[0] < 0)
- return false;
-
- // If this is a VEXT shuffle, the immediate value is the index of the first
- // element. The other shuffle indices must be the successive elements after
- // the first one.
- unsigned ExpectedElt = M[0];
- for (unsigned I = 1; I < NumElts; ++I) {
- // Increment the expected index. If it wraps around, just follow it
- // back to index zero and keep going.
- ++ExpectedElt;
- if (ExpectedElt == NumElts)
- ExpectedElt = 0;
-
- if (M[I] < 0)
- continue; // Ignore UNDEF indices.
- if (ExpectedElt != static_cast<unsigned>(M[I]))
- return false;
- }
-
- return true;
-}
-
bool matchEXT(MachineInstr &MI, MachineRegisterInfo &MRI,
ShuffleVectorPseudo &MatchInfo) {
assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
@@ -363,25 +292,26 @@ bool matchEXT(MachineInstr &MI, MachineRegisterInfo &MRI,
Register V1 = MI.getOperand(1).getReg();
Register V2 = MI.getOperand(2).getReg();
auto Mask = MI.getOperand(3).getShuffleMask();
- uint64_t Imm;
- auto ExtInfo = getExtMask(Mask, DstTy.getNumElements());
uint64_t ExtFactor = MRI.getType(V1).getScalarSizeInBits() / 8;
- if (!ExtInfo) {
- if (!getOpcodeDef<GImplicitDef>(V2, MRI) ||
- !isSingletonExtMask(Mask, DstTy))
- return false;
-
- Imm = Mask[0] * ExtFactor;
- MatchInfo = ShuffleVectorPseudo(AArch64::G_EXT, Dst, {V1, V1, Imm});
+ unsigned Imm;
+ bool ReverseExt;
+ if (isEXTMask(Mask, DstTy.getNumElements(), ReverseExt, Imm)) {
+ if (ReverseExt)
+ std::swap(V1, V2);
+ Imm *= ExtFactor;
+ MatchInfo = ShuffleVectorPseudo(AArch64::G_EXT, Dst,
+ {V1, V2, static_cast<uint64_t>(Imm)});
return true;
}
- bool ReverseExt;
- std::tie(ReverseExt, Imm) = *ExtInfo;
- if (ReverseExt)
- std::swap(V1, V2);
- Imm *= ExtFactor;
- MatchInfo = ShuffleVectorPseudo(AArch64::G_EXT, Dst, {V1, V2, Imm});
+
+ if (!getOpcodeDef<GImplicitDef>(V2, MRI) ||
+ !isSingletonEXTMask(Mask, DstTy.getNumElements(), Imm))
+ return false;
+
+ Imm = Mask[0] * ExtFactor;
+ MatchInfo = ShuffleVectorPseudo(AArch64::G_EXT, Dst,
+ {V1, V1, static_cast<uint64_t>(Imm)});
return true;
}
>From 69de2f70a657f050112d41c029cb81fae500e517 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Thu, 24 Sep 2026 12:12:05 +0100
Subject: [PATCH 2/2] [AArch64] Add ext mask shuffle costs
These were not being handled by the existing shuffle mask costs, leading to
higher fallback costs being used.
---
.../AArch64/AArch64TargetTransformInfo.cpp | 3 ++
.../CostModel/AArch64/shuffle-extract.ll | 20 ++++----
.../CostModel/AArch64/shuffle-other.ll | 10 ++--
.../AArch64/sve-vls-shuffle-extract.ll | 48 +++++++++---------
.../alternate-cmp-swapped-pred-parent.ll | 49 ++++++++++++-------
5 files changed, 73 insertions(+), 57 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 581f37d4c17bc..396ffcde738ca 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -7167,6 +7167,7 @@ InstructionCost AArch64TTIImpl::getShuffleCost(
// Check for other shuffles that are not SK_ kinds but we have native
// instructions for, for example ZIP and UZP.
unsigned Unused;
+ bool UnusedRev;
if (LT.second.isFixedLengthVector() &&
LT.second.getVectorNumElements() == Mask.size() &&
(Kind == TTI::SK_PermuteTwoSrc || Kind == TTI::SK_PermuteSingleSrc ||
@@ -7183,6 +7184,8 @@ InstructionCost AArch64TTIImpl::getShuffleCost(
LT.second.getVectorNumElements(), 32) ||
isREVMask(Mask, LT.second.getScalarSizeInBits(),
LT.second.getVectorNumElements(), 64) ||
+ isEXTMask(Mask, LT.second.getVectorNumElements(), UnusedRev, Unused) ||
+ isSingletonEXTMask(Mask, LT.second.getVectorNumElements(), Unused) ||
// Check for non-zero lane splats
all_of(drop_begin(Mask),
[&Mask](int M) { return M < 0 || M == Mask[0]; })))
diff --git a/llvm/test/Analysis/CostModel/AArch64/shuffle-extract.ll b/llvm/test/Analysis/CostModel/AArch64/shuffle-extract.ll
index 07764fbf4acf3..09276eade5023 100644
--- a/llvm/test/Analysis/CostModel/AArch64/shuffle-extract.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/shuffle-extract.ll
@@ -28,24 +28,24 @@ define void @extract_half() {
; CHECK-NEXT: Cost Model: Found costs of 8 for: %v8i16_mi = shufflevector <8 x i16> poison, <8 x i16> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i16_hi = shufflevector <8 x i16> poison, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: Cost Model: Found costs of 0 for: %v16i16_lo = shufflevector <16 x i16> poison, <16 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: Cost Model: Found costs of 16 for: %v16i16_mi = shufflevector <16 x i16> poison, <16 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: Cost Model: Found costs of 16 for: %v16i16_hi = shufflevector <16 x i16> poison, <16 x i16> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %v16i16_mi = shufflevector <16 x i16> poison, <16 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %v16i16_hi = shufflevector <16 x i16> poison, <16 x i16> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
; CHECK-NEXT: Cost Model: Found costs of 0 for: %v2i32_lo = shufflevector <2 x i32> poison, <2 x i32> poison, <1 x i32> zeroinitializer
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v2i32_hi = shufflevector <2 x i32> poison, <2 x i32> poison, <1 x i32> <i32 1>
; CHECK-NEXT: Cost Model: Found costs of 0 for: %v4i32_lo = shufflevector <4 x i32> poison, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
; CHECK-NEXT: Cost Model: Found costs of 3 for: %v4i32_mi = shufflevector <4 x i32> poison, <4 x i32> poison, <2 x i32> <i32 1, i32 2>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i32_hi = shufflevector <4 x i32> poison, <4 x i32> poison, <2 x i32> <i32 2, i32 3>
; CHECK-NEXT: Cost Model: Found costs of 0 for: %v8i32_lo = shufflevector <8 x i32> poison, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: Cost Model: Found costs of 6 for: %v8i32_mi = shufflevector <8 x i32> poison, <8 x i32> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-; CHECK-NEXT: Cost Model: Found costs of 6 for: %v8i32_hi = shufflevector <8 x i32> poison, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i32_mi = shufflevector <8 x i32> poison, <8 x i32> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i32_hi = shufflevector <8 x i32> poison, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: Cost Model: Found costs of 0 for: %v16i32_lo = shufflevector <16 x i32> poison, <16 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: Cost Model: Found costs of 12 for: %v16i32_mi = shufflevector <16 x i32> poison, <16 x i32> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
; CHECK-NEXT: Cost Model: Found costs of 12 for: %v16i32_hi = shufflevector <16 x i32> poison, <16 x i32> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
; CHECK-NEXT: Cost Model: Found costs of 0 for: %v2i64_lo = shufflevector <2 x i64> poison, <2 x i64> poison, <1 x i32> zeroinitializer
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v2i64_hi = shufflevector <2 x i64> poison, <2 x i64> poison, <1 x i32> <i32 1>
; CHECK-NEXT: Cost Model: Found costs of 0 for: %v4i64_lo = shufflevector <4 x i64> poison, <4 x i64> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4i64_mi = shufflevector <4 x i64> poison, <4 x i64> poison, <2 x i32> <i32 1, i32 2>
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4i64_hi = shufflevector <4 x i64> poison, <4 x i64> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i64_mi = shufflevector <4 x i64> poison, <4 x i64> poison, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i64_hi = shufflevector <4 x i64> poison, <4 x i64> poison, <2 x i32> <i32 2, i32 3>
; CHECK-NEXT: Cost Model: Found costs of 0 for: %v8i64_lo = shufflevector <8 x i64> poison, <8 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: Cost Model: Found costs of 4 for: %v8i64_mi = shufflevector <8 x i64> poison, <8 x i64> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
; CHECK-NEXT: Cost Model: Found costs of 4 for: %v8i64_hi = shufflevector <8 x i64> poison, <8 x i64> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -126,14 +126,14 @@ define void @extract_qtr() {
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i32_mi = shufflevector <8 x i32> poison, <8 x i32> poison, <2 x i32> <i32 2, i32 3>
; CHECK-NEXT: Cost Model: Found costs of 6 for: %v8i32_hi = shufflevector <8 x i32> poison, <8 x i32> poison, <2 x i32> <i32 4, i32 5>
; CHECK-NEXT: Cost Model: Found costs of 0 for: %v16i32_lo = shufflevector <16 x i32> poison, <16 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: Cost Model: Found costs of 12 for: %v16i32_mi = shufflevector <16 x i32> poison, <16 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: Cost Model: Found costs of 12 for: %v16i32_hi = shufflevector <16 x i32> poison, <16 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %v16i32_mi = shufflevector <16 x i32> poison, <16 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %v16i32_hi = shufflevector <16 x i32> poison, <16 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
; CHECK-NEXT: Cost Model: Found costs of 0 for: %v4i64_lo = shufflevector <4 x i64> poison, <4 x i64> poison, <1 x i32> zeroinitializer
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i64_mi = shufflevector <4 x i64> poison, <4 x i64> poison, <1 x i32> <i32 1>
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4i64_hi = shufflevector <4 x i64> poison, <4 x i64> poison, <1 x i32> <i32 2>
; CHECK-NEXT: Cost Model: Found costs of 0 for: %v8i64_lo = shufflevector <8 x i64> poison, <8 x i64> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: Cost Model: Found costs of 4 for: %v8i64_mi = shufflevector <8 x i64> poison, <8 x i64> poison, <2 x i32> <i32 2, i32 3>
-; CHECK-NEXT: Cost Model: Found costs of 4 for: %v8i64_hi = shufflevector <8 x i64> poison, <8 x i64> poison, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i64_mi = shufflevector <8 x i64> poison, <8 x i64> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i64_hi = shufflevector <8 x i64> poison, <8 x i64> poison, <2 x i32> <i32 4, i32 5>
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v4i8_lo = shufflevector <4 x i8> poison, <4 x i8> poison, <1 x i32> <i32 0>
diff --git a/llvm/test/Analysis/CostModel/AArch64/shuffle-other.ll b/llvm/test/Analysis/CostModel/AArch64/shuffle-other.ll
index defe22b17a92f..cde6a70931df3 100644
--- a/llvm/test/Analysis/CostModel/AArch64/shuffle-other.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/shuffle-other.ll
@@ -131,17 +131,17 @@ define void @splice() {
define void @splice1() {
; CHECK-LABEL: 'splice1'
-; CHECK-NEXT: Cost Model: Found costs of 3 for: %v4i8 = shufflevector <4 x i8> poison, <4 x i8> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
-; CHECK-NEXT: Cost Model: Found costs of 8 for: %v8i8 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0>
-; CHECK-NEXT: Cost Model: Found costs of 8 for: %v16i8 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0>
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i8 = shufflevector <4 x i8> poison, <4 x i8> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i8 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0>
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %v16i8 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i16 = shufflevector <4 x i16> poison, <4 x i16> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
-; CHECK-NEXT: Cost Model: Found costs of 8 for: %v8i16 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0>
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8i16 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0>
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v16i16 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i32 = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v8i32 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0>
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4i64 = shufflevector <4 x i64> poison, <4 x i64> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4f16 = shufflevector <4 x half> poison, <4 x half> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
-; CHECK-NEXT: Cost Model: Found costs of 8 for: %v8f16 = shufflevector <8 x half> poison, <8 x half> poison, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0>
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %v8f16 = shufflevector <8 x half> poison, <8 x half> poison, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0>
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v16f16 = shufflevector <16 x half> poison, <16 x half> poison, <16 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4f32 = shufflevector <4 x float> poison, <4 x float> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
; CHECK-NEXT: Cost Model: Found costs of 2 for: %v8f32 = shufflevector <8 x float> poison, <8 x float> poison, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0>
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-vls-shuffle-extract.ll b/llvm/test/Analysis/CostModel/AArch64/sve-vls-shuffle-extract.ll
index 8845454b8f015..de46dc4f09f19 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-vls-shuffle-extract.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-vls-shuffle-extract.ll
@@ -60,32 +60,32 @@ define void @extract_half_lo() {
define void @extract_half_hi() {
; VSCALE-1-LABEL: 'extract_half_hi'
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i8_hi = shufflevector <16 x i8> poison, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v32i8_hi = shufflevector <32 x i8> poison, <32 x i8> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32i8_hi = shufflevector <32 x i8> poison, <32 x i8> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v64i8_hi = shufflevector <64 x i8> poison, <64 x i8> poison, <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i16_hi = shufflevector <8 x i16> poison, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i16_hi = shufflevector <16 x i16> poison, <16 x i16> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i16_hi = shufflevector <16 x i16> poison, <16 x i16> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v32i16_hi = shufflevector <32 x i16> poison, <32 x i16> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i32_hi = shufflevector <4 x i32> poison, <4 x i32> poison, <2 x i32> <i32 2, i32 3>
-; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_hi = shufflevector <8 x i32> poison, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i32_hi = shufflevector <8 x i32> poison, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_hi = shufflevector <16 x i32> poison, <16 x i32> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_hi = shufflevector <2 x i64> poison, <2 x i64> poison, <1 x i32> <i32 1>
-; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_hi = shufflevector <4 x i64> poison, <4 x i64> poison, <2 x i32> <i32 2, i32 3>
+; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i64_hi = shufflevector <4 x i64> poison, <4 x i64> poison, <2 x i32> <i32 2, i32 3>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_hi = shufflevector <8 x i64> poison, <8 x i64> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
; VSCALE-2-LABEL: 'extract_half_hi'
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i8_hi = shufflevector <16 x i8> poison, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32i8_hi = shufflevector <32 x i8> poison, <32 x i8> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 192 for instruction: %v64i8_hi = shufflevector <64 x i8> poison, <64 x i8> poison, <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v64i8_hi = shufflevector <64 x i8> poison, <64 x i8> poison, <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i16_hi = shufflevector <8 x i16> poison, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i16_hi = shufflevector <16 x i16> poison, <16 x i16> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 96 for instruction: %v32i16_hi = shufflevector <32 x i16> poison, <32 x i16> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32i16_hi = shufflevector <32 x i16> poison, <32 x i16> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i32_hi = shufflevector <4 x i32> poison, <4 x i32> poison, <2 x i32> <i32 2, i32 3>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i32_hi = shufflevector <8 x i32> poison, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %v16i32_hi = shufflevector <16 x i32> poison, <16 x i32> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i32_hi = shufflevector <16 x i32> poison, <16 x i32> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_hi = shufflevector <2 x i64> poison, <2 x i64> poison, <1 x i32> <i32 1>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i64_hi = shufflevector <4 x i64> poison, <4 x i64> poison, <2 x i32> <i32 2, i32 3>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %v8i64_hi = shufflevector <8 x i64> poison, <8 x i64> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i64_hi = shufflevector <8 x i64> poison, <8 x i64> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
; VSCALE-4-LABEL: 'extract_half_hi'
@@ -125,30 +125,30 @@ define void @extract_half_hi() {
define void @extract_half_unaligned() {
; VSCALE-1-LABEL: 'extract_half_unaligned'
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i8_mi = shufflevector <16 x i8> poison, <16 x i8> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v32i8_mi = shufflevector <32 x i8> poison, <32 x i8> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32i8_mi = shufflevector <32 x i8> poison, <32 x i8> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v64i8_mi = shufflevector <64 x i8> poison, <64 x i8> poison, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i16_mi = shufflevector <8 x i16> poison, <8 x i16> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i16_mi = shufflevector <16 x i16> poison, <16 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i16_mi = shufflevector <16 x i16> poison, <16 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v32i16_mi = shufflevector <32 x i16> poison, <32 x i16> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_mi = shufflevector <4 x i32> poison, <4 x i32> poison, <2 x i32> <i32 1, i32 2>
-; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_mi = shufflevector <8 x i32> poison, <8 x i32> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i32_mi = shufflevector <8 x i32> poison, <8 x i32> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_mi = shufflevector <16 x i32> poison, <16 x i32> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_mi = shufflevector <4 x i64> poison, <4 x i64> poison, <2 x i32> <i32 1, i32 2>
+; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i64_mi = shufflevector <4 x i64> poison, <4 x i64> poison, <2 x i32> <i32 1, i32 2>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_mi = shufflevector <8 x i64> poison, <8 x i64> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
; VSCALE-2-LABEL: 'extract_half_unaligned'
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i8_mi = shufflevector <16 x i8> poison, <16 x i8> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 72 for instruction: %v32i8_mi = shufflevector <32 x i8> poison, <32 x i8> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 192 for instruction: %v64i8_mi = shufflevector <64 x i8> poison, <64 x i8> poison, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v64i8_mi = shufflevector <64 x i8> poison, <64 x i8> poison, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i16_mi = shufflevector <8 x i16> poison, <8 x i16> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %v16i16_mi = shufflevector <16 x i16> poison, <16 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 96 for instruction: %v32i16_mi = shufflevector <32 x i16> poison, <32 x i16> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32i16_mi = shufflevector <32 x i16> poison, <32 x i16> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_mi = shufflevector <4 x i32> poison, <4 x i32> poison, <2 x i32> <i32 1, i32 2>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v8i32_mi = shufflevector <8 x i32> poison, <8 x i32> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %v16i32_mi = shufflevector <16 x i32> poison, <16 x i32> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i32_mi = shufflevector <16 x i32> poison, <16 x i32> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v4i64_mi = shufflevector <4 x i64> poison, <4 x i64> poison, <2 x i32> <i32 1, i32 2>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %v8i64_mi = shufflevector <8 x i64> poison, <8 x i64> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i64_mi = shufflevector <8 x i64> poison, <8 x i64> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
; VSCALE-4-LABEL: 'extract_half_unaligned'
@@ -220,15 +220,15 @@ define void @extract_qtr_hi() {
; VSCALE-1-LABEL: 'extract_qtr_hi'
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i8_hi = shufflevector <16 x i8> poison, <16 x i8> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v32i8_hi = shufflevector <32 x i8> poison, <32 x i8> poison, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v64i8_hi = shufflevector <64 x i8> poison, <64 x i8> poison, <16 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v64i8_hi = shufflevector <64 x i8> poison, <64 x i8> poison, <16 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i16_hi = shufflevector <8 x i16> poison, <8 x i16> poison, <2 x i32> <i32 4, i32 5>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i16_hi = shufflevector <16 x i16> poison, <16 x i16> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v32i16_hi = shufflevector <32 x i16> poison, <32 x i16> poison, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32i16_hi = shufflevector <32 x i16> poison, <32 x i16> poison, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_hi = shufflevector <4 x i32> poison, <4 x i32> poison, <1 x i32> <i32 2>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_hi = shufflevector <8 x i32> poison, <8 x i32> poison, <2 x i32> <i32 4, i32 5>
-; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_hi = shufflevector <16 x i32> poison, <16 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i32_hi = shufflevector <16 x i32> poison, <16 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_hi = shufflevector <4 x i64> poison, <4 x i64> poison, <1 x i32> <i32 2>
-; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_hi = shufflevector <8 x i64> poison, <8 x i64> poison, <2 x i32> <i32 4, i32 5>
+; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i64_hi = shufflevector <8 x i64> poison, <8 x i64> poison, <2 x i32> <i32 4, i32 5>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
; VSCALE-2-LABEL: 'extract_qtr_hi'
@@ -281,15 +281,15 @@ define void @extract_qtr_unaligned() {
; VSCALE-1-LABEL: 'extract_qtr_unaligned'
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i8_mi = shufflevector <16 x i8> poison, <16 x i8> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v32i8_mi = shufflevector <32 x i8> poison, <32 x i8> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v64i8_mi = shufflevector <64 x i8> poison, <64 x i8> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v64i8_mi = shufflevector <64 x i8> poison, <64 x i8> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i16_mi = shufflevector <8 x i16> poison, <8 x i16> poison, <2 x i32> <i32 2, i32 3>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i16_mi = shufflevector <16 x i16> poison, <16 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v32i16_mi = shufflevector <32 x i16> poison, <32 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32i16_mi = shufflevector <32 x i16> poison, <32 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_mi = shufflevector <4 x i32> poison, <4 x i32> poison, <1 x i32> <i32 1>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i32_mi = shufflevector <8 x i32> poison, <8 x i32> poison, <2 x i32> <i32 2, i32 3>
-; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_mi = shufflevector <16 x i32> poison, <16 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i32_mi = shufflevector <16 x i32> poison, <16 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i64_mi = shufflevector <4 x i64> poison, <4 x i64> poison, <1 x i32> <i32 1>
-; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_mi = shufflevector <8 x i64> poison, <8 x i64> poison, <2 x i32> <i32 2, i32 3>
+; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i64_mi = shufflevector <8 x i64> poison, <8 x i64> poison, <2 x i32> <i32 2, i32 3>
; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
; VSCALE-2-LABEL: 'extract_qtr_unaligned'
diff --git a/llvm/test/Transforms/SLPVectorizer/alternate-cmp-swapped-pred-parent.ll b/llvm/test/Transforms/SLPVectorizer/alternate-cmp-swapped-pred-parent.ll
index dd84918956e11..a167ebf658479 100644
--- a/llvm/test/Transforms/SLPVectorizer/alternate-cmp-swapped-pred-parent.ll
+++ b/llvm/test/Transforms/SLPVectorizer/alternate-cmp-swapped-pred-parent.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: %if aarch64-registered-target %{ opt < %s -mtriple=aarch64-unknown-linux-gnu -passes=slp-vectorizer -S -slp-threshold=-1000 | FileCheck %s %}
-; RUN: %if x86-registered-target %{ opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S -slp-threshold=-1000 | FileCheck %s %}
+; RUN: %if aarch64-registered-target %{ opt < %s -mtriple=aarch64-unknown-linux-gnu -passes=slp-vectorizer -S -slp-threshold=-1000 | FileCheck %s --check-prefixes=CHECK,AARCH64 %}
+; RUN: %if x86-registered-target %{ opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S -slp-threshold=-1000 | FileCheck %s --check-prefixes=CHECK,X86 %}
define void @test() {
; CHECK-LABEL: @test(
@@ -12,9 +12,9 @@ define void @test() {
; CHECK-NEXT: [[TMP0:%.*]] = insertelement <8 x i16> <i16 0, i16 0, i16 0, i16 poison, i16 poison, i16 poison, i16 0, i16 poison>, i16 [[CALL37]], i64 3
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <8 x i16> [[TMP0]], i16 [[CALL]], i64 5
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 3, i32 5, i32 6, i32 3>
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 poison, i16 poison>, i16 [[CALL37]], i64 6
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i16> [[TMP5]], i16 [[CALL]], i64 7
-; CHECK-NEXT: [[TMP4:%.*]] = icmp slt <8 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 poison, i16 poison>, i16 [[CALL37]], i64 6
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <8 x i16> [[TMP3]], i16 [[CALL]], i64 7
+; CHECK-NEXT: [[TMP5:%.*]] = icmp slt <8 x i16> [[TMP2]], [[TMP4]]
; CHECK-NEXT: ret void
;
entry:
@@ -35,19 +35,32 @@ bb:
}
define void @test1() {
-; CHECK-LABEL: @test1(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CALL:%.*]] = load i16, ptr poison, align 2
-; CHECK-NEXT: br label [[BB:%.*]]
-; CHECK: bb:
-; CHECK-NEXT: [[CALL37:%.*]] = load i16, ptr poison, align 2
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <8 x i16> <i16 0, i16 0, i16 0, i16 poison, i16 poison, i16 poison, i16 poison, i16 0>, i16 [[CALL]], i64 3
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <8 x i16> [[TMP0]], i16 [[CALL37]], i64 4
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 7>
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 poison, i16 poison>, i16 [[CALL]], i64 6
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i16> [[TMP5]], i16 [[CALL37]], i64 7
-; CHECK-NEXT: [[TMP4:%.*]] = icmp slt <8 x i16> [[TMP2]], [[TMP3]]
-; CHECK-NEXT: ret void
+; AARCH64-LABEL: @test1(
+; AARCH64-NEXT: entry:
+; AARCH64-NEXT: [[CALL:%.*]] = load i16, ptr poison, align 2
+; AARCH64-NEXT: br label [[BB:%.*]]
+; AARCH64: bb:
+; AARCH64-NEXT: [[CALL37:%.*]] = load i16, ptr poison, align 2
+; AARCH64-NEXT: [[TMP0:%.*]] = insertelement <8 x i16> <i16 0, i16 0, i16 0, i16 poison, i16 poison, i16 poison, i16 poison, i16 0>, i16 [[CALL]], i64 3
+; AARCH64-NEXT: [[TMP1:%.*]] = insertelement <8 x i16> [[TMP0]], i16 [[CALL37]], i64 4
+; AARCH64-NEXT: [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 7>
+; AARCH64-NEXT: [[TMP3:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 poison, i16 poison>, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 3, i32 4>
+; AARCH64-NEXT: [[TMP4:%.*]] = icmp slt <8 x i16> [[TMP2]], [[TMP3]]
+; AARCH64-NEXT: ret void
+;
+; X86-LABEL: @test1(
+; X86-NEXT: entry:
+; X86-NEXT: [[CALL:%.*]] = load i16, ptr poison, align 2
+; X86-NEXT: br label [[BB:%.*]]
+; X86: bb:
+; X86-NEXT: [[CALL37:%.*]] = load i16, ptr poison, align 2
+; X86-NEXT: [[TMP0:%.*]] = insertelement <8 x i16> <i16 0, i16 0, i16 0, i16 poison, i16 poison, i16 poison, i16 poison, i16 0>, i16 [[CALL]], i64 3
+; X86-NEXT: [[TMP1:%.*]] = insertelement <8 x i16> [[TMP0]], i16 [[CALL37]], i64 4
+; X86-NEXT: [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 7>
+; X86-NEXT: [[TMP3:%.*]] = insertelement <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 poison, i16 poison>, i16 [[CALL]], i64 6
+; X86-NEXT: [[TMP4:%.*]] = insertelement <8 x i16> [[TMP3]], i16 [[CALL37]], i64 7
+; X86-NEXT: [[TMP5:%.*]] = icmp slt <8 x i16> [[TMP2]], [[TMP4]]
+; X86-NEXT: ret void
;
entry:
%call = load i16, ptr poison, align 2
More information about the llvm-commits
mailing list