[llvm] [AArch64][SME] Enable dynamic shuffle for fixed length types. (PR #72490)
Dinar Temirbulatov via llvm-commits
llvm-commits at lists.llvm.org
Thu Nov 16 00:19:37 PST 2023
https://github.com/dtemirbulatov created https://github.com/llvm/llvm-project/pull/72490
When SVE register size is unknown or the minimal size is not equal to the maximum size then we could determine the actual SVE register size in the runtime and adjust shuffle mask in the runtime.
>From 290675ce5ff4994fdf3a80449c2057d614d7742a Mon Sep 17 00:00:00 2001
From: Dinar Temirbulatov <Dinar.Temirbulatov at arm.com>
Date: Thu, 16 Nov 2023 08:15:44 +0000
Subject: [PATCH] [AArch64][SME] Enable dynamic shuffle for fixed length types.
When SVE register size is unknown or the minimal size is not equal to
the maximum size then we could determine the actual SVE register size in
the runtime and adjust shuffle mask in the runtime.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 73 +++-
.../sve-fixed-length-vector-shuffle-tbl.ll | 413 +++++++++++++++++-
2 files changed, 448 insertions(+), 38 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 9ff6d6f0f565edb..2423ef6f8962a53 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -26123,7 +26123,7 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
// Ignore two operands if no SVE2 or all index numbers couldn't
// be represented.
- if (!IsSingleOp && (!Subtarget.hasSVE2() || MinSVESize != MaxSVESize))
+ if (!IsSingleOp && !Subtarget.hasSVE2())
return SDValue();
EVT VTOp1 = Op.getOperand(0).getValueType();
@@ -26131,18 +26131,40 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
unsigned IndexLen = MinSVESize / BitsPerElt;
unsigned ElementsPerVectorReg = VTOp1.getVectorNumElements();
uint64_t MaxOffset = APInt(BitsPerElt, -1, false).getZExtValue();
+ EVT MaskEltType = EVT::getIntegerVT(*DAG.getContext(), BitsPerElt);
+ EVT MaskType = EVT::getVectorVT(*DAG.getContext(), MaskEltType, IndexLen);
+ bool MinMaxEqual = (MinSVESize == MaxSVESize);
assert(ElementsPerVectorReg <= IndexLen && ShuffleMask.size() <= IndexLen &&
"Incorrectly legalised shuffle operation");
SmallVector<SDValue, 8> TBLMask;
+ // If MinSVESize is not equal to MaxSVESize then we need to know which
+ // TBL mask element needs adjustment.
+ SmallVector<SDValue, 8> MaskNormalized;
+
+ if (BitsPerElt == 8 && !MinMaxEqual && !IsSingleOp)
+ return SDValue();
+
for (int Index : ShuffleMask) {
// Handling poison index value.
if (Index < 0)
Index = 0;
// If we refer to the second operand then we have to add elements
- // number in hardware register minus number of elements in a type.
- if ((unsigned)Index >= ElementsPerVectorReg)
- Index += IndexLen - ElementsPerVectorReg;
+ // number in hardware register minus number of elements in a type in
+ // case if MinSVESize equals to MaxSVESize, otherwise just add normalized
+ // value and record this element in MaskNormalized to be adjusted in the
+ // runtime.
+ if ((unsigned)Index >= ElementsPerVectorReg) {
+ if (!MinMaxEqual) {
+ Index = Index - ElementsPerVectorReg;
+ MaskNormalized.push_back(DAG.getConstant(1, DL, MVT::i64));
+ } else {
+ Index += IndexLen - ElementsPerVectorReg;
+ }
+ } else {
+ if (!MinMaxEqual)
+ MaskNormalized.push_back(DAG.getConstant(0, DL, MVT::i64));
+ }
// For 8-bit elements and 1024-bit SVE registers and MaxOffset equals
// to 255, this might point to the last element of in the second operand
// of the shufflevector, thus we are rejecting this transform.
@@ -26155,11 +26177,12 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
// value where it would perform first lane duplication for out of
// index elements. For i8 elements an out-of-range index could be a valid
// for 2048-bit vector register size.
- for (unsigned i = 0; i < IndexLen - ElementsPerVectorReg; ++i)
+ for (unsigned i = 0; i < IndexLen - ElementsPerVectorReg; ++i) {
TBLMask.push_back(DAG.getConstant((int)MaxOffset, DL, MVT::i64));
+ if (!MinMaxEqual)
+ MaskNormalized.push_back(DAG.getConstant(0, DL, MVT::i64));
+ }
- EVT MaskEltType = EVT::getIntegerVT(*DAG.getContext(), BitsPerElt);
- EVT MaskType = EVT::getVectorVT(*DAG.getContext(), MaskEltType, IndexLen);
EVT MaskContainerVT = getContainerForFixedLengthVector(DAG, MaskType);
SDValue VecMask =
DAG.getBuildVector(MaskType, DL, ArrayRef(TBLMask.data(), IndexLen));
@@ -26171,13 +26194,35 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
DAG.getConstant(Intrinsic::aarch64_sve_tbl, DL, MVT::i32),
Op1, SVEMask);
- else if (Subtarget.hasSVE2())
- Shuffle =
- DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
- DAG.getConstant(Intrinsic::aarch64_sve_tbl2, DL, MVT::i32),
- Op1, Op2, SVEMask);
- else
- llvm_unreachable("Cannot lower shuffle without SVE2 TBL");
+ else if (Subtarget.hasSVE2()) {
+ if (!MinMaxEqual) {
+ SDValue VScale = DAG.getVScale(DL, MVT::i32, APInt(32, 1));
+ SDValue Mul =
+ DAG.getNode(ISD::MUL, DL, MVT::i32,
+ DAG.getConstant(128 / BitsPerElt, DL, MVT::i32), VScale);
+ SDValue VecMask =
+ DAG.getBuildVector(MaskType, DL, ArrayRef(TBLMask.data(), IndexLen));
+ SDValue MulMask = DAG.getBuildVector(
+ MaskType, DL, ArrayRef(MaskNormalized.data(), IndexLen));
+ SDValue SplatPred = DAG.getNode(ISD::SPLAT_VECTOR, DL, MaskType, Mul);
+ SDValue MulMaskNormalized =
+ DAG.getNode(ISD::MUL, DL, MaskType, SplatPred, MulMask);
+ SDValue UpdatedVecMask =
+ DAG.getNode(ISD::ADD, DL, MaskType, VecMask, MulMaskNormalized);
+ EVT MaskContainerVT = getContainerForFixedLengthVector(DAG, MaskType);
+ SDValue SVEMask =
+ convertToScalableVector(DAG, MaskContainerVT, UpdatedVecMask);
+ Shuffle = DAG.getNode(
+ ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
+ DAG.getConstant(Intrinsic::aarch64_sve_tbl2, DL, MVT::i32), Op1, Op2,
+ SVEMask);
+ } else {
+ Shuffle = DAG.getNode(
+ ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
+ DAG.getConstant(Intrinsic::aarch64_sve_tbl2, DL, MVT::i32), Op1, Op2,
+ SVEMask);
+ }
+ }
Shuffle = convertFromScalableVector(DAG, VT, Shuffle);
return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Shuffle);
}
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
index f646319ba5fccb3..3beccdf278ebd10 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mattr=+sve2 -force-streaming-compatible-sve -aarch64-sve-vector-bits-min=128 -aarch64-sve-vector-bits-max=128 < %s | FileCheck %s -check-prefixes=CHECK,SVE2_128
; RUN: llc -mattr=+sve2 -force-streaming-compatible-sve -aarch64-sve-vector-bits-min=128 < %s | FileCheck %s -check-prefixes=CHECK,SVE2_128_NOMAX
+; RUN: llc -mattr=+sve2 -force-streaming-compatible-sve < %s | FileCheck %s -check-prefixes=CHECK,SVE2_NOMIN_NOMAX
+; RUN: llc -mattr=+sve2 -force-streaming-compatible-sve -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,SVE2_MIN_256_NOMAX
target triple = "aarch64-unknown-linux-gnu"
@@ -16,14 +18,43 @@ target triple = "aarch64-unknown-linux-gnu"
; SVE2_128-NEXT: .byte 255 // 0xff
; SVE2_128-NEXT: .byte 255 // 0xff
define <8 x i8> @shuffle_index_indices_from_op1(ptr %a, ptr %b) {
-; CHECK-LABEL: shuffle_index_indices_from_op1:
-; CHECK: // %bb.0:
-; CHECK-NEXT: adrp x8, .LCPI0_0
-; CHECK-NEXT: ldr d0, [x0]
-; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI0_0]
-; CHECK-NEXT: tbl z0.b, { z0.b }, z1.b
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
-; CHECK-NEXT: ret
+; SVE2_128-LABEL: shuffle_index_indices_from_op1:
+; SVE2_128: // %bb.0:
+; SVE2_128-NEXT: adrp x8, .LCPI0_0
+; SVE2_128-NEXT: ldr d0, [x0]
+; SVE2_128-NEXT: ldr q1, [x8, :lo12:.LCPI0_0]
+; SVE2_128-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_128-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_128-NEXT: ret
+;
+; SVE2_128_NOMAX-LABEL: shuffle_index_indices_from_op1:
+; SVE2_128_NOMAX: // %bb.0:
+; SVE2_128_NOMAX-NEXT: adrp x8, .LCPI0_0
+; SVE2_128_NOMAX-NEXT: ldr d0, [x0]
+; SVE2_128_NOMAX-NEXT: ldr q1, [x8, :lo12:.LCPI0_0]
+; SVE2_128_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_128_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_128_NOMAX-NEXT: ret
+;
+; SVE2_NOMIN_NOMAX-LABEL: shuffle_index_indices_from_op1:
+; SVE2_NOMIN_NOMAX: // %bb.0:
+; SVE2_NOMIN_NOMAX-NEXT: adrp x8, .LCPI0_0
+; SVE2_NOMIN_NOMAX-NEXT: ldr d0, [x0]
+; SVE2_NOMIN_NOMAX-NEXT: ldr q1, [x8, :lo12:.LCPI0_0]
+; SVE2_NOMIN_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_NOMIN_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_NOMIN_NOMAX-NEXT: ret
+;
+; SVE2_MIN_256_NOMAX-LABEL: shuffle_index_indices_from_op1:
+; SVE2_MIN_256_NOMAX: // %bb.0:
+; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.b, vl32
+; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI0_0
+; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI0_0
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x0]
+; SVE2_MIN_256_NOMAX-NEXT: ld1b { z1.b }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_MIN_256_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i8>, ptr %a
%op2 = load <8 x i8>, ptr %b
%1 = shufflevector <8 x i8> %op1, <8 x i8> %op2, <8 x i32> <i32 0, i32 7, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -42,14 +73,43 @@ define <8 x i8> @shuffle_index_indices_from_op1(ptr %a, ptr %b) {
; SVE2_128-NEXT: .byte 255 // 0xff
; SVE2_128-NEXT: .byte 255 // 0xff
define <8 x i8> @shuffle_index_indices_from_op2(ptr %a, ptr %b) {
-; CHECK-LABEL: shuffle_index_indices_from_op2:
-; CHECK: // %bb.0:
-; CHECK-NEXT: adrp x8, .LCPI1_0
-; CHECK-NEXT: ldr d0, [x1]
-; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI1_0]
-; CHECK-NEXT: tbl z0.b, { z0.b }, z1.b
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
-; CHECK-NEXT: ret
+; SVE2_128-LABEL: shuffle_index_indices_from_op2:
+; SVE2_128: // %bb.0:
+; SVE2_128-NEXT: adrp x8, .LCPI1_0
+; SVE2_128-NEXT: ldr d0, [x1]
+; SVE2_128-NEXT: ldr q1, [x8, :lo12:.LCPI1_0]
+; SVE2_128-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_128-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_128-NEXT: ret
+;
+; SVE2_128_NOMAX-LABEL: shuffle_index_indices_from_op2:
+; SVE2_128_NOMAX: // %bb.0:
+; SVE2_128_NOMAX-NEXT: adrp x8, .LCPI1_0
+; SVE2_128_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_128_NOMAX-NEXT: ldr q1, [x8, :lo12:.LCPI1_0]
+; SVE2_128_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_128_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_128_NOMAX-NEXT: ret
+;
+; SVE2_NOMIN_NOMAX-LABEL: shuffle_index_indices_from_op2:
+; SVE2_NOMIN_NOMAX: // %bb.0:
+; SVE2_NOMIN_NOMAX-NEXT: adrp x8, .LCPI1_0
+; SVE2_NOMIN_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_NOMIN_NOMAX-NEXT: ldr q1, [x8, :lo12:.LCPI1_0]
+; SVE2_NOMIN_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_NOMIN_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_NOMIN_NOMAX-NEXT: ret
+;
+; SVE2_MIN_256_NOMAX-LABEL: shuffle_index_indices_from_op2:
+; SVE2_MIN_256_NOMAX: // %bb.0:
+; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.b, vl32
+; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI1_0
+; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI1_0
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: ld1b { z1.b }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_MIN_256_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i8>, ptr %a
%op2 = load <8 x i8>, ptr %b
%1 = shufflevector <8 x i8> %op1, <8 x i8> %op2, <8 x i32> <i32 8, i32 9, i32 9, i32 11, i32 12, i32 15, i32 14, i32 15>
@@ -109,6 +169,70 @@ define <8 x i8> @shuffle_index_indices_from_both_ops(ptr %a, ptr %b) {
; SVE2_128_NOMAX-NEXT: ldr d0, [sp, #8]
; SVE2_128_NOMAX-NEXT: add sp, sp, #16
; SVE2_128_NOMAX-NEXT: ret
+;
+; SVE2_NOMIN_NOMAX-LABEL: shuffle_index_indices_from_both_ops:
+; SVE2_NOMIN_NOMAX: // %bb.0:
+; SVE2_NOMIN_NOMAX-NEXT: sub sp, sp, #16
+; SVE2_NOMIN_NOMAX-NEXT: .cfi_def_cfa_offset 16
+; SVE2_NOMIN_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_NOMIN_NOMAX-NEXT: mov z1.b, z0.b[7]
+; SVE2_NOMIN_NOMAX-NEXT: mov z2.b, z0.b[6]
+; SVE2_NOMIN_NOMAX-NEXT: mov z3.b, z0.b[4]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w8, s1
+; SVE2_NOMIN_NOMAX-NEXT: ldr d1, [x0]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w9, s2
+; SVE2_NOMIN_NOMAX-NEXT: mov z2.b, z0.b[3]
+; SVE2_NOMIN_NOMAX-NEXT: mov z1.b, z1.b[1]
+; SVE2_NOMIN_NOMAX-NEXT: strb w8, [sp, #15]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w8, s3
+; SVE2_NOMIN_NOMAX-NEXT: mov z3.b, z0.b[2]
+; SVE2_NOMIN_NOMAX-NEXT: strb w9, [sp, #14]
+; SVE2_NOMIN_NOMAX-NEXT: mov z0.b, z0.b[1]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w9, s2
+; SVE2_NOMIN_NOMAX-NEXT: strb w8, [sp, #13]
+; SVE2_NOMIN_NOMAX-NEXT: strb w8, [sp, #12]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w8, s3
+; SVE2_NOMIN_NOMAX-NEXT: strb w9, [sp, #11]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w9, s0
+; SVE2_NOMIN_NOMAX-NEXT: strb w8, [sp, #10]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w8, s1
+; SVE2_NOMIN_NOMAX-NEXT: strb w9, [sp, #9]
+; SVE2_NOMIN_NOMAX-NEXT: strb w8, [sp, #8]
+; SVE2_NOMIN_NOMAX-NEXT: ldr d0, [sp, #8]
+; SVE2_NOMIN_NOMAX-NEXT: add sp, sp, #16
+; SVE2_NOMIN_NOMAX-NEXT: ret
+;
+; SVE2_MIN_256_NOMAX-LABEL: shuffle_index_indices_from_both_ops:
+; SVE2_MIN_256_NOMAX: // %bb.0:
+; SVE2_MIN_256_NOMAX-NEXT: sub sp, sp, #16
+; SVE2_MIN_256_NOMAX-NEXT: .cfi_def_cfa_offset 16
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: mov z1.b, z0.b[7]
+; SVE2_MIN_256_NOMAX-NEXT: mov z2.b, z0.b[6]
+; SVE2_MIN_256_NOMAX-NEXT: mov z3.b, z0.b[4]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w8, s1
+; SVE2_MIN_256_NOMAX-NEXT: ldr d1, [x0]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w9, s2
+; SVE2_MIN_256_NOMAX-NEXT: mov z2.b, z0.b[3]
+; SVE2_MIN_256_NOMAX-NEXT: mov z1.b, z1.b[1]
+; SVE2_MIN_256_NOMAX-NEXT: strb w8, [sp, #15]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w8, s3
+; SVE2_MIN_256_NOMAX-NEXT: mov z3.b, z0.b[2]
+; SVE2_MIN_256_NOMAX-NEXT: strb w9, [sp, #14]
+; SVE2_MIN_256_NOMAX-NEXT: mov z0.b, z0.b[1]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w9, s2
+; SVE2_MIN_256_NOMAX-NEXT: strb w8, [sp, #13]
+; SVE2_MIN_256_NOMAX-NEXT: strb w8, [sp, #12]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w8, s3
+; SVE2_MIN_256_NOMAX-NEXT: strb w9, [sp, #11]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w9, s0
+; SVE2_MIN_256_NOMAX-NEXT: strb w8, [sp, #10]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w8, s1
+; SVE2_MIN_256_NOMAX-NEXT: strb w9, [sp, #9]
+; SVE2_MIN_256_NOMAX-NEXT: strb w8, [sp, #8]
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [sp, #8]
+; SVE2_MIN_256_NOMAX-NEXT: add sp, sp, #16
+; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i8>, ptr %a
%op2 = load <8 x i8>, ptr %b
%1 = shufflevector <8 x i8> %op1, <8 x i8> %op2, <8 x i32> <i32 1, i32 9, i32 10, i32 11, i32 12, i32 12, i32 14, i32 15>
@@ -165,6 +289,64 @@ define <8 x i8> @shuffle_index_poison_value(ptr %a, ptr %b) {
; SVE2_128_NOMAX-NEXT: ldr d0, [sp, #8]
; SVE2_128_NOMAX-NEXT: add sp, sp, #16
; SVE2_128_NOMAX-NEXT: ret
+;
+; SVE2_NOMIN_NOMAX-LABEL: shuffle_index_poison_value:
+; SVE2_NOMIN_NOMAX: // %bb.0:
+; SVE2_NOMIN_NOMAX-NEXT: sub sp, sp, #16
+; SVE2_NOMIN_NOMAX-NEXT: .cfi_def_cfa_offset 16
+; SVE2_NOMIN_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_NOMIN_NOMAX-NEXT: ldr d3, [x0]
+; SVE2_NOMIN_NOMAX-NEXT: mov z1.b, z0.b[6]
+; SVE2_NOMIN_NOMAX-NEXT: mov z2.b, z0.b[4]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w8, s1
+; SVE2_NOMIN_NOMAX-NEXT: mov z1.b, z0.b[3]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w9, s2
+; SVE2_NOMIN_NOMAX-NEXT: mov z2.b, z0.b[2]
+; SVE2_NOMIN_NOMAX-NEXT: mov z0.b, z0.b[1]
+; SVE2_NOMIN_NOMAX-NEXT: strb w8, [sp, #14]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w8, s1
+; SVE2_NOMIN_NOMAX-NEXT: mov z1.b, z3.b[1]
+; SVE2_NOMIN_NOMAX-NEXT: strb w9, [sp, #13]
+; SVE2_NOMIN_NOMAX-NEXT: strb w9, [sp, #12]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w9, s2
+; SVE2_NOMIN_NOMAX-NEXT: strb w8, [sp, #11]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w8, s0
+; SVE2_NOMIN_NOMAX-NEXT: strb w9, [sp, #10]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w9, s1
+; SVE2_NOMIN_NOMAX-NEXT: strb w8, [sp, #9]
+; SVE2_NOMIN_NOMAX-NEXT: strb w9, [sp, #8]
+; SVE2_NOMIN_NOMAX-NEXT: ldr d0, [sp, #8]
+; SVE2_NOMIN_NOMAX-NEXT: add sp, sp, #16
+; SVE2_NOMIN_NOMAX-NEXT: ret
+;
+; SVE2_MIN_256_NOMAX-LABEL: shuffle_index_poison_value:
+; SVE2_MIN_256_NOMAX: // %bb.0:
+; SVE2_MIN_256_NOMAX-NEXT: sub sp, sp, #16
+; SVE2_MIN_256_NOMAX-NEXT: .cfi_def_cfa_offset 16
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: ldr d3, [x0]
+; SVE2_MIN_256_NOMAX-NEXT: mov z1.b, z0.b[6]
+; SVE2_MIN_256_NOMAX-NEXT: mov z2.b, z0.b[4]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w8, s1
+; SVE2_MIN_256_NOMAX-NEXT: mov z1.b, z0.b[3]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w9, s2
+; SVE2_MIN_256_NOMAX-NEXT: mov z2.b, z0.b[2]
+; SVE2_MIN_256_NOMAX-NEXT: mov z0.b, z0.b[1]
+; SVE2_MIN_256_NOMAX-NEXT: strb w8, [sp, #14]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w8, s1
+; SVE2_MIN_256_NOMAX-NEXT: mov z1.b, z3.b[1]
+; SVE2_MIN_256_NOMAX-NEXT: strb w9, [sp, #13]
+; SVE2_MIN_256_NOMAX-NEXT: strb w9, [sp, #12]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w9, s2
+; SVE2_MIN_256_NOMAX-NEXT: strb w8, [sp, #11]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w8, s0
+; SVE2_MIN_256_NOMAX-NEXT: strb w9, [sp, #10]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w9, s1
+; SVE2_MIN_256_NOMAX-NEXT: strb w8, [sp, #9]
+; SVE2_MIN_256_NOMAX-NEXT: strb w9, [sp, #8]
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [sp, #8]
+; SVE2_MIN_256_NOMAX-NEXT: add sp, sp, #16
+; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i8>, ptr %a
%op2 = load <8 x i8>, ptr %b
%1 = shufflevector <8 x i8> %op1, <8 x i8> %op2, <8 x i32> <i32 1, i32 9, i32 10, i32 11, i32 12, i32 12, i32 14, i32 poison>
@@ -172,14 +354,43 @@ define <8 x i8> @shuffle_index_poison_value(ptr %a, ptr %b) {
}
define <8 x i8> @shuffle_op1_poison(ptr %a, ptr %b) {
-; CHECK-LABEL: shuffle_op1_poison:
-; CHECK: // %bb.0:
-; CHECK-NEXT: adrp x8, .LCPI4_0
-; CHECK-NEXT: ldr d0, [x1]
-; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI4_0]
-; CHECK-NEXT: tbl z0.b, { z0.b }, z1.b
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
-; CHECK-NEXT: ret
+; SVE2_128-LABEL: shuffle_op1_poison:
+; SVE2_128: // %bb.0:
+; SVE2_128-NEXT: adrp x8, .LCPI4_0
+; SVE2_128-NEXT: ldr d0, [x1]
+; SVE2_128-NEXT: ldr q1, [x8, :lo12:.LCPI4_0]
+; SVE2_128-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_128-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_128-NEXT: ret
+;
+; SVE2_128_NOMAX-LABEL: shuffle_op1_poison:
+; SVE2_128_NOMAX: // %bb.0:
+; SVE2_128_NOMAX-NEXT: adrp x8, .LCPI4_0
+; SVE2_128_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_128_NOMAX-NEXT: ldr q1, [x8, :lo12:.LCPI4_0]
+; SVE2_128_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_128_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_128_NOMAX-NEXT: ret
+;
+; SVE2_NOMIN_NOMAX-LABEL: shuffle_op1_poison:
+; SVE2_NOMIN_NOMAX: // %bb.0:
+; SVE2_NOMIN_NOMAX-NEXT: adrp x8, .LCPI4_0
+; SVE2_NOMIN_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_NOMIN_NOMAX-NEXT: ldr q1, [x8, :lo12:.LCPI4_0]
+; SVE2_NOMIN_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_NOMIN_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_NOMIN_NOMAX-NEXT: ret
+;
+; SVE2_MIN_256_NOMAX-LABEL: shuffle_op1_poison:
+; SVE2_MIN_256_NOMAX: // %bb.0:
+; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.b, vl32
+; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI4_0
+; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI4_0
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: ld1b { z1.b }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_MIN_256_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_MIN_256_NOMAX-NEXT: ret
%op2 = load <8 x i8>, ptr %b
%1 = shufflevector <8 x i8> poison, <8 x i8> %op2, <8 x i32> <i32 1, i32 9, i32 10, i32 11, i32 12, i32 12, i32 14, i32 15>
ret <8 x i8> %1
@@ -252,3 +463,157 @@ define <8 x i8> @shuffle_index_size_op1_maxhw(ptr %a, ptr %b) "target-features"=
%1 = shufflevector <8 x i8> %op1, <8 x i8> %op2, <8 x i32> <i32 0, i32 7, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
ret <8 x i8> %1
}
+
+; SVE2_128: .LCPI7_0:
+; SVE2_128-NEXT: .hword 1 // 0x1
+; SVE2_128-NEXT: .hword 9 // 0x9
+; SVE2_128-NEXT: .hword 10 // 0xa
+; SVE2_128-NEXT: .hword 11 // 0xb
+; SVE2_128-NEXT: .hword 12 // 0xc
+; SVE2_128-NEXT: .hword 12 // 0xc
+; SVE2_128-NEXT: .hword 14 // 0xe
+; SVE2_128-NEXT: .hword 15 // 0xf
+
+; SVE2_128_NOMAX: .LCPI7_0:
+; SVE2_128_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT:.LCPI7_1:
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT: .hword 2 // 0x2
+; SVE2_128_NOMAX-NEXT: .hword 3 // 0x3
+; SVE2_128_NOMAX-NEXT: .hword 4 // 0x4
+; SVE2_128_NOMAX-NEXT: .hword 4 // 0x4
+; SVE2_128_NOMAX-NEXT: .hword 6 // 0x6
+; SVE2_128_NOMAX-NEXT: .hword 7 // 0x7
+
+; SVE2_NOMIN_NOMAX: .LCPI7_0:
+; SVE2_NOMIN_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT:.LCPI7_1:
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT: .hword 2 // 0x2
+; SVE2_NOMIN_NOMAX-NEXT: .hword 3 // 0x3
+; SVE2_NOMIN_NOMAX-NEXT: .hword 4 // 0x4
+; SVE2_NOMIN_NOMAX-NEXT: .hword 4 // 0x4
+; SVE2_NOMIN_NOMAX-NEXT: .hword 6 // 0x6
+; SVE2_NOMIN_NOMAX-NEXT: .hword 7 // 0x7
+
+; SVE2_MIN_256_NOMAX: .LCPI7_0:
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT:.LCPI7_1:
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 2 // 0x2
+; SVE2_MIN_256_NOMAX-NEXT: .hword 3 // 0x3
+; SVE2_MIN_256_NOMAX-NEXT: .hword 4 // 0x4
+; SVE2_MIN_256_NOMAX-NEXT: .hword 4 // 0x4
+; SVE2_MIN_256_NOMAX-NEXT: .hword 6 // 0x6
+; SVE2_MIN_256_NOMAX-NEXT: .hword 7 // 0x7
+; SVE2_MIN_256_NOMAX-NEXT: .hword 65535 // 0xffff
+; SVE2_MIN_256_NOMAX-NEXT: .hword 65535 // 0xffff
+; SVE2_MIN_256_NOMAX-NEXT: .hword 65535 // 0xffff
+; SVE2_MIN_256_NOMAX-NEXT: .hword 65535 // 0xffff
+; SVE2_MIN_256_NOMAX-NEXT: .hword 65535 // 0xffff
+; SVE2_MIN_256_NOMAX-NEXT: .hword 65535 // 0xffff
+; SVE2_MIN_256_NOMAX-NEXT: .hword 65535 // 0xffff
+; SVE2_MIN_256_NOMAX-NEXT: .hword 65535 // 0xffff
+define <8 x i16> @shuffle_index_indices_from_both_ops_i16(ptr %a, ptr %b) {
+; SVE2_128-LABEL: shuffle_index_indices_from_both_ops_i16:
+; SVE2_128: // %bb.0:
+; SVE2_128-NEXT: adrp x8, .LCPI7_0
+; SVE2_128-NEXT: ldr q0, [x0]
+; SVE2_128-NEXT: ldr q1, [x1]
+; SVE2_128-NEXT: ldr q2, [x8, :lo12:.LCPI7_0]
+; SVE2_128-NEXT: tbl z0.h, { z0.h, z1.h }, z2.h
+; SVE2_128-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2_128-NEXT: ret
+;
+; SVE2_128_NOMAX-LABEL: shuffle_index_indices_from_both_ops_i16:
+; SVE2_128_NOMAX: // %bb.0:
+; SVE2_128_NOMAX-NEXT: rdvl x8, #1
+; SVE2_128_NOMAX-NEXT: ptrue p0.h, vl8
+; SVE2_128_NOMAX-NEXT: adrp x9, .LCPI7_0
+; SVE2_128_NOMAX-NEXT: lsr x8, x8, #4
+; SVE2_128_NOMAX-NEXT: ldr q1, [x9, :lo12:.LCPI7_0]
+; SVE2_128_NOMAX-NEXT: lsl w8, w8, #3
+; SVE2_128_NOMAX-NEXT: mov z0.h, w8
+; SVE2_128_NOMAX-NEXT: adrp x8, .LCPI7_1
+; SVE2_128_NOMAX-NEXT: ldr q2, [x8, :lo12:.LCPI7_1]
+; SVE2_128_NOMAX-NEXT: mad z0.h, p0/m, z1.h, z2.h
+; SVE2_128_NOMAX-NEXT: ldr q1, [x0]
+; SVE2_128_NOMAX-NEXT: ldr q2, [x1]
+; SVE2_128_NOMAX-NEXT: tbl z0.h, { z1.h, z2.h }, z0.h
+; SVE2_128_NOMAX-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2_128_NOMAX-NEXT: ret
+;
+; SVE2_NOMIN_NOMAX-LABEL: shuffle_index_indices_from_both_ops_i16:
+; SVE2_NOMIN_NOMAX: // %bb.0:
+; SVE2_NOMIN_NOMAX-NEXT: rdvl x8, #1
+; SVE2_NOMIN_NOMAX-NEXT: ptrue p0.h, vl8
+; SVE2_NOMIN_NOMAX-NEXT: adrp x9, .LCPI7_0
+; SVE2_NOMIN_NOMAX-NEXT: lsr x8, x8, #4
+; SVE2_NOMIN_NOMAX-NEXT: ldr q1, [x9, :lo12:.LCPI7_0]
+; SVE2_NOMIN_NOMAX-NEXT: lsl w8, w8, #3
+; SVE2_NOMIN_NOMAX-NEXT: mov z0.h, w8
+; SVE2_NOMIN_NOMAX-NEXT: adrp x8, .LCPI7_1
+; SVE2_NOMIN_NOMAX-NEXT: ldr q2, [x8, :lo12:.LCPI7_1]
+; SVE2_NOMIN_NOMAX-NEXT: mad z0.h, p0/m, z1.h, z2.h
+; SVE2_NOMIN_NOMAX-NEXT: ldr q1, [x0]
+; SVE2_NOMIN_NOMAX-NEXT: ldr q2, [x1]
+; SVE2_NOMIN_NOMAX-NEXT: tbl z0.h, { z1.h, z2.h }, z0.h
+; SVE2_NOMIN_NOMAX-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2_NOMIN_NOMAX-NEXT: ret
+;
+; SVE2_MIN_256_NOMAX-LABEL: shuffle_index_indices_from_both_ops_i16:
+; SVE2_MIN_256_NOMAX: // %bb.0:
+; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.h, vl16
+; SVE2_MIN_256_NOMAX-NEXT: rdvl x8, #1
+; SVE2_MIN_256_NOMAX-NEXT: ldr q0, [x0]
+; SVE2_MIN_256_NOMAX-NEXT: lsr x8, x8, #4
+; SVE2_MIN_256_NOMAX-NEXT: adrp x9, .LCPI7_0
+; SVE2_MIN_256_NOMAX-NEXT: add x9, x9, :lo12:.LCPI7_0
+; SVE2_MIN_256_NOMAX-NEXT: adrp x10, .LCPI7_1
+; SVE2_MIN_256_NOMAX-NEXT: add x10, x10, :lo12:.LCPI7_1
+; SVE2_MIN_256_NOMAX-NEXT: ldr q1, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: lsl w8, w8, #3
+; SVE2_MIN_256_NOMAX-NEXT: mov z4.h, w8
+; SVE2_MIN_256_NOMAX-NEXT: ld1h { z2.h }, p0/z, [x9]
+; SVE2_MIN_256_NOMAX-NEXT: ld1h { z3.h }, p0/z, [x10]
+; SVE2_MIN_256_NOMAX-NEXT: mad z2.h, p0/m, z4.h, z3.h
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.h, { z0.h, z1.h }, z2.h
+; SVE2_MIN_256_NOMAX-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2_MIN_256_NOMAX-NEXT: ret
+ %op1 = load <8 x i16>, ptr %a
+ %op2 = load <8 x i16>, ptr %b
+ %1 = shufflevector <8 x i16> %op1, <8 x i16> %op2, <8 x i32> <i32 1, i32 9, i32 10, i32 11, i32 12, i32 12, i32 14, i32 15>
+ ret <8 x i16> %1
+}
More information about the llvm-commits
mailing list