[llvm] [AArch64][SVE2] Enable dynamic shuffle for fixed length types. (PR #72490)
Dinar Temirbulatov via llvm-commits
llvm-commits at lists.llvm.org
Wed Feb 21 05:35:15 PST 2024
https://github.com/dtemirbulatov updated https://github.com/llvm/llvm-project/pull/72490
>From 7f8abed26306962f7edab1ef3957404c02d6ebc1 Mon Sep 17 00:00:00 2001
From: Dinar Temirbulatov <Dinar.Temirbulatov at arm.com>
Date: Thu, 16 Nov 2023 08:15:44 +0000
Subject: [PATCH 1/7] [AArch64][SME] Enable dynamic shuffle for fixed length
types.
When SVE register size is unknown or the minimal size is not equal to
the maximum size then we could determine the actual SVE register size in
the runtime and adjust shuffle mask in the runtime.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 73 +++-
.../sve-fixed-length-vector-shuffle-tbl.ll | 413 +++++++++++++++++-
2 files changed, 448 insertions(+), 38 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 8c5a4cdae11634..1ba7666fada73f 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -26798,7 +26798,7 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
// Ignore two operands if no SVE2 or all index numbers couldn't
// be represented.
- if (!IsSingleOp && (!Subtarget.hasSVE2() || MinSVESize != MaxSVESize))
+ if (!IsSingleOp && !Subtarget.hasSVE2())
return SDValue();
EVT VTOp1 = Op.getOperand(0).getValueType();
@@ -26806,18 +26806,40 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
unsigned IndexLen = MinSVESize / BitsPerElt;
unsigned ElementsPerVectorReg = VTOp1.getVectorNumElements();
uint64_t MaxOffset = APInt(BitsPerElt, -1, false).getZExtValue();
+ EVT MaskEltType = EVT::getIntegerVT(*DAG.getContext(), BitsPerElt);
+ EVT MaskType = EVT::getVectorVT(*DAG.getContext(), MaskEltType, IndexLen);
+ bool MinMaxEqual = (MinSVESize == MaxSVESize);
assert(ElementsPerVectorReg <= IndexLen && ShuffleMask.size() <= IndexLen &&
"Incorrectly legalised shuffle operation");
SmallVector<SDValue, 8> TBLMask;
+ // If MinSVESize is not equal to MaxSVESize then we need to know which
+ // TBL mask element needs adjustment.
+ SmallVector<SDValue, 8> MaskNormalized;
+
+ if (BitsPerElt == 8 && !MinMaxEqual && !IsSingleOp)
+ return SDValue();
+
for (int Index : ShuffleMask) {
// Handling poison index value.
if (Index < 0)
Index = 0;
// If we refer to the second operand then we have to add elements
- // number in hardware register minus number of elements in a type.
- if ((unsigned)Index >= ElementsPerVectorReg)
- Index += IndexLen - ElementsPerVectorReg;
+ // number in hardware register minus number of elements in a type in
+ // case if MinSVESize equals to MaxSVESize, otherwise just add normalized
+ // value and record this element in MaskNormalized to be adjusted in the
+ // runtime.
+ if ((unsigned)Index >= ElementsPerVectorReg) {
+ if (!MinMaxEqual) {
+ Index = Index - ElementsPerVectorReg;
+ MaskNormalized.push_back(DAG.getConstant(1, DL, MVT::i64));
+ } else {
+ Index += IndexLen - ElementsPerVectorReg;
+ }
+ } else {
+ if (!MinMaxEqual)
+ MaskNormalized.push_back(DAG.getConstant(0, DL, MVT::i64));
+ }
// For 8-bit elements and 1024-bit SVE registers and MaxOffset equals
// to 255, this might point to the last element of in the second operand
// of the shufflevector, thus we are rejecting this transform.
@@ -26830,11 +26852,12 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
// value where it would perform first lane duplication for out of
// index elements. For i8 elements an out-of-range index could be a valid
// for 2048-bit vector register size.
- for (unsigned i = 0; i < IndexLen - ElementsPerVectorReg; ++i)
+ for (unsigned i = 0; i < IndexLen - ElementsPerVectorReg; ++i) {
TBLMask.push_back(DAG.getConstant((int)MaxOffset, DL, MVT::i64));
+ if (!MinMaxEqual)
+ MaskNormalized.push_back(DAG.getConstant(0, DL, MVT::i64));
+ }
- EVT MaskEltType = EVT::getIntegerVT(*DAG.getContext(), BitsPerElt);
- EVT MaskType = EVT::getVectorVT(*DAG.getContext(), MaskEltType, IndexLen);
EVT MaskContainerVT = getContainerForFixedLengthVector(DAG, MaskType);
SDValue VecMask =
DAG.getBuildVector(MaskType, DL, ArrayRef(TBLMask.data(), IndexLen));
@@ -26846,13 +26869,35 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
DAG.getConstant(Intrinsic::aarch64_sve_tbl, DL, MVT::i32),
Op1, SVEMask);
- else if (Subtarget.hasSVE2())
- Shuffle =
- DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
- DAG.getConstant(Intrinsic::aarch64_sve_tbl2, DL, MVT::i32),
- Op1, Op2, SVEMask);
- else
- llvm_unreachable("Cannot lower shuffle without SVE2 TBL");
+ else if (Subtarget.hasSVE2()) {
+ if (!MinMaxEqual) {
+ SDValue VScale = DAG.getVScale(DL, MVT::i32, APInt(32, 1));
+ SDValue Mul =
+ DAG.getNode(ISD::MUL, DL, MVT::i32,
+ DAG.getConstant(128 / BitsPerElt, DL, MVT::i32), VScale);
+ SDValue VecMask =
+ DAG.getBuildVector(MaskType, DL, ArrayRef(TBLMask.data(), IndexLen));
+ SDValue MulMask = DAG.getBuildVector(
+ MaskType, DL, ArrayRef(MaskNormalized.data(), IndexLen));
+ SDValue SplatPred = DAG.getNode(ISD::SPLAT_VECTOR, DL, MaskType, Mul);
+ SDValue MulMaskNormalized =
+ DAG.getNode(ISD::MUL, DL, MaskType, SplatPred, MulMask);
+ SDValue UpdatedVecMask =
+ DAG.getNode(ISD::ADD, DL, MaskType, VecMask, MulMaskNormalized);
+ EVT MaskContainerVT = getContainerForFixedLengthVector(DAG, MaskType);
+ SDValue SVEMask =
+ convertToScalableVector(DAG, MaskContainerVT, UpdatedVecMask);
+ Shuffle = DAG.getNode(
+ ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
+ DAG.getConstant(Intrinsic::aarch64_sve_tbl2, DL, MVT::i32), Op1, Op2,
+ SVEMask);
+ } else {
+ Shuffle = DAG.getNode(
+ ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
+ DAG.getConstant(Intrinsic::aarch64_sve_tbl2, DL, MVT::i32), Op1, Op2,
+ SVEMask);
+ }
+ }
Shuffle = convertFromScalableVector(DAG, VT, Shuffle);
return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Shuffle);
}
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
index bae3c6582c6b0c..331f9609586b1a 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mattr=+sve2 -force-streaming-compatible-sve -aarch64-sve-vector-bits-min=128 -aarch64-sve-vector-bits-max=128 < %s | FileCheck %s -check-prefixes=CHECK,SVE2_128
; RUN: llc -mattr=+sve2 -force-streaming-compatible-sve -aarch64-sve-vector-bits-min=128 < %s | FileCheck %s -check-prefixes=CHECK,SVE2_128_NOMAX
+; RUN: llc -mattr=+sve2 -force-streaming-compatible-sve < %s | FileCheck %s -check-prefixes=CHECK,SVE2_NOMIN_NOMAX
+; RUN: llc -mattr=+sve2 -force-streaming-compatible-sve -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,SVE2_MIN_256_NOMAX
target triple = "aarch64-unknown-linux-gnu"
@@ -16,14 +18,43 @@ target triple = "aarch64-unknown-linux-gnu"
; SVE2_128-NEXT: .byte 255 // 0xff
; SVE2_128-NEXT: .byte 255 // 0xff
define <8 x i8> @shuffle_index_indices_from_op1(ptr %a, ptr %b) {
-; CHECK-LABEL: shuffle_index_indices_from_op1:
-; CHECK: // %bb.0:
-; CHECK-NEXT: adrp x8, .LCPI0_0
-; CHECK-NEXT: ldr d0, [x0]
-; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI0_0]
-; CHECK-NEXT: tbl z0.b, { z0.b }, z1.b
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
-; CHECK-NEXT: ret
+; SVE2_128-LABEL: shuffle_index_indices_from_op1:
+; SVE2_128: // %bb.0:
+; SVE2_128-NEXT: adrp x8, .LCPI0_0
+; SVE2_128-NEXT: ldr d0, [x0]
+; SVE2_128-NEXT: ldr q1, [x8, :lo12:.LCPI0_0]
+; SVE2_128-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_128-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_128-NEXT: ret
+;
+; SVE2_128_NOMAX-LABEL: shuffle_index_indices_from_op1:
+; SVE2_128_NOMAX: // %bb.0:
+; SVE2_128_NOMAX-NEXT: adrp x8, .LCPI0_0
+; SVE2_128_NOMAX-NEXT: ldr d0, [x0]
+; SVE2_128_NOMAX-NEXT: ldr q1, [x8, :lo12:.LCPI0_0]
+; SVE2_128_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_128_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_128_NOMAX-NEXT: ret
+;
+; SVE2_NOMIN_NOMAX-LABEL: shuffle_index_indices_from_op1:
+; SVE2_NOMIN_NOMAX: // %bb.0:
+; SVE2_NOMIN_NOMAX-NEXT: adrp x8, .LCPI0_0
+; SVE2_NOMIN_NOMAX-NEXT: ldr d0, [x0]
+; SVE2_NOMIN_NOMAX-NEXT: ldr q1, [x8, :lo12:.LCPI0_0]
+; SVE2_NOMIN_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_NOMIN_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_NOMIN_NOMAX-NEXT: ret
+;
+; SVE2_MIN_256_NOMAX-LABEL: shuffle_index_indices_from_op1:
+; SVE2_MIN_256_NOMAX: // %bb.0:
+; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.b, vl32
+; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI0_0
+; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI0_0
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x0]
+; SVE2_MIN_256_NOMAX-NEXT: ld1b { z1.b }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_MIN_256_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i8>, ptr %a
%op2 = load <8 x i8>, ptr %b
%1 = shufflevector <8 x i8> %op1, <8 x i8> %op2, <8 x i32> <i32 0, i32 7, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -42,14 +73,43 @@ define <8 x i8> @shuffle_index_indices_from_op1(ptr %a, ptr %b) {
; SVE2_128-NEXT: .byte 255 // 0xff
; SVE2_128-NEXT: .byte 255 // 0xff
define <8 x i8> @shuffle_index_indices_from_op2(ptr %a, ptr %b) {
-; CHECK-LABEL: shuffle_index_indices_from_op2:
-; CHECK: // %bb.0:
-; CHECK-NEXT: adrp x8, .LCPI1_0
-; CHECK-NEXT: ldr d0, [x1]
-; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI1_0]
-; CHECK-NEXT: tbl z0.b, { z0.b }, z1.b
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
-; CHECK-NEXT: ret
+; SVE2_128-LABEL: shuffle_index_indices_from_op2:
+; SVE2_128: // %bb.0:
+; SVE2_128-NEXT: adrp x8, .LCPI1_0
+; SVE2_128-NEXT: ldr d0, [x1]
+; SVE2_128-NEXT: ldr q1, [x8, :lo12:.LCPI1_0]
+; SVE2_128-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_128-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_128-NEXT: ret
+;
+; SVE2_128_NOMAX-LABEL: shuffle_index_indices_from_op2:
+; SVE2_128_NOMAX: // %bb.0:
+; SVE2_128_NOMAX-NEXT: adrp x8, .LCPI1_0
+; SVE2_128_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_128_NOMAX-NEXT: ldr q1, [x8, :lo12:.LCPI1_0]
+; SVE2_128_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_128_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_128_NOMAX-NEXT: ret
+;
+; SVE2_NOMIN_NOMAX-LABEL: shuffle_index_indices_from_op2:
+; SVE2_NOMIN_NOMAX: // %bb.0:
+; SVE2_NOMIN_NOMAX-NEXT: adrp x8, .LCPI1_0
+; SVE2_NOMIN_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_NOMIN_NOMAX-NEXT: ldr q1, [x8, :lo12:.LCPI1_0]
+; SVE2_NOMIN_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_NOMIN_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_NOMIN_NOMAX-NEXT: ret
+;
+; SVE2_MIN_256_NOMAX-LABEL: shuffle_index_indices_from_op2:
+; SVE2_MIN_256_NOMAX: // %bb.0:
+; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.b, vl32
+; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI1_0
+; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI1_0
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: ld1b { z1.b }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_MIN_256_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i8>, ptr %a
%op2 = load <8 x i8>, ptr %b
%1 = shufflevector <8 x i8> %op1, <8 x i8> %op2, <8 x i32> <i32 8, i32 9, i32 9, i32 11, i32 12, i32 15, i32 14, i32 15>
@@ -109,6 +169,70 @@ define <8 x i8> @shuffle_index_indices_from_both_ops(ptr %a, ptr %b) {
; SVE2_128_NOMAX-NEXT: ldr d0, [sp, #8]
; SVE2_128_NOMAX-NEXT: add sp, sp, #16
; SVE2_128_NOMAX-NEXT: ret
+;
+; SVE2_NOMIN_NOMAX-LABEL: shuffle_index_indices_from_both_ops:
+; SVE2_NOMIN_NOMAX: // %bb.0:
+; SVE2_NOMIN_NOMAX-NEXT: sub sp, sp, #16
+; SVE2_NOMIN_NOMAX-NEXT: .cfi_def_cfa_offset 16
+; SVE2_NOMIN_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_NOMIN_NOMAX-NEXT: mov z1.b, z0.b[7]
+; SVE2_NOMIN_NOMAX-NEXT: mov z2.b, z0.b[6]
+; SVE2_NOMIN_NOMAX-NEXT: mov z3.b, z0.b[4]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w8, s1
+; SVE2_NOMIN_NOMAX-NEXT: ldr d1, [x0]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w9, s2
+; SVE2_NOMIN_NOMAX-NEXT: mov z2.b, z0.b[3]
+; SVE2_NOMIN_NOMAX-NEXT: mov z1.b, z1.b[1]
+; SVE2_NOMIN_NOMAX-NEXT: strb w8, [sp, #15]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w8, s3
+; SVE2_NOMIN_NOMAX-NEXT: mov z3.b, z0.b[2]
+; SVE2_NOMIN_NOMAX-NEXT: strb w9, [sp, #14]
+; SVE2_NOMIN_NOMAX-NEXT: mov z0.b, z0.b[1]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w9, s2
+; SVE2_NOMIN_NOMAX-NEXT: strb w8, [sp, #13]
+; SVE2_NOMIN_NOMAX-NEXT: strb w8, [sp, #12]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w8, s3
+; SVE2_NOMIN_NOMAX-NEXT: strb w9, [sp, #11]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w9, s0
+; SVE2_NOMIN_NOMAX-NEXT: strb w8, [sp, #10]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w8, s1
+; SVE2_NOMIN_NOMAX-NEXT: strb w9, [sp, #9]
+; SVE2_NOMIN_NOMAX-NEXT: strb w8, [sp, #8]
+; SVE2_NOMIN_NOMAX-NEXT: ldr d0, [sp, #8]
+; SVE2_NOMIN_NOMAX-NEXT: add sp, sp, #16
+; SVE2_NOMIN_NOMAX-NEXT: ret
+;
+; SVE2_MIN_256_NOMAX-LABEL: shuffle_index_indices_from_both_ops:
+; SVE2_MIN_256_NOMAX: // %bb.0:
+; SVE2_MIN_256_NOMAX-NEXT: sub sp, sp, #16
+; SVE2_MIN_256_NOMAX-NEXT: .cfi_def_cfa_offset 16
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: mov z1.b, z0.b[7]
+; SVE2_MIN_256_NOMAX-NEXT: mov z2.b, z0.b[6]
+; SVE2_MIN_256_NOMAX-NEXT: mov z3.b, z0.b[4]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w8, s1
+; SVE2_MIN_256_NOMAX-NEXT: ldr d1, [x0]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w9, s2
+; SVE2_MIN_256_NOMAX-NEXT: mov z2.b, z0.b[3]
+; SVE2_MIN_256_NOMAX-NEXT: mov z1.b, z1.b[1]
+; SVE2_MIN_256_NOMAX-NEXT: strb w8, [sp, #15]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w8, s3
+; SVE2_MIN_256_NOMAX-NEXT: mov z3.b, z0.b[2]
+; SVE2_MIN_256_NOMAX-NEXT: strb w9, [sp, #14]
+; SVE2_MIN_256_NOMAX-NEXT: mov z0.b, z0.b[1]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w9, s2
+; SVE2_MIN_256_NOMAX-NEXT: strb w8, [sp, #13]
+; SVE2_MIN_256_NOMAX-NEXT: strb w8, [sp, #12]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w8, s3
+; SVE2_MIN_256_NOMAX-NEXT: strb w9, [sp, #11]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w9, s0
+; SVE2_MIN_256_NOMAX-NEXT: strb w8, [sp, #10]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w8, s1
+; SVE2_MIN_256_NOMAX-NEXT: strb w9, [sp, #9]
+; SVE2_MIN_256_NOMAX-NEXT: strb w8, [sp, #8]
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [sp, #8]
+; SVE2_MIN_256_NOMAX-NEXT: add sp, sp, #16
+; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i8>, ptr %a
%op2 = load <8 x i8>, ptr %b
%1 = shufflevector <8 x i8> %op1, <8 x i8> %op2, <8 x i32> <i32 1, i32 9, i32 10, i32 11, i32 12, i32 12, i32 14, i32 15>
@@ -165,6 +289,64 @@ define <8 x i8> @shuffle_index_poison_value(ptr %a, ptr %b) {
; SVE2_128_NOMAX-NEXT: ldr d0, [sp, #8]
; SVE2_128_NOMAX-NEXT: add sp, sp, #16
; SVE2_128_NOMAX-NEXT: ret
+;
+; SVE2_NOMIN_NOMAX-LABEL: shuffle_index_poison_value:
+; SVE2_NOMIN_NOMAX: // %bb.0:
+; SVE2_NOMIN_NOMAX-NEXT: sub sp, sp, #16
+; SVE2_NOMIN_NOMAX-NEXT: .cfi_def_cfa_offset 16
+; SVE2_NOMIN_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_NOMIN_NOMAX-NEXT: ldr d3, [x0]
+; SVE2_NOMIN_NOMAX-NEXT: mov z1.b, z0.b[6]
+; SVE2_NOMIN_NOMAX-NEXT: mov z2.b, z0.b[4]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w8, s1
+; SVE2_NOMIN_NOMAX-NEXT: mov z1.b, z0.b[3]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w9, s2
+; SVE2_NOMIN_NOMAX-NEXT: mov z2.b, z0.b[2]
+; SVE2_NOMIN_NOMAX-NEXT: mov z0.b, z0.b[1]
+; SVE2_NOMIN_NOMAX-NEXT: strb w8, [sp, #14]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w8, s1
+; SVE2_NOMIN_NOMAX-NEXT: mov z1.b, z3.b[1]
+; SVE2_NOMIN_NOMAX-NEXT: strb w9, [sp, #13]
+; SVE2_NOMIN_NOMAX-NEXT: strb w9, [sp, #12]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w9, s2
+; SVE2_NOMIN_NOMAX-NEXT: strb w8, [sp, #11]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w8, s0
+; SVE2_NOMIN_NOMAX-NEXT: strb w9, [sp, #10]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w9, s1
+; SVE2_NOMIN_NOMAX-NEXT: strb w8, [sp, #9]
+; SVE2_NOMIN_NOMAX-NEXT: strb w9, [sp, #8]
+; SVE2_NOMIN_NOMAX-NEXT: ldr d0, [sp, #8]
+; SVE2_NOMIN_NOMAX-NEXT: add sp, sp, #16
+; SVE2_NOMIN_NOMAX-NEXT: ret
+;
+; SVE2_MIN_256_NOMAX-LABEL: shuffle_index_poison_value:
+; SVE2_MIN_256_NOMAX: // %bb.0:
+; SVE2_MIN_256_NOMAX-NEXT: sub sp, sp, #16
+; SVE2_MIN_256_NOMAX-NEXT: .cfi_def_cfa_offset 16
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: ldr d3, [x0]
+; SVE2_MIN_256_NOMAX-NEXT: mov z1.b, z0.b[6]
+; SVE2_MIN_256_NOMAX-NEXT: mov z2.b, z0.b[4]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w8, s1
+; SVE2_MIN_256_NOMAX-NEXT: mov z1.b, z0.b[3]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w9, s2
+; SVE2_MIN_256_NOMAX-NEXT: mov z2.b, z0.b[2]
+; SVE2_MIN_256_NOMAX-NEXT: mov z0.b, z0.b[1]
+; SVE2_MIN_256_NOMAX-NEXT: strb w8, [sp, #14]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w8, s1
+; SVE2_MIN_256_NOMAX-NEXT: mov z1.b, z3.b[1]
+; SVE2_MIN_256_NOMAX-NEXT: strb w9, [sp, #13]
+; SVE2_MIN_256_NOMAX-NEXT: strb w9, [sp, #12]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w9, s2
+; SVE2_MIN_256_NOMAX-NEXT: strb w8, [sp, #11]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w8, s0
+; SVE2_MIN_256_NOMAX-NEXT: strb w9, [sp, #10]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w9, s1
+; SVE2_MIN_256_NOMAX-NEXT: strb w8, [sp, #9]
+; SVE2_MIN_256_NOMAX-NEXT: strb w9, [sp, #8]
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [sp, #8]
+; SVE2_MIN_256_NOMAX-NEXT: add sp, sp, #16
+; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i8>, ptr %a
%op2 = load <8 x i8>, ptr %b
%1 = shufflevector <8 x i8> %op1, <8 x i8> %op2, <8 x i32> <i32 1, i32 9, i32 10, i32 11, i32 12, i32 12, i32 14, i32 poison>
@@ -172,14 +354,43 @@ define <8 x i8> @shuffle_index_poison_value(ptr %a, ptr %b) {
}
define <8 x i8> @shuffle_op1_poison(ptr %a, ptr %b) {
-; CHECK-LABEL: shuffle_op1_poison:
-; CHECK: // %bb.0:
-; CHECK-NEXT: adrp x8, .LCPI4_0
-; CHECK-NEXT: ldr d0, [x1]
-; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI4_0]
-; CHECK-NEXT: tbl z0.b, { z0.b }, z1.b
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
-; CHECK-NEXT: ret
+; SVE2_128-LABEL: shuffle_op1_poison:
+; SVE2_128: // %bb.0:
+; SVE2_128-NEXT: adrp x8, .LCPI4_0
+; SVE2_128-NEXT: ldr d0, [x1]
+; SVE2_128-NEXT: ldr q1, [x8, :lo12:.LCPI4_0]
+; SVE2_128-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_128-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_128-NEXT: ret
+;
+; SVE2_128_NOMAX-LABEL: shuffle_op1_poison:
+; SVE2_128_NOMAX: // %bb.0:
+; SVE2_128_NOMAX-NEXT: adrp x8, .LCPI4_0
+; SVE2_128_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_128_NOMAX-NEXT: ldr q1, [x8, :lo12:.LCPI4_0]
+; SVE2_128_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_128_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_128_NOMAX-NEXT: ret
+;
+; SVE2_NOMIN_NOMAX-LABEL: shuffle_op1_poison:
+; SVE2_NOMIN_NOMAX: // %bb.0:
+; SVE2_NOMIN_NOMAX-NEXT: adrp x8, .LCPI4_0
+; SVE2_NOMIN_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_NOMIN_NOMAX-NEXT: ldr q1, [x8, :lo12:.LCPI4_0]
+; SVE2_NOMIN_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_NOMIN_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_NOMIN_NOMAX-NEXT: ret
+;
+; SVE2_MIN_256_NOMAX-LABEL: shuffle_op1_poison:
+; SVE2_MIN_256_NOMAX: // %bb.0:
+; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.b, vl32
+; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI4_0
+; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI4_0
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: ld1b { z1.b }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_MIN_256_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_MIN_256_NOMAX-NEXT: ret
%op2 = load <8 x i8>, ptr %b
%1 = shufflevector <8 x i8> poison, <8 x i8> %op2, <8 x i32> <i32 1, i32 9, i32 10, i32 11, i32 12, i32 12, i32 14, i32 15>
ret <8 x i8> %1
@@ -252,3 +463,157 @@ define <8 x i8> @shuffle_index_size_op1_maxhw(ptr %a, ptr %b) "target-features"=
%1 = shufflevector <8 x i8> %op1, <8 x i8> %op2, <8 x i32> <i32 0, i32 7, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
ret <8 x i8> %1
}
+
+; SVE2_128: .LCPI7_0:
+; SVE2_128-NEXT: .hword 1 // 0x1
+; SVE2_128-NEXT: .hword 9 // 0x9
+; SVE2_128-NEXT: .hword 10 // 0xa
+; SVE2_128-NEXT: .hword 11 // 0xb
+; SVE2_128-NEXT: .hword 12 // 0xc
+; SVE2_128-NEXT: .hword 12 // 0xc
+; SVE2_128-NEXT: .hword 14 // 0xe
+; SVE2_128-NEXT: .hword 15 // 0xf
+
+; SVE2_128_NOMAX: .LCPI7_0:
+; SVE2_128_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT:.LCPI7_1:
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT: .hword 2 // 0x2
+; SVE2_128_NOMAX-NEXT: .hword 3 // 0x3
+; SVE2_128_NOMAX-NEXT: .hword 4 // 0x4
+; SVE2_128_NOMAX-NEXT: .hword 4 // 0x4
+; SVE2_128_NOMAX-NEXT: .hword 6 // 0x6
+; SVE2_128_NOMAX-NEXT: .hword 7 // 0x7
+
+; SVE2_NOMIN_NOMAX: .LCPI7_0:
+; SVE2_NOMIN_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT:.LCPI7_1:
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT: .hword 2 // 0x2
+; SVE2_NOMIN_NOMAX-NEXT: .hword 3 // 0x3
+; SVE2_NOMIN_NOMAX-NEXT: .hword 4 // 0x4
+; SVE2_NOMIN_NOMAX-NEXT: .hword 4 // 0x4
+; SVE2_NOMIN_NOMAX-NEXT: .hword 6 // 0x6
+; SVE2_NOMIN_NOMAX-NEXT: .hword 7 // 0x7
+
+; SVE2_MIN_256_NOMAX: .LCPI7_0:
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT:.LCPI7_1:
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 2 // 0x2
+; SVE2_MIN_256_NOMAX-NEXT: .hword 3 // 0x3
+; SVE2_MIN_256_NOMAX-NEXT: .hword 4 // 0x4
+; SVE2_MIN_256_NOMAX-NEXT: .hword 4 // 0x4
+; SVE2_MIN_256_NOMAX-NEXT: .hword 6 // 0x6
+; SVE2_MIN_256_NOMAX-NEXT: .hword 7 // 0x7
+; SVE2_MIN_256_NOMAX-NEXT: .hword 65535 // 0xffff
+; SVE2_MIN_256_NOMAX-NEXT: .hword 65535 // 0xffff
+; SVE2_MIN_256_NOMAX-NEXT: .hword 65535 // 0xffff
+; SVE2_MIN_256_NOMAX-NEXT: .hword 65535 // 0xffff
+; SVE2_MIN_256_NOMAX-NEXT: .hword 65535 // 0xffff
+; SVE2_MIN_256_NOMAX-NEXT: .hword 65535 // 0xffff
+; SVE2_MIN_256_NOMAX-NEXT: .hword 65535 // 0xffff
+; SVE2_MIN_256_NOMAX-NEXT: .hword 65535 // 0xffff
+define <8 x i16> @shuffle_index_indices_from_both_ops_i16(ptr %a, ptr %b) {
+; SVE2_128-LABEL: shuffle_index_indices_from_both_ops_i16:
+; SVE2_128: // %bb.0:
+; SVE2_128-NEXT: adrp x8, .LCPI7_0
+; SVE2_128-NEXT: ldr q0, [x0]
+; SVE2_128-NEXT: ldr q1, [x1]
+; SVE2_128-NEXT: ldr q2, [x8, :lo12:.LCPI7_0]
+; SVE2_128-NEXT: tbl z0.h, { z0.h, z1.h }, z2.h
+; SVE2_128-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2_128-NEXT: ret
+;
+; SVE2_128_NOMAX-LABEL: shuffle_index_indices_from_both_ops_i16:
+; SVE2_128_NOMAX: // %bb.0:
+; SVE2_128_NOMAX-NEXT: rdvl x8, #1
+; SVE2_128_NOMAX-NEXT: ptrue p0.h, vl8
+; SVE2_128_NOMAX-NEXT: adrp x9, .LCPI7_0
+; SVE2_128_NOMAX-NEXT: lsr x8, x8, #4
+; SVE2_128_NOMAX-NEXT: ldr q1, [x9, :lo12:.LCPI7_0]
+; SVE2_128_NOMAX-NEXT: lsl w8, w8, #3
+; SVE2_128_NOMAX-NEXT: mov z0.h, w8
+; SVE2_128_NOMAX-NEXT: adrp x8, .LCPI7_1
+; SVE2_128_NOMAX-NEXT: ldr q2, [x8, :lo12:.LCPI7_1]
+; SVE2_128_NOMAX-NEXT: mad z0.h, p0/m, z1.h, z2.h
+; SVE2_128_NOMAX-NEXT: ldr q1, [x0]
+; SVE2_128_NOMAX-NEXT: ldr q2, [x1]
+; SVE2_128_NOMAX-NEXT: tbl z0.h, { z1.h, z2.h }, z0.h
+; SVE2_128_NOMAX-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2_128_NOMAX-NEXT: ret
+;
+; SVE2_NOMIN_NOMAX-LABEL: shuffle_index_indices_from_both_ops_i16:
+; SVE2_NOMIN_NOMAX: // %bb.0:
+; SVE2_NOMIN_NOMAX-NEXT: rdvl x8, #1
+; SVE2_NOMIN_NOMAX-NEXT: ptrue p0.h, vl8
+; SVE2_NOMIN_NOMAX-NEXT: adrp x9, .LCPI7_0
+; SVE2_NOMIN_NOMAX-NEXT: lsr x8, x8, #4
+; SVE2_NOMIN_NOMAX-NEXT: ldr q1, [x9, :lo12:.LCPI7_0]
+; SVE2_NOMIN_NOMAX-NEXT: lsl w8, w8, #3
+; SVE2_NOMIN_NOMAX-NEXT: mov z0.h, w8
+; SVE2_NOMIN_NOMAX-NEXT: adrp x8, .LCPI7_1
+; SVE2_NOMIN_NOMAX-NEXT: ldr q2, [x8, :lo12:.LCPI7_1]
+; SVE2_NOMIN_NOMAX-NEXT: mad z0.h, p0/m, z1.h, z2.h
+; SVE2_NOMIN_NOMAX-NEXT: ldr q1, [x0]
+; SVE2_NOMIN_NOMAX-NEXT: ldr q2, [x1]
+; SVE2_NOMIN_NOMAX-NEXT: tbl z0.h, { z1.h, z2.h }, z0.h
+; SVE2_NOMIN_NOMAX-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2_NOMIN_NOMAX-NEXT: ret
+;
+; SVE2_MIN_256_NOMAX-LABEL: shuffle_index_indices_from_both_ops_i16:
+; SVE2_MIN_256_NOMAX: // %bb.0:
+; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.h, vl16
+; SVE2_MIN_256_NOMAX-NEXT: rdvl x8, #1
+; SVE2_MIN_256_NOMAX-NEXT: ldr q0, [x0]
+; SVE2_MIN_256_NOMAX-NEXT: lsr x8, x8, #4
+; SVE2_MIN_256_NOMAX-NEXT: adrp x9, .LCPI7_0
+; SVE2_MIN_256_NOMAX-NEXT: add x9, x9, :lo12:.LCPI7_0
+; SVE2_MIN_256_NOMAX-NEXT: adrp x10, .LCPI7_1
+; SVE2_MIN_256_NOMAX-NEXT: add x10, x10, :lo12:.LCPI7_1
+; SVE2_MIN_256_NOMAX-NEXT: ldr q1, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: lsl w8, w8, #3
+; SVE2_MIN_256_NOMAX-NEXT: mov z4.h, w8
+; SVE2_MIN_256_NOMAX-NEXT: ld1h { z2.h }, p0/z, [x9]
+; SVE2_MIN_256_NOMAX-NEXT: ld1h { z3.h }, p0/z, [x10]
+; SVE2_MIN_256_NOMAX-NEXT: mad z2.h, p0/m, z4.h, z3.h
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.h, { z0.h, z1.h }, z2.h
+; SVE2_MIN_256_NOMAX-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2_MIN_256_NOMAX-NEXT: ret
+ %op1 = load <8 x i16>, ptr %a
+ %op2 = load <8 x i16>, ptr %b
+ %1 = shufflevector <8 x i16> %op1, <8 x i16> %op2, <8 x i32> <i32 1, i32 9, i32 10, i32 11, i32 12, i32 12, i32 14, i32 15>
+ ret <8 x i16> %1
+}
>From 44536556d5ddf82f55f66f3cf25dce1582f42b8c Mon Sep 17 00:00:00 2001
From: Dinar Temirbulatov <Dinar.Temirbulatov at arm.com>
Date: Mon, 11 Dec 2023 12:02:30 +0000
Subject: [PATCH 2/7] Resolved issue with 64-bit element types incorrect
lowering.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 14 +++++---
.../sve-fixed-length-vector-shuffle-tbl.ll | 34 +++++++++----------
2 files changed, 27 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 1ba7666fada73f..0a06c6a099e339 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -26817,7 +26817,9 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
// TBL mask element needs adjustment.
SmallVector<SDValue, 8> MaskNormalized;
- if (BitsPerElt == 8 && !MinMaxEqual && !IsSingleOp)
+ // Bail out for 8-bits element types, because with 2048-bit SVE register
+ // size we could not repersent index correctly.
+ if (!IsSingleOp && !MinMaxEqual && BitsPerElt == 8)
return SDValue();
for (int Index : ShuffleMask) {
@@ -26871,10 +26873,14 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
Op1, SVEMask);
else if (Subtarget.hasSVE2()) {
if (!MinMaxEqual) {
- SDValue VScale = DAG.getVScale(DL, MVT::i32, APInt(32, 1));
+ SDValue VScale = (BitsPerElt == 64)
+ ? DAG.getVScale(DL, MVT::i64, APInt(64, 1))
+ : DAG.getVScale(DL, MVT::i32, APInt(32, 1));
SDValue Mul =
- DAG.getNode(ISD::MUL, DL, MVT::i32,
- DAG.getConstant(128 / BitsPerElt, DL, MVT::i32), VScale);
+ DAG.getNode(ISD::MUL, DL, (BitsPerElt == 64) ? MVT::i64 : MVT::i32,
+ DAG.getConstant(128 / BitsPerElt, DL,
+ (BitsPerElt == 64) ? MVT::i64 : MVT::i32),
+ VScale);
SDValue VecMask =
DAG.getBuildVector(MaskType, DL, ArrayRef(TBLMask.data(), IndexLen));
SDValue MulMask = DAG.getBuildVector(
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
index 331f9609586b1a..43735e0cbe0b3c 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
@@ -50,9 +50,9 @@ define <8 x i8> @shuffle_index_indices_from_op1(ptr %a, ptr %b) {
; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.b, vl32
; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI0_0
; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI0_0
-; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x0]
-; SVE2_MIN_256_NOMAX-NEXT: ld1b { z1.b }, p0/z, [x8]
-; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_MIN_256_NOMAX-NEXT: ldr d1, [x0]
+; SVE2_MIN_256_NOMAX-NEXT: ld1b { z0.b }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z1.b }, z0.b
; SVE2_MIN_256_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i8>, ptr %a
@@ -105,9 +105,9 @@ define <8 x i8> @shuffle_index_indices_from_op2(ptr %a, ptr %b) {
; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.b, vl32
; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI1_0
; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI1_0
-; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x1]
-; SVE2_MIN_256_NOMAX-NEXT: ld1b { z1.b }, p0/z, [x8]
-; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_MIN_256_NOMAX-NEXT: ldr d1, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: ld1b { z0.b }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z1.b }, z0.b
; SVE2_MIN_256_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i8>, ptr %a
@@ -386,9 +386,9 @@ define <8 x i8> @shuffle_op1_poison(ptr %a, ptr %b) {
; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.b, vl32
; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI4_0
; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI4_0
-; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x1]
-; SVE2_MIN_256_NOMAX-NEXT: ld1b { z1.b }, p0/z, [x8]
-; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_MIN_256_NOMAX-NEXT: ldr d1, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: ld1b { z0.b }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z1.b }, z0.b
; SVE2_MIN_256_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE2_MIN_256_NOMAX-NEXT: ret
%op2 = load <8 x i8>, ptr %b
@@ -597,19 +597,19 @@ define <8 x i16> @shuffle_index_indices_from_both_ops_i16(ptr %a, ptr %b) {
; SVE2_MIN_256_NOMAX: // %bb.0:
; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.h, vl16
; SVE2_MIN_256_NOMAX-NEXT: rdvl x8, #1
-; SVE2_MIN_256_NOMAX-NEXT: ldr q0, [x0]
-; SVE2_MIN_256_NOMAX-NEXT: lsr x8, x8, #4
; SVE2_MIN_256_NOMAX-NEXT: adrp x9, .LCPI7_0
; SVE2_MIN_256_NOMAX-NEXT: add x9, x9, :lo12:.LCPI7_0
+; SVE2_MIN_256_NOMAX-NEXT: lsr x8, x8, #4
; SVE2_MIN_256_NOMAX-NEXT: adrp x10, .LCPI7_1
; SVE2_MIN_256_NOMAX-NEXT: add x10, x10, :lo12:.LCPI7_1
-; SVE2_MIN_256_NOMAX-NEXT: ldr q1, [x1]
; SVE2_MIN_256_NOMAX-NEXT: lsl w8, w8, #3
-; SVE2_MIN_256_NOMAX-NEXT: mov z4.h, w8
-; SVE2_MIN_256_NOMAX-NEXT: ld1h { z2.h }, p0/z, [x9]
-; SVE2_MIN_256_NOMAX-NEXT: ld1h { z3.h }, p0/z, [x10]
-; SVE2_MIN_256_NOMAX-NEXT: mad z2.h, p0/m, z4.h, z3.h
-; SVE2_MIN_256_NOMAX-NEXT: tbl z0.h, { z0.h, z1.h }, z2.h
+; SVE2_MIN_256_NOMAX-NEXT: mov z2.h, w8
+; SVE2_MIN_256_NOMAX-NEXT: ld1h { z0.h }, p0/z, [x9]
+; SVE2_MIN_256_NOMAX-NEXT: ld1h { z1.h }, p0/z, [x10]
+; SVE2_MIN_256_NOMAX-NEXT: mad z0.h, p0/m, z2.h, z1.h
+; SVE2_MIN_256_NOMAX-NEXT: ldr q1, [x0]
+; SVE2_MIN_256_NOMAX-NEXT: ldr q2, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.h, { z1.h, z2.h }, z0.h
; SVE2_MIN_256_NOMAX-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i16>, ptr %a
>From d981d1ef62d5c48dbe98f2afa2861c1575d3406e Mon Sep 17 00:00:00 2001
From: Dinar Temirbulatov <Dinar.Temirbulatov at arm.com>
Date: Thu, 14 Dec 2023 13:40:07 +0000
Subject: [PATCH 3/7] Corrected comment.
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 0a06c6a099e339..88670b86773749 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -26817,7 +26817,7 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
// TBL mask element needs adjustment.
SmallVector<SDValue, 8> MaskNormalized;
- // Bail out for 8-bits element types, because with 2048-bit SVE register
+ // Avoid if 8-bits element types, since with 2048-bit SVE register
// size we could not repersent index correctly.
if (!IsSingleOp && !MinMaxEqual && BitsPerElt == 8)
return SDValue();
>From 990730409b55ad1e787ef095419e9c019acd8c38 Mon Sep 17 00:00:00 2001
From: Dinar Temirbulatov <Dinar.Temirbulatov at arm.com>
Date: Thu, 14 Dec 2023 15:37:14 +0000
Subject: [PATCH 4/7] Corrected test.
---
.../sve-fixed-length-vector-shuffle-tbl.ll | 40 +++++++++----------
1 file changed, 20 insertions(+), 20 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
index 43735e0cbe0b3c..3beccdf278ebd1 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
@@ -50,9 +50,9 @@ define <8 x i8> @shuffle_index_indices_from_op1(ptr %a, ptr %b) {
; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.b, vl32
; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI0_0
; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI0_0
-; SVE2_MIN_256_NOMAX-NEXT: ldr d1, [x0]
-; SVE2_MIN_256_NOMAX-NEXT: ld1b { z0.b }, p0/z, [x8]
-; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z1.b }, z0.b
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x0]
+; SVE2_MIN_256_NOMAX-NEXT: ld1b { z1.b }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
; SVE2_MIN_256_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i8>, ptr %a
@@ -105,9 +105,9 @@ define <8 x i8> @shuffle_index_indices_from_op2(ptr %a, ptr %b) {
; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.b, vl32
; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI1_0
; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI1_0
-; SVE2_MIN_256_NOMAX-NEXT: ldr d1, [x1]
-; SVE2_MIN_256_NOMAX-NEXT: ld1b { z0.b }, p0/z, [x8]
-; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z1.b }, z0.b
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: ld1b { z1.b }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
; SVE2_MIN_256_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i8>, ptr %a
@@ -386,9 +386,9 @@ define <8 x i8> @shuffle_op1_poison(ptr %a, ptr %b) {
; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.b, vl32
; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI4_0
; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI4_0
-; SVE2_MIN_256_NOMAX-NEXT: ldr d1, [x1]
-; SVE2_MIN_256_NOMAX-NEXT: ld1b { z0.b }, p0/z, [x8]
-; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z1.b }, z0.b
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: ld1b { z1.b }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
; SVE2_MIN_256_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE2_MIN_256_NOMAX-NEXT: ret
%op2 = load <8 x i8>, ptr %b
@@ -453,9 +453,9 @@ define <8 x i8> @shuffle_index_size_op1_maxhw(ptr %a, ptr %b) "target-features"=
; CHECK-NEXT: ptrue p0.b
; CHECK-NEXT: adrp x8, .LCPI6_0
; CHECK-NEXT: add x8, x8, :lo12:.LCPI6_0
-; CHECK-NEXT: ldr d1, [x0]
-; CHECK-NEXT: ld1b { z0.b }, p0/z, [x8]
-; CHECK-NEXT: tbl z0.b, { z1.b }, z0.b
+; CHECK-NEXT: ldr d0, [x0]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x8]
+; CHECK-NEXT: tbl z0.b, { z0.b }, z1.b
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT: ret
%op1 = load <8 x i8>, ptr %a
@@ -597,19 +597,19 @@ define <8 x i16> @shuffle_index_indices_from_both_ops_i16(ptr %a, ptr %b) {
; SVE2_MIN_256_NOMAX: // %bb.0:
; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.h, vl16
; SVE2_MIN_256_NOMAX-NEXT: rdvl x8, #1
+; SVE2_MIN_256_NOMAX-NEXT: ldr q0, [x0]
+; SVE2_MIN_256_NOMAX-NEXT: lsr x8, x8, #4
; SVE2_MIN_256_NOMAX-NEXT: adrp x9, .LCPI7_0
; SVE2_MIN_256_NOMAX-NEXT: add x9, x9, :lo12:.LCPI7_0
-; SVE2_MIN_256_NOMAX-NEXT: lsr x8, x8, #4
; SVE2_MIN_256_NOMAX-NEXT: adrp x10, .LCPI7_1
; SVE2_MIN_256_NOMAX-NEXT: add x10, x10, :lo12:.LCPI7_1
+; SVE2_MIN_256_NOMAX-NEXT: ldr q1, [x1]
; SVE2_MIN_256_NOMAX-NEXT: lsl w8, w8, #3
-; SVE2_MIN_256_NOMAX-NEXT: mov z2.h, w8
-; SVE2_MIN_256_NOMAX-NEXT: ld1h { z0.h }, p0/z, [x9]
-; SVE2_MIN_256_NOMAX-NEXT: ld1h { z1.h }, p0/z, [x10]
-; SVE2_MIN_256_NOMAX-NEXT: mad z0.h, p0/m, z2.h, z1.h
-; SVE2_MIN_256_NOMAX-NEXT: ldr q1, [x0]
-; SVE2_MIN_256_NOMAX-NEXT: ldr q2, [x1]
-; SVE2_MIN_256_NOMAX-NEXT: tbl z0.h, { z1.h, z2.h }, z0.h
+; SVE2_MIN_256_NOMAX-NEXT: mov z4.h, w8
+; SVE2_MIN_256_NOMAX-NEXT: ld1h { z2.h }, p0/z, [x9]
+; SVE2_MIN_256_NOMAX-NEXT: ld1h { z3.h }, p0/z, [x10]
+; SVE2_MIN_256_NOMAX-NEXT: mad z2.h, p0/m, z4.h, z3.h
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.h, { z0.h, z1.h }, z2.h
; SVE2_MIN_256_NOMAX-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i16>, ptr %a
>From 3cd68b434b76b587064d244148cf50a6222dfd8e Mon Sep 17 00:00:00 2001
From: Dinar Temirbulatov <Dinar.Temirbulatov at arm.com>
Date: Thu, 11 Jan 2024 01:36:19 +0000
Subject: [PATCH 5/7] Resolved remarks.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 58 ++++++++-----------
.../sve-fixed-length-vector-shuffle-tbl.ll | 38 +++++-------
2 files changed, 40 insertions(+), 56 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 88670b86773749..b1191aa14750e3 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -26806,7 +26806,7 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
unsigned IndexLen = MinSVESize / BitsPerElt;
unsigned ElementsPerVectorReg = VTOp1.getVectorNumElements();
uint64_t MaxOffset = APInt(BitsPerElt, -1, false).getZExtValue();
- EVT MaskEltType = EVT::getIntegerVT(*DAG.getContext(), BitsPerElt);
+ EVT MaskEltType = VTOp1.getVectorElementType().changeTypeToInteger();
EVT MaskType = EVT::getVectorVT(*DAG.getContext(), MaskEltType, IndexLen);
bool MinMaxEqual = (MinSVESize == MaxSVESize);
assert(ElementsPerVectorReg <= IndexLen && ShuffleMask.size() <= IndexLen &&
@@ -26815,10 +26815,10 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
SmallVector<SDValue, 8> TBLMask;
// If MinSVESize is not equal to MaxSVESize then we need to know which
// TBL mask element needs adjustment.
- SmallVector<SDValue, 8> MaskNormalized;
+ SmallVector<SDValue, 8> MulByVLMask;
- // Avoid if 8-bits element types, since with 2048-bit SVE register
- // size we could not repersent index correctly.
+ // Bail out for 8-bits element types, because with 2048-bit SVE register
+ // size 8 bits is only sufficient to index into the first source vector.
if (!IsSingleOp && !MinMaxEqual && BitsPerElt == 8)
return SDValue();
@@ -26829,18 +26829,18 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
// If we refer to the second operand then we have to add elements
// number in hardware register minus number of elements in a type in
// case if MinSVESize equals to MaxSVESize, otherwise just add normalized
- // value and record this element in MaskNormalized to be adjusted in the
+ // value and record this element in MulByVLMask to be adjusted in the
// runtime.
if ((unsigned)Index >= ElementsPerVectorReg) {
if (!MinMaxEqual) {
Index = Index - ElementsPerVectorReg;
- MaskNormalized.push_back(DAG.getConstant(1, DL, MVT::i64));
+ MulByVLMask.push_back(DAG.getConstant(1, DL, MVT::i64));
} else {
Index += IndexLen - ElementsPerVectorReg;
}
} else {
if (!MinMaxEqual)
- MaskNormalized.push_back(DAG.getConstant(0, DL, MVT::i64));
+ MulByVLMask.push_back(DAG.getConstant(0, DL, MVT::i64));
}
// For 8-bit elements and 1024-bit SVE registers and MaxOffset equals
// to 255, this might point to the last element of in the second operand
@@ -26857,7 +26857,7 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
for (unsigned i = 0; i < IndexLen - ElementsPerVectorReg; ++i) {
TBLMask.push_back(DAG.getConstant((int)MaxOffset, DL, MVT::i64));
if (!MinMaxEqual)
- MaskNormalized.push_back(DAG.getConstant(0, DL, MVT::i64));
+ MulByVLMask.push_back(DAG.getConstant(0, DL, MVT::i64));
}
EVT MaskContainerVT = getContainerForFixedLengthVector(DAG, MaskType);
@@ -26873,36 +26873,26 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
Op1, SVEMask);
else if (Subtarget.hasSVE2()) {
if (!MinMaxEqual) {
- SDValue VScale = (BitsPerElt == 64)
- ? DAG.getVScale(DL, MVT::i64, APInt(64, 1))
- : DAG.getVScale(DL, MVT::i32, APInt(32, 1));
- SDValue Mul =
- DAG.getNode(ISD::MUL, DL, (BitsPerElt == 64) ? MVT::i64 : MVT::i32,
- DAG.getConstant(128 / BitsPerElt, DL,
- (BitsPerElt == 64) ? MVT::i64 : MVT::i32),
- VScale);
+ SDValue VScale =
+ (BitsPerElt == 64)
+ ? DAG.getVScale(DL, MVT::i64, APInt(64, 128 / BitsPerElt))
+ : DAG.getVScale(DL, MVT::i32, APInt(32, 128 / BitsPerElt));
SDValue VecMask =
DAG.getBuildVector(MaskType, DL, ArrayRef(TBLMask.data(), IndexLen));
- SDValue MulMask = DAG.getBuildVector(
- MaskType, DL, ArrayRef(MaskNormalized.data(), IndexLen));
- SDValue SplatPred = DAG.getNode(ISD::SPLAT_VECTOR, DL, MaskType, Mul);
- SDValue MulMaskNormalized =
- DAG.getNode(ISD::MUL, DL, MaskType, SplatPred, MulMask);
+ SDValue MulByMask = DAG.getNode(
+ ISD::MUL, DL, MaskType,
+ DAG.getNode(ISD::SPLAT_VECTOR, DL, MaskType, VScale),
+ DAG.getBuildVector(MaskType, DL,
+ ArrayRef(MulByVLMask.data(), IndexLen)));
SDValue UpdatedVecMask =
- DAG.getNode(ISD::ADD, DL, MaskType, VecMask, MulMaskNormalized);
- EVT MaskContainerVT = getContainerForFixedLengthVector(DAG, MaskType);
- SDValue SVEMask =
- convertToScalableVector(DAG, MaskContainerVT, UpdatedVecMask);
- Shuffle = DAG.getNode(
- ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
- DAG.getConstant(Intrinsic::aarch64_sve_tbl2, DL, MVT::i32), Op1, Op2,
- SVEMask);
- } else {
- Shuffle = DAG.getNode(
- ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
- DAG.getConstant(Intrinsic::aarch64_sve_tbl2, DL, MVT::i32), Op1, Op2,
- SVEMask);
+ DAG.getNode(ISD::ADD, DL, MaskType, VecMask, MulByMask);
+ SVEMask = convertToScalableVector(
+ DAG, getContainerForFixedLengthVector(DAG, MaskType), UpdatedVecMask);
}
+ Shuffle =
+ DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
+ DAG.getConstant(Intrinsic::aarch64_sve_tbl2, DL, MVT::i32),
+ Op1, Op2, SVEMask);
}
Shuffle = convertFromScalableVector(DAG, VT, Shuffle);
return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Shuffle);
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
index 3beccdf278ebd1..57b1ab3f9eacff 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
@@ -559,15 +559,13 @@ define <8 x i16> @shuffle_index_indices_from_both_ops_i16(ptr %a, ptr %b) {
;
; SVE2_128_NOMAX-LABEL: shuffle_index_indices_from_both_ops_i16:
; SVE2_128_NOMAX: // %bb.0:
-; SVE2_128_NOMAX-NEXT: rdvl x8, #1
; SVE2_128_NOMAX-NEXT: ptrue p0.h, vl8
+; SVE2_128_NOMAX-NEXT: cnth x8
; SVE2_128_NOMAX-NEXT: adrp x9, .LCPI7_0
-; SVE2_128_NOMAX-NEXT: lsr x8, x8, #4
-; SVE2_128_NOMAX-NEXT: ldr q1, [x9, :lo12:.LCPI7_0]
-; SVE2_128_NOMAX-NEXT: lsl w8, w8, #3
+; SVE2_128_NOMAX-NEXT: adrp x10, .LCPI7_1
; SVE2_128_NOMAX-NEXT: mov z0.h, w8
-; SVE2_128_NOMAX-NEXT: adrp x8, .LCPI7_1
-; SVE2_128_NOMAX-NEXT: ldr q2, [x8, :lo12:.LCPI7_1]
+; SVE2_128_NOMAX-NEXT: ldr q1, [x9, :lo12:.LCPI7_0]
+; SVE2_128_NOMAX-NEXT: ldr q2, [x10, :lo12:.LCPI7_1]
; SVE2_128_NOMAX-NEXT: mad z0.h, p0/m, z1.h, z2.h
; SVE2_128_NOMAX-NEXT: ldr q1, [x0]
; SVE2_128_NOMAX-NEXT: ldr q2, [x1]
@@ -577,15 +575,13 @@ define <8 x i16> @shuffle_index_indices_from_both_ops_i16(ptr %a, ptr %b) {
;
; SVE2_NOMIN_NOMAX-LABEL: shuffle_index_indices_from_both_ops_i16:
; SVE2_NOMIN_NOMAX: // %bb.0:
-; SVE2_NOMIN_NOMAX-NEXT: rdvl x8, #1
; SVE2_NOMIN_NOMAX-NEXT: ptrue p0.h, vl8
+; SVE2_NOMIN_NOMAX-NEXT: cnth x8
; SVE2_NOMIN_NOMAX-NEXT: adrp x9, .LCPI7_0
-; SVE2_NOMIN_NOMAX-NEXT: lsr x8, x8, #4
-; SVE2_NOMIN_NOMAX-NEXT: ldr q1, [x9, :lo12:.LCPI7_0]
-; SVE2_NOMIN_NOMAX-NEXT: lsl w8, w8, #3
+; SVE2_NOMIN_NOMAX-NEXT: adrp x10, .LCPI7_1
; SVE2_NOMIN_NOMAX-NEXT: mov z0.h, w8
-; SVE2_NOMIN_NOMAX-NEXT: adrp x8, .LCPI7_1
-; SVE2_NOMIN_NOMAX-NEXT: ldr q2, [x8, :lo12:.LCPI7_1]
+; SVE2_NOMIN_NOMAX-NEXT: ldr q1, [x9, :lo12:.LCPI7_0]
+; SVE2_NOMIN_NOMAX-NEXT: ldr q2, [x10, :lo12:.LCPI7_1]
; SVE2_NOMIN_NOMAX-NEXT: mad z0.h, p0/m, z1.h, z2.h
; SVE2_NOMIN_NOMAX-NEXT: ldr q1, [x0]
; SVE2_NOMIN_NOMAX-NEXT: ldr q2, [x1]
@@ -596,18 +592,16 @@ define <8 x i16> @shuffle_index_indices_from_both_ops_i16(ptr %a, ptr %b) {
; SVE2_MIN_256_NOMAX-LABEL: shuffle_index_indices_from_both_ops_i16:
; SVE2_MIN_256_NOMAX: // %bb.0:
; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.h, vl16
-; SVE2_MIN_256_NOMAX-NEXT: rdvl x8, #1
; SVE2_MIN_256_NOMAX-NEXT: ldr q0, [x0]
-; SVE2_MIN_256_NOMAX-NEXT: lsr x8, x8, #4
-; SVE2_MIN_256_NOMAX-NEXT: adrp x9, .LCPI7_0
-; SVE2_MIN_256_NOMAX-NEXT: add x9, x9, :lo12:.LCPI7_0
-; SVE2_MIN_256_NOMAX-NEXT: adrp x10, .LCPI7_1
-; SVE2_MIN_256_NOMAX-NEXT: add x10, x10, :lo12:.LCPI7_1
+; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI7_0
+; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI7_0
+; SVE2_MIN_256_NOMAX-NEXT: adrp x9, .LCPI7_1
+; SVE2_MIN_256_NOMAX-NEXT: add x9, x9, :lo12:.LCPI7_1
+; SVE2_MIN_256_NOMAX-NEXT: cnth x10
; SVE2_MIN_256_NOMAX-NEXT: ldr q1, [x1]
-; SVE2_MIN_256_NOMAX-NEXT: lsl w8, w8, #3
-; SVE2_MIN_256_NOMAX-NEXT: mov z4.h, w8
-; SVE2_MIN_256_NOMAX-NEXT: ld1h { z2.h }, p0/z, [x9]
-; SVE2_MIN_256_NOMAX-NEXT: ld1h { z3.h }, p0/z, [x10]
+; SVE2_MIN_256_NOMAX-NEXT: mov z4.h, w10
+; SVE2_MIN_256_NOMAX-NEXT: ld1h { z2.h }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: ld1h { z3.h }, p0/z, [x9]
; SVE2_MIN_256_NOMAX-NEXT: mad z2.h, p0/m, z4.h, z3.h
; SVE2_MIN_256_NOMAX-NEXT: tbl z0.h, { z0.h, z1.h }, z2.h
; SVE2_MIN_256_NOMAX-NEXT: // kill: def $q0 killed $q0 killed $z0
>From c7dc3c3b4ffa10b3168e2697ff68dea589f87826 Mon Sep 17 00:00:00 2001
From: Dinar Temirbulatov <Dinar.Temirbulatov at arm.com>
Date: Wed, 14 Feb 2024 12:45:19 +0000
Subject: [PATCH 6/7] Resolved comments.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 37 +++++++++----------
1 file changed, 17 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index b1191aa14750e3..184ebc19bc9ede 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -26815,7 +26815,7 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
SmallVector<SDValue, 8> TBLMask;
// If MinSVESize is not equal to MaxSVESize then we need to know which
// TBL mask element needs adjustment.
- SmallVector<SDValue, 8> MulByVLMask;
+ SmallVector<SDValue, 8> AddRuntimeVLMask;
// Bail out for 8-bits element types, because with 2048-bit SVE register
// size 8 bits is only sufficient to index into the first source vector.
@@ -26826,22 +26826,19 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
// Handling poison index value.
if (Index < 0)
Index = 0;
- // If we refer to the second operand then we have to add elements
- // number in hardware register minus number of elements in a type in
- // case if MinSVESize equals to MaxSVESize, otherwise just add normalized
- // value and record this element in MulByVLMask to be adjusted in the
- // runtime.
+ // If the mask refers to elements in the second operand, then we have to
+ // offset the index by the number of elements in a vector. If this is number
+ // is not known at compile-time, we need to maintain a mask with 'VL' values
+ // to add at runtime.
if ((unsigned)Index >= ElementsPerVectorReg) {
- if (!MinMaxEqual) {
- Index = Index - ElementsPerVectorReg;
- MulByVLMask.push_back(DAG.getConstant(1, DL, MVT::i64));
- } else {
+ if (MinMaxEqual) {
Index += IndexLen - ElementsPerVectorReg;
+ } else {
+ Index = Index - ElementsPerVectorReg;
+ AddRuntimeVLMask.push_back(DAG.getConstant(1, DL, MVT::i64));
}
- } else {
- if (!MinMaxEqual)
- MulByVLMask.push_back(DAG.getConstant(0, DL, MVT::i64));
- }
+ } else if (!MinMaxEqual)
+ AddRuntimeVLMask.push_back(DAG.getConstant(0, DL, MVT::i64));
// For 8-bit elements and 1024-bit SVE registers and MaxOffset equals
// to 255, this might point to the last element of in the second operand
// of the shufflevector, thus we are rejecting this transform.
@@ -26857,7 +26854,7 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
for (unsigned i = 0; i < IndexLen - ElementsPerVectorReg; ++i) {
TBLMask.push_back(DAG.getConstant((int)MaxOffset, DL, MVT::i64));
if (!MinMaxEqual)
- MulByVLMask.push_back(DAG.getConstant(0, DL, MVT::i64));
+ AddRuntimeVLMask.push_back(DAG.getConstant(0, DL, MVT::i64));
}
EVT MaskContainerVT = getContainerForFixedLengthVector(DAG, MaskType);
@@ -26873,17 +26870,17 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
Op1, SVEMask);
else if (Subtarget.hasSVE2()) {
if (!MinMaxEqual) {
- SDValue VScale =
- (BitsPerElt == 64)
- ? DAG.getVScale(DL, MVT::i64, APInt(64, 128 / BitsPerElt))
- : DAG.getVScale(DL, MVT::i32, APInt(32, 128 / BitsPerElt));
+ unsigned MinNumElts = AArch64::SVEBitsPerBlock / BitsPerElt;
+ SDValue VScale = (BitsPerElt == 64)
+ ? DAG.getVScale(DL, MVT::i64, APInt(64, MinNumElts))
+ : DAG.getVScale(DL, MVT::i32, APInt(32, MinNumElts));
SDValue VecMask =
DAG.getBuildVector(MaskType, DL, ArrayRef(TBLMask.data(), IndexLen));
SDValue MulByMask = DAG.getNode(
ISD::MUL, DL, MaskType,
DAG.getNode(ISD::SPLAT_VECTOR, DL, MaskType, VScale),
DAG.getBuildVector(MaskType, DL,
- ArrayRef(MulByVLMask.data(), IndexLen)));
+ ArrayRef(AddRuntimeVLMask.data(), IndexLen)));
SDValue UpdatedVecMask =
DAG.getNode(ISD::ADD, DL, MaskType, VecMask, MulByMask);
SVEMask = convertToScalableVector(
>From 8d9ed31f179aa20c59d9da69c4a9b50d5d1f45e5 Mon Sep 17 00:00:00 2001
From: Dinar Temirbulatov <Dinar.Temirbulatov at arm.com>
Date: Wed, 21 Feb 2024 13:26:27 +0000
Subject: [PATCH 7/7] Update sve-fixed-length-vector-shuffle-tbl.ll test.
---
.../sve-fixed-length-vector-shuffle-tbl.ll | 38 +++++++++----------
1 file changed, 19 insertions(+), 19 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
index 57b1ab3f9eacff..68c234a20d1108 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
@@ -50,9 +50,9 @@ define <8 x i8> @shuffle_index_indices_from_op1(ptr %a, ptr %b) {
; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.b, vl32
; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI0_0
; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI0_0
-; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x0]
-; SVE2_MIN_256_NOMAX-NEXT: ld1b { z1.b }, p0/z, [x8]
-; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_MIN_256_NOMAX-NEXT: ldr d1, [x0]
+; SVE2_MIN_256_NOMAX-NEXT: ld1b { z0.b }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z1.b }, z0.b
; SVE2_MIN_256_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i8>, ptr %a
@@ -105,9 +105,9 @@ define <8 x i8> @shuffle_index_indices_from_op2(ptr %a, ptr %b) {
; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.b, vl32
; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI1_0
; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI1_0
-; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x1]
-; SVE2_MIN_256_NOMAX-NEXT: ld1b { z1.b }, p0/z, [x8]
-; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_MIN_256_NOMAX-NEXT: ldr d1, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: ld1b { z0.b }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z1.b }, z0.b
; SVE2_MIN_256_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i8>, ptr %a
@@ -386,9 +386,9 @@ define <8 x i8> @shuffle_op1_poison(ptr %a, ptr %b) {
; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.b, vl32
; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI4_0
; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI4_0
-; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x1]
-; SVE2_MIN_256_NOMAX-NEXT: ld1b { z1.b }, p0/z, [x8]
-; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_MIN_256_NOMAX-NEXT: ldr d1, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: ld1b { z0.b }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z1.b }, z0.b
; SVE2_MIN_256_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE2_MIN_256_NOMAX-NEXT: ret
%op2 = load <8 x i8>, ptr %b
@@ -453,9 +453,9 @@ define <8 x i8> @shuffle_index_size_op1_maxhw(ptr %a, ptr %b) "target-features"=
; CHECK-NEXT: ptrue p0.b
; CHECK-NEXT: adrp x8, .LCPI6_0
; CHECK-NEXT: add x8, x8, :lo12:.LCPI6_0
-; CHECK-NEXT: ldr d0, [x0]
-; CHECK-NEXT: ld1b { z1.b }, p0/z, [x8]
-; CHECK-NEXT: tbl z0.b, { z0.b }, z1.b
+; CHECK-NEXT: ldr d1, [x0]
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x8]
+; CHECK-NEXT: tbl z0.b, { z1.b }, z0.b
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT: ret
%op1 = load <8 x i8>, ptr %a
@@ -592,18 +592,18 @@ define <8 x i16> @shuffle_index_indices_from_both_ops_i16(ptr %a, ptr %b) {
; SVE2_MIN_256_NOMAX-LABEL: shuffle_index_indices_from_both_ops_i16:
; SVE2_MIN_256_NOMAX: // %bb.0:
; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.h, vl16
-; SVE2_MIN_256_NOMAX-NEXT: ldr q0, [x0]
; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI7_0
; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI7_0
; SVE2_MIN_256_NOMAX-NEXT: adrp x9, .LCPI7_1
; SVE2_MIN_256_NOMAX-NEXT: add x9, x9, :lo12:.LCPI7_1
; SVE2_MIN_256_NOMAX-NEXT: cnth x10
-; SVE2_MIN_256_NOMAX-NEXT: ldr q1, [x1]
-; SVE2_MIN_256_NOMAX-NEXT: mov z4.h, w10
-; SVE2_MIN_256_NOMAX-NEXT: ld1h { z2.h }, p0/z, [x8]
-; SVE2_MIN_256_NOMAX-NEXT: ld1h { z3.h }, p0/z, [x9]
-; SVE2_MIN_256_NOMAX-NEXT: mad z2.h, p0/m, z4.h, z3.h
-; SVE2_MIN_256_NOMAX-NEXT: tbl z0.h, { z0.h, z1.h }, z2.h
+; SVE2_MIN_256_NOMAX-NEXT: mov z2.h, w10
+; SVE2_MIN_256_NOMAX-NEXT: ld1h { z0.h }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: ld1h { z1.h }, p0/z, [x9]
+; SVE2_MIN_256_NOMAX-NEXT: mad z0.h, p0/m, z2.h, z1.h
+; SVE2_MIN_256_NOMAX-NEXT: ldr q1, [x0]
+; SVE2_MIN_256_NOMAX-NEXT: ldr q2, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.h, { z1.h, z2.h }, z0.h
; SVE2_MIN_256_NOMAX-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i16>, ptr %a
More information about the llvm-commits
mailing list