[llvm] [AArch64][SVE2] Enable dynamic shuffle for fixed length types. (PR #72490)
Dinar Temirbulatov via llvm-commits
llvm-commits at lists.llvm.org
Wed Feb 14 04:46:55 PST 2024
https://github.com/dtemirbulatov updated https://github.com/llvm/llvm-project/pull/72490
>From 290675ce5ff4994fdf3a80449c2057d614d7742a Mon Sep 17 00:00:00 2001
From: Dinar Temirbulatov <Dinar.Temirbulatov at arm.com>
Date: Thu, 16 Nov 2023 08:15:44 +0000
Subject: [PATCH 1/7] [AArch64][SME] Enable dynamic shuffle for fixed length
types.
When SVE register size is unknown or the minimal size is not equal to
the maximum size then we could determine the actual SVE register size in
the runtime and adjust shuffle mask in the runtime.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 73 +++-
.../sve-fixed-length-vector-shuffle-tbl.ll | 413 +++++++++++++++++-
2 files changed, 448 insertions(+), 38 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 9ff6d6f0f565ed..2423ef6f8962a5 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -26123,7 +26123,7 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
// Ignore two operands if no SVE2 or all index numbers couldn't
// be represented.
- if (!IsSingleOp && (!Subtarget.hasSVE2() || MinSVESize != MaxSVESize))
+ if (!IsSingleOp && !Subtarget.hasSVE2())
return SDValue();
EVT VTOp1 = Op.getOperand(0).getValueType();
@@ -26131,18 +26131,40 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
unsigned IndexLen = MinSVESize / BitsPerElt;
unsigned ElementsPerVectorReg = VTOp1.getVectorNumElements();
uint64_t MaxOffset = APInt(BitsPerElt, -1, false).getZExtValue();
+ EVT MaskEltType = EVT::getIntegerVT(*DAG.getContext(), BitsPerElt);
+ EVT MaskType = EVT::getVectorVT(*DAG.getContext(), MaskEltType, IndexLen);
+ bool MinMaxEqual = (MinSVESize == MaxSVESize);
assert(ElementsPerVectorReg <= IndexLen && ShuffleMask.size() <= IndexLen &&
"Incorrectly legalised shuffle operation");
SmallVector<SDValue, 8> TBLMask;
+ // If MinSVESize is not equal to MaxSVESize then we need to know which
+ // TBL mask element needs adjustment.
+ SmallVector<SDValue, 8> MaskNormalized;
+
+ if (BitsPerElt == 8 && !MinMaxEqual && !IsSingleOp)
+ return SDValue();
+
for (int Index : ShuffleMask) {
// Handling poison index value.
if (Index < 0)
Index = 0;
// If we refer to the second operand then we have to add elements
- // number in hardware register minus number of elements in a type.
- if ((unsigned)Index >= ElementsPerVectorReg)
- Index += IndexLen - ElementsPerVectorReg;
+ // number in hardware register minus number of elements in a type in
+ // case if MinSVESize equals to MaxSVESize, otherwise just add normalized
+ // value and record this element in MaskNormalized to be adjusted in the
+ // runtime.
+ if ((unsigned)Index >= ElementsPerVectorReg) {
+ if (!MinMaxEqual) {
+ Index = Index - ElementsPerVectorReg;
+ MaskNormalized.push_back(DAG.getConstant(1, DL, MVT::i64));
+ } else {
+ Index += IndexLen - ElementsPerVectorReg;
+ }
+ } else {
+ if (!MinMaxEqual)
+ MaskNormalized.push_back(DAG.getConstant(0, DL, MVT::i64));
+ }
// For 8-bit elements and 1024-bit SVE registers and MaxOffset equals
// to 255, this might point to the last element of in the second operand
// of the shufflevector, thus we are rejecting this transform.
@@ -26155,11 +26177,12 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
// value where it would perform first lane duplication for out of
// index elements. For i8 elements an out-of-range index could be a valid
// for 2048-bit vector register size.
- for (unsigned i = 0; i < IndexLen - ElementsPerVectorReg; ++i)
+ for (unsigned i = 0; i < IndexLen - ElementsPerVectorReg; ++i) {
TBLMask.push_back(DAG.getConstant((int)MaxOffset, DL, MVT::i64));
+ if (!MinMaxEqual)
+ MaskNormalized.push_back(DAG.getConstant(0, DL, MVT::i64));
+ }
- EVT MaskEltType = EVT::getIntegerVT(*DAG.getContext(), BitsPerElt);
- EVT MaskType = EVT::getVectorVT(*DAG.getContext(), MaskEltType, IndexLen);
EVT MaskContainerVT = getContainerForFixedLengthVector(DAG, MaskType);
SDValue VecMask =
DAG.getBuildVector(MaskType, DL, ArrayRef(TBLMask.data(), IndexLen));
@@ -26171,13 +26194,35 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
DAG.getConstant(Intrinsic::aarch64_sve_tbl, DL, MVT::i32),
Op1, SVEMask);
- else if (Subtarget.hasSVE2())
- Shuffle =
- DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
- DAG.getConstant(Intrinsic::aarch64_sve_tbl2, DL, MVT::i32),
- Op1, Op2, SVEMask);
- else
- llvm_unreachable("Cannot lower shuffle without SVE2 TBL");
+ else if (Subtarget.hasSVE2()) {
+ if (!MinMaxEqual) {
+ SDValue VScale = DAG.getVScale(DL, MVT::i32, APInt(32, 1));
+ SDValue Mul =
+ DAG.getNode(ISD::MUL, DL, MVT::i32,
+ DAG.getConstant(128 / BitsPerElt, DL, MVT::i32), VScale);
+ SDValue VecMask =
+ DAG.getBuildVector(MaskType, DL, ArrayRef(TBLMask.data(), IndexLen));
+ SDValue MulMask = DAG.getBuildVector(
+ MaskType, DL, ArrayRef(MaskNormalized.data(), IndexLen));
+ SDValue SplatPred = DAG.getNode(ISD::SPLAT_VECTOR, DL, MaskType, Mul);
+ SDValue MulMaskNormalized =
+ DAG.getNode(ISD::MUL, DL, MaskType, SplatPred, MulMask);
+ SDValue UpdatedVecMask =
+ DAG.getNode(ISD::ADD, DL, MaskType, VecMask, MulMaskNormalized);
+ EVT MaskContainerVT = getContainerForFixedLengthVector(DAG, MaskType);
+ SDValue SVEMask =
+ convertToScalableVector(DAG, MaskContainerVT, UpdatedVecMask);
+ Shuffle = DAG.getNode(
+ ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
+ DAG.getConstant(Intrinsic::aarch64_sve_tbl2, DL, MVT::i32), Op1, Op2,
+ SVEMask);
+ } else {
+ Shuffle = DAG.getNode(
+ ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
+ DAG.getConstant(Intrinsic::aarch64_sve_tbl2, DL, MVT::i32), Op1, Op2,
+ SVEMask);
+ }
+ }
Shuffle = convertFromScalableVector(DAG, VT, Shuffle);
return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Shuffle);
}
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
index f646319ba5fccb..3beccdf278ebd1 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mattr=+sve2 -force-streaming-compatible-sve -aarch64-sve-vector-bits-min=128 -aarch64-sve-vector-bits-max=128 < %s | FileCheck %s -check-prefixes=CHECK,SVE2_128
; RUN: llc -mattr=+sve2 -force-streaming-compatible-sve -aarch64-sve-vector-bits-min=128 < %s | FileCheck %s -check-prefixes=CHECK,SVE2_128_NOMAX
+; RUN: llc -mattr=+sve2 -force-streaming-compatible-sve < %s | FileCheck %s -check-prefixes=CHECK,SVE2_NOMIN_NOMAX
+; RUN: llc -mattr=+sve2 -force-streaming-compatible-sve -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,SVE2_MIN_256_NOMAX
target triple = "aarch64-unknown-linux-gnu"
@@ -16,14 +18,43 @@ target triple = "aarch64-unknown-linux-gnu"
; SVE2_128-NEXT: .byte 255 // 0xff
; SVE2_128-NEXT: .byte 255 // 0xff
define <8 x i8> @shuffle_index_indices_from_op1(ptr %a, ptr %b) {
-; CHECK-LABEL: shuffle_index_indices_from_op1:
-; CHECK: // %bb.0:
-; CHECK-NEXT: adrp x8, .LCPI0_0
-; CHECK-NEXT: ldr d0, [x0]
-; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI0_0]
-; CHECK-NEXT: tbl z0.b, { z0.b }, z1.b
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
-; CHECK-NEXT: ret
+; SVE2_128-LABEL: shuffle_index_indices_from_op1:
+; SVE2_128: // %bb.0:
+; SVE2_128-NEXT: adrp x8, .LCPI0_0
+; SVE2_128-NEXT: ldr d0, [x0]
+; SVE2_128-NEXT: ldr q1, [x8, :lo12:.LCPI0_0]
+; SVE2_128-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_128-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_128-NEXT: ret
+;
+; SVE2_128_NOMAX-LABEL: shuffle_index_indices_from_op1:
+; SVE2_128_NOMAX: // %bb.0:
+; SVE2_128_NOMAX-NEXT: adrp x8, .LCPI0_0
+; SVE2_128_NOMAX-NEXT: ldr d0, [x0]
+; SVE2_128_NOMAX-NEXT: ldr q1, [x8, :lo12:.LCPI0_0]
+; SVE2_128_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_128_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_128_NOMAX-NEXT: ret
+;
+; SVE2_NOMIN_NOMAX-LABEL: shuffle_index_indices_from_op1:
+; SVE2_NOMIN_NOMAX: // %bb.0:
+; SVE2_NOMIN_NOMAX-NEXT: adrp x8, .LCPI0_0
+; SVE2_NOMIN_NOMAX-NEXT: ldr d0, [x0]
+; SVE2_NOMIN_NOMAX-NEXT: ldr q1, [x8, :lo12:.LCPI0_0]
+; SVE2_NOMIN_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_NOMIN_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_NOMIN_NOMAX-NEXT: ret
+;
+; SVE2_MIN_256_NOMAX-LABEL: shuffle_index_indices_from_op1:
+; SVE2_MIN_256_NOMAX: // %bb.0:
+; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.b, vl32
+; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI0_0
+; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI0_0
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x0]
+; SVE2_MIN_256_NOMAX-NEXT: ld1b { z1.b }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_MIN_256_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i8>, ptr %a
%op2 = load <8 x i8>, ptr %b
%1 = shufflevector <8 x i8> %op1, <8 x i8> %op2, <8 x i32> <i32 0, i32 7, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -42,14 +73,43 @@ define <8 x i8> @shuffle_index_indices_from_op1(ptr %a, ptr %b) {
; SVE2_128-NEXT: .byte 255 // 0xff
; SVE2_128-NEXT: .byte 255 // 0xff
define <8 x i8> @shuffle_index_indices_from_op2(ptr %a, ptr %b) {
-; CHECK-LABEL: shuffle_index_indices_from_op2:
-; CHECK: // %bb.0:
-; CHECK-NEXT: adrp x8, .LCPI1_0
-; CHECK-NEXT: ldr d0, [x1]
-; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI1_0]
-; CHECK-NEXT: tbl z0.b, { z0.b }, z1.b
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
-; CHECK-NEXT: ret
+; SVE2_128-LABEL: shuffle_index_indices_from_op2:
+; SVE2_128: // %bb.0:
+; SVE2_128-NEXT: adrp x8, .LCPI1_0
+; SVE2_128-NEXT: ldr d0, [x1]
+; SVE2_128-NEXT: ldr q1, [x8, :lo12:.LCPI1_0]
+; SVE2_128-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_128-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_128-NEXT: ret
+;
+; SVE2_128_NOMAX-LABEL: shuffle_index_indices_from_op2:
+; SVE2_128_NOMAX: // %bb.0:
+; SVE2_128_NOMAX-NEXT: adrp x8, .LCPI1_0
+; SVE2_128_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_128_NOMAX-NEXT: ldr q1, [x8, :lo12:.LCPI1_0]
+; SVE2_128_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_128_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_128_NOMAX-NEXT: ret
+;
+; SVE2_NOMIN_NOMAX-LABEL: shuffle_index_indices_from_op2:
+; SVE2_NOMIN_NOMAX: // %bb.0:
+; SVE2_NOMIN_NOMAX-NEXT: adrp x8, .LCPI1_0
+; SVE2_NOMIN_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_NOMIN_NOMAX-NEXT: ldr q1, [x8, :lo12:.LCPI1_0]
+; SVE2_NOMIN_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_NOMIN_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_NOMIN_NOMAX-NEXT: ret
+;
+; SVE2_MIN_256_NOMAX-LABEL: shuffle_index_indices_from_op2:
+; SVE2_MIN_256_NOMAX: // %bb.0:
+; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.b, vl32
+; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI1_0
+; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI1_0
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: ld1b { z1.b }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_MIN_256_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i8>, ptr %a
%op2 = load <8 x i8>, ptr %b
%1 = shufflevector <8 x i8> %op1, <8 x i8> %op2, <8 x i32> <i32 8, i32 9, i32 9, i32 11, i32 12, i32 15, i32 14, i32 15>
@@ -109,6 +169,70 @@ define <8 x i8> @shuffle_index_indices_from_both_ops(ptr %a, ptr %b) {
; SVE2_128_NOMAX-NEXT: ldr d0, [sp, #8]
; SVE2_128_NOMAX-NEXT: add sp, sp, #16
; SVE2_128_NOMAX-NEXT: ret
+;
+; SVE2_NOMIN_NOMAX-LABEL: shuffle_index_indices_from_both_ops:
+; SVE2_NOMIN_NOMAX: // %bb.0:
+; SVE2_NOMIN_NOMAX-NEXT: sub sp, sp, #16
+; SVE2_NOMIN_NOMAX-NEXT: .cfi_def_cfa_offset 16
+; SVE2_NOMIN_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_NOMIN_NOMAX-NEXT: mov z1.b, z0.b[7]
+; SVE2_NOMIN_NOMAX-NEXT: mov z2.b, z0.b[6]
+; SVE2_NOMIN_NOMAX-NEXT: mov z3.b, z0.b[4]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w8, s1
+; SVE2_NOMIN_NOMAX-NEXT: ldr d1, [x0]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w9, s2
+; SVE2_NOMIN_NOMAX-NEXT: mov z2.b, z0.b[3]
+; SVE2_NOMIN_NOMAX-NEXT: mov z1.b, z1.b[1]
+; SVE2_NOMIN_NOMAX-NEXT: strb w8, [sp, #15]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w8, s3
+; SVE2_NOMIN_NOMAX-NEXT: mov z3.b, z0.b[2]
+; SVE2_NOMIN_NOMAX-NEXT: strb w9, [sp, #14]
+; SVE2_NOMIN_NOMAX-NEXT: mov z0.b, z0.b[1]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w9, s2
+; SVE2_NOMIN_NOMAX-NEXT: strb w8, [sp, #13]
+; SVE2_NOMIN_NOMAX-NEXT: strb w8, [sp, #12]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w8, s3
+; SVE2_NOMIN_NOMAX-NEXT: strb w9, [sp, #11]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w9, s0
+; SVE2_NOMIN_NOMAX-NEXT: strb w8, [sp, #10]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w8, s1
+; SVE2_NOMIN_NOMAX-NEXT: strb w9, [sp, #9]
+; SVE2_NOMIN_NOMAX-NEXT: strb w8, [sp, #8]
+; SVE2_NOMIN_NOMAX-NEXT: ldr d0, [sp, #8]
+; SVE2_NOMIN_NOMAX-NEXT: add sp, sp, #16
+; SVE2_NOMIN_NOMAX-NEXT: ret
+;
+; SVE2_MIN_256_NOMAX-LABEL: shuffle_index_indices_from_both_ops:
+; SVE2_MIN_256_NOMAX: // %bb.0:
+; SVE2_MIN_256_NOMAX-NEXT: sub sp, sp, #16
+; SVE2_MIN_256_NOMAX-NEXT: .cfi_def_cfa_offset 16
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: mov z1.b, z0.b[7]
+; SVE2_MIN_256_NOMAX-NEXT: mov z2.b, z0.b[6]
+; SVE2_MIN_256_NOMAX-NEXT: mov z3.b, z0.b[4]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w8, s1
+; SVE2_MIN_256_NOMAX-NEXT: ldr d1, [x0]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w9, s2
+; SVE2_MIN_256_NOMAX-NEXT: mov z2.b, z0.b[3]
+; SVE2_MIN_256_NOMAX-NEXT: mov z1.b, z1.b[1]
+; SVE2_MIN_256_NOMAX-NEXT: strb w8, [sp, #15]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w8, s3
+; SVE2_MIN_256_NOMAX-NEXT: mov z3.b, z0.b[2]
+; SVE2_MIN_256_NOMAX-NEXT: strb w9, [sp, #14]
+; SVE2_MIN_256_NOMAX-NEXT: mov z0.b, z0.b[1]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w9, s2
+; SVE2_MIN_256_NOMAX-NEXT: strb w8, [sp, #13]
+; SVE2_MIN_256_NOMAX-NEXT: strb w8, [sp, #12]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w8, s3
+; SVE2_MIN_256_NOMAX-NEXT: strb w9, [sp, #11]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w9, s0
+; SVE2_MIN_256_NOMAX-NEXT: strb w8, [sp, #10]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w8, s1
+; SVE2_MIN_256_NOMAX-NEXT: strb w9, [sp, #9]
+; SVE2_MIN_256_NOMAX-NEXT: strb w8, [sp, #8]
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [sp, #8]
+; SVE2_MIN_256_NOMAX-NEXT: add sp, sp, #16
+; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i8>, ptr %a
%op2 = load <8 x i8>, ptr %b
%1 = shufflevector <8 x i8> %op1, <8 x i8> %op2, <8 x i32> <i32 1, i32 9, i32 10, i32 11, i32 12, i32 12, i32 14, i32 15>
@@ -165,6 +289,64 @@ define <8 x i8> @shuffle_index_poison_value(ptr %a, ptr %b) {
; SVE2_128_NOMAX-NEXT: ldr d0, [sp, #8]
; SVE2_128_NOMAX-NEXT: add sp, sp, #16
; SVE2_128_NOMAX-NEXT: ret
+;
+; SVE2_NOMIN_NOMAX-LABEL: shuffle_index_poison_value:
+; SVE2_NOMIN_NOMAX: // %bb.0:
+; SVE2_NOMIN_NOMAX-NEXT: sub sp, sp, #16
+; SVE2_NOMIN_NOMAX-NEXT: .cfi_def_cfa_offset 16
+; SVE2_NOMIN_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_NOMIN_NOMAX-NEXT: ldr d3, [x0]
+; SVE2_NOMIN_NOMAX-NEXT: mov z1.b, z0.b[6]
+; SVE2_NOMIN_NOMAX-NEXT: mov z2.b, z0.b[4]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w8, s1
+; SVE2_NOMIN_NOMAX-NEXT: mov z1.b, z0.b[3]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w9, s2
+; SVE2_NOMIN_NOMAX-NEXT: mov z2.b, z0.b[2]
+; SVE2_NOMIN_NOMAX-NEXT: mov z0.b, z0.b[1]
+; SVE2_NOMIN_NOMAX-NEXT: strb w8, [sp, #14]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w8, s1
+; SVE2_NOMIN_NOMAX-NEXT: mov z1.b, z3.b[1]
+; SVE2_NOMIN_NOMAX-NEXT: strb w9, [sp, #13]
+; SVE2_NOMIN_NOMAX-NEXT: strb w9, [sp, #12]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w9, s2
+; SVE2_NOMIN_NOMAX-NEXT: strb w8, [sp, #11]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w8, s0
+; SVE2_NOMIN_NOMAX-NEXT: strb w9, [sp, #10]
+; SVE2_NOMIN_NOMAX-NEXT: fmov w9, s1
+; SVE2_NOMIN_NOMAX-NEXT: strb w8, [sp, #9]
+; SVE2_NOMIN_NOMAX-NEXT: strb w9, [sp, #8]
+; SVE2_NOMIN_NOMAX-NEXT: ldr d0, [sp, #8]
+; SVE2_NOMIN_NOMAX-NEXT: add sp, sp, #16
+; SVE2_NOMIN_NOMAX-NEXT: ret
+;
+; SVE2_MIN_256_NOMAX-LABEL: shuffle_index_poison_value:
+; SVE2_MIN_256_NOMAX: // %bb.0:
+; SVE2_MIN_256_NOMAX-NEXT: sub sp, sp, #16
+; SVE2_MIN_256_NOMAX-NEXT: .cfi_def_cfa_offset 16
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: ldr d3, [x0]
+; SVE2_MIN_256_NOMAX-NEXT: mov z1.b, z0.b[6]
+; SVE2_MIN_256_NOMAX-NEXT: mov z2.b, z0.b[4]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w8, s1
+; SVE2_MIN_256_NOMAX-NEXT: mov z1.b, z0.b[3]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w9, s2
+; SVE2_MIN_256_NOMAX-NEXT: mov z2.b, z0.b[2]
+; SVE2_MIN_256_NOMAX-NEXT: mov z0.b, z0.b[1]
+; SVE2_MIN_256_NOMAX-NEXT: strb w8, [sp, #14]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w8, s1
+; SVE2_MIN_256_NOMAX-NEXT: mov z1.b, z3.b[1]
+; SVE2_MIN_256_NOMAX-NEXT: strb w9, [sp, #13]
+; SVE2_MIN_256_NOMAX-NEXT: strb w9, [sp, #12]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w9, s2
+; SVE2_MIN_256_NOMAX-NEXT: strb w8, [sp, #11]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w8, s0
+; SVE2_MIN_256_NOMAX-NEXT: strb w9, [sp, #10]
+; SVE2_MIN_256_NOMAX-NEXT: fmov w9, s1
+; SVE2_MIN_256_NOMAX-NEXT: strb w8, [sp, #9]
+; SVE2_MIN_256_NOMAX-NEXT: strb w9, [sp, #8]
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [sp, #8]
+; SVE2_MIN_256_NOMAX-NEXT: add sp, sp, #16
+; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i8>, ptr %a
%op2 = load <8 x i8>, ptr %b
%1 = shufflevector <8 x i8> %op1, <8 x i8> %op2, <8 x i32> <i32 1, i32 9, i32 10, i32 11, i32 12, i32 12, i32 14, i32 poison>
@@ -172,14 +354,43 @@ define <8 x i8> @shuffle_index_poison_value(ptr %a, ptr %b) {
}
define <8 x i8> @shuffle_op1_poison(ptr %a, ptr %b) {
-; CHECK-LABEL: shuffle_op1_poison:
-; CHECK: // %bb.0:
-; CHECK-NEXT: adrp x8, .LCPI4_0
-; CHECK-NEXT: ldr d0, [x1]
-; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI4_0]
-; CHECK-NEXT: tbl z0.b, { z0.b }, z1.b
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
-; CHECK-NEXT: ret
+; SVE2_128-LABEL: shuffle_op1_poison:
+; SVE2_128: // %bb.0:
+; SVE2_128-NEXT: adrp x8, .LCPI4_0
+; SVE2_128-NEXT: ldr d0, [x1]
+; SVE2_128-NEXT: ldr q1, [x8, :lo12:.LCPI4_0]
+; SVE2_128-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_128-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_128-NEXT: ret
+;
+; SVE2_128_NOMAX-LABEL: shuffle_op1_poison:
+; SVE2_128_NOMAX: // %bb.0:
+; SVE2_128_NOMAX-NEXT: adrp x8, .LCPI4_0
+; SVE2_128_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_128_NOMAX-NEXT: ldr q1, [x8, :lo12:.LCPI4_0]
+; SVE2_128_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_128_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_128_NOMAX-NEXT: ret
+;
+; SVE2_NOMIN_NOMAX-LABEL: shuffle_op1_poison:
+; SVE2_NOMIN_NOMAX: // %bb.0:
+; SVE2_NOMIN_NOMAX-NEXT: adrp x8, .LCPI4_0
+; SVE2_NOMIN_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_NOMIN_NOMAX-NEXT: ldr q1, [x8, :lo12:.LCPI4_0]
+; SVE2_NOMIN_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_NOMIN_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_NOMIN_NOMAX-NEXT: ret
+;
+; SVE2_MIN_256_NOMAX-LABEL: shuffle_op1_poison:
+; SVE2_MIN_256_NOMAX: // %bb.0:
+; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.b, vl32
+; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI4_0
+; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI4_0
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: ld1b { z1.b }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_MIN_256_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE2_MIN_256_NOMAX-NEXT: ret
%op2 = load <8 x i8>, ptr %b
%1 = shufflevector <8 x i8> poison, <8 x i8> %op2, <8 x i32> <i32 1, i32 9, i32 10, i32 11, i32 12, i32 12, i32 14, i32 15>
ret <8 x i8> %1
@@ -252,3 +463,157 @@ define <8 x i8> @shuffle_index_size_op1_maxhw(ptr %a, ptr %b) "target-features"=
%1 = shufflevector <8 x i8> %op1, <8 x i8> %op2, <8 x i32> <i32 0, i32 7, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
ret <8 x i8> %1
}
+
+; SVE2_128: .LCPI7_0:
+; SVE2_128-NEXT: .hword 1 // 0x1
+; SVE2_128-NEXT: .hword 9 // 0x9
+; SVE2_128-NEXT: .hword 10 // 0xa
+; SVE2_128-NEXT: .hword 11 // 0xb
+; SVE2_128-NEXT: .hword 12 // 0xc
+; SVE2_128-NEXT: .hword 12 // 0xc
+; SVE2_128-NEXT: .hword 14 // 0xe
+; SVE2_128-NEXT: .hword 15 // 0xf
+
+; SVE2_128_NOMAX: .LCPI7_0:
+; SVE2_128_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT:.LCPI7_1:
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_128_NOMAX-NEXT: .hword 2 // 0x2
+; SVE2_128_NOMAX-NEXT: .hword 3 // 0x3
+; SVE2_128_NOMAX-NEXT: .hword 4 // 0x4
+; SVE2_128_NOMAX-NEXT: .hword 4 // 0x4
+; SVE2_128_NOMAX-NEXT: .hword 6 // 0x6
+; SVE2_128_NOMAX-NEXT: .hword 7 // 0x7
+
+; SVE2_NOMIN_NOMAX: .LCPI7_0:
+; SVE2_NOMIN_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT:.LCPI7_1:
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_NOMIN_NOMAX-NEXT: .hword 2 // 0x2
+; SVE2_NOMIN_NOMAX-NEXT: .hword 3 // 0x3
+; SVE2_NOMIN_NOMAX-NEXT: .hword 4 // 0x4
+; SVE2_NOMIN_NOMAX-NEXT: .hword 4 // 0x4
+; SVE2_NOMIN_NOMAX-NEXT: .hword 6 // 0x6
+; SVE2_NOMIN_NOMAX-NEXT: .hword 7 // 0x7
+
+; SVE2_MIN_256_NOMAX: .LCPI7_0:
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT: .hword 0 // 0x0
+; SVE2_MIN_256_NOMAX-NEXT:.LCPI7_1:
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 1 // 0x1
+; SVE2_MIN_256_NOMAX-NEXT: .hword 2 // 0x2
+; SVE2_MIN_256_NOMAX-NEXT: .hword 3 // 0x3
+; SVE2_MIN_256_NOMAX-NEXT: .hword 4 // 0x4
+; SVE2_MIN_256_NOMAX-NEXT: .hword 4 // 0x4
+; SVE2_MIN_256_NOMAX-NEXT: .hword 6 // 0x6
+; SVE2_MIN_256_NOMAX-NEXT: .hword 7 // 0x7
+; SVE2_MIN_256_NOMAX-NEXT: .hword 65535 // 0xffff
+; SVE2_MIN_256_NOMAX-NEXT: .hword 65535 // 0xffff
+; SVE2_MIN_256_NOMAX-NEXT: .hword 65535 // 0xffff
+; SVE2_MIN_256_NOMAX-NEXT: .hword 65535 // 0xffff
+; SVE2_MIN_256_NOMAX-NEXT: .hword 65535 // 0xffff
+; SVE2_MIN_256_NOMAX-NEXT: .hword 65535 // 0xffff
+; SVE2_MIN_256_NOMAX-NEXT: .hword 65535 // 0xffff
+; SVE2_MIN_256_NOMAX-NEXT: .hword 65535 // 0xffff
+define <8 x i16> @shuffle_index_indices_from_both_ops_i16(ptr %a, ptr %b) {
+; SVE2_128-LABEL: shuffle_index_indices_from_both_ops_i16:
+; SVE2_128: // %bb.0:
+; SVE2_128-NEXT: adrp x8, .LCPI7_0
+; SVE2_128-NEXT: ldr q0, [x0]
+; SVE2_128-NEXT: ldr q1, [x1]
+; SVE2_128-NEXT: ldr q2, [x8, :lo12:.LCPI7_0]
+; SVE2_128-NEXT: tbl z0.h, { z0.h, z1.h }, z2.h
+; SVE2_128-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2_128-NEXT: ret
+;
+; SVE2_128_NOMAX-LABEL: shuffle_index_indices_from_both_ops_i16:
+; SVE2_128_NOMAX: // %bb.0:
+; SVE2_128_NOMAX-NEXT: rdvl x8, #1
+; SVE2_128_NOMAX-NEXT: ptrue p0.h, vl8
+; SVE2_128_NOMAX-NEXT: adrp x9, .LCPI7_0
+; SVE2_128_NOMAX-NEXT: lsr x8, x8, #4
+; SVE2_128_NOMAX-NEXT: ldr q1, [x9, :lo12:.LCPI7_0]
+; SVE2_128_NOMAX-NEXT: lsl w8, w8, #3
+; SVE2_128_NOMAX-NEXT: mov z0.h, w8
+; SVE2_128_NOMAX-NEXT: adrp x8, .LCPI7_1
+; SVE2_128_NOMAX-NEXT: ldr q2, [x8, :lo12:.LCPI7_1]
+; SVE2_128_NOMAX-NEXT: mad z0.h, p0/m, z1.h, z2.h
+; SVE2_128_NOMAX-NEXT: ldr q1, [x0]
+; SVE2_128_NOMAX-NEXT: ldr q2, [x1]
+; SVE2_128_NOMAX-NEXT: tbl z0.h, { z1.h, z2.h }, z0.h
+; SVE2_128_NOMAX-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2_128_NOMAX-NEXT: ret
+;
+; SVE2_NOMIN_NOMAX-LABEL: shuffle_index_indices_from_both_ops_i16:
+; SVE2_NOMIN_NOMAX: // %bb.0:
+; SVE2_NOMIN_NOMAX-NEXT: rdvl x8, #1
+; SVE2_NOMIN_NOMAX-NEXT: ptrue p0.h, vl8
+; SVE2_NOMIN_NOMAX-NEXT: adrp x9, .LCPI7_0
+; SVE2_NOMIN_NOMAX-NEXT: lsr x8, x8, #4
+; SVE2_NOMIN_NOMAX-NEXT: ldr q1, [x9, :lo12:.LCPI7_0]
+; SVE2_NOMIN_NOMAX-NEXT: lsl w8, w8, #3
+; SVE2_NOMIN_NOMAX-NEXT: mov z0.h, w8
+; SVE2_NOMIN_NOMAX-NEXT: adrp x8, .LCPI7_1
+; SVE2_NOMIN_NOMAX-NEXT: ldr q2, [x8, :lo12:.LCPI7_1]
+; SVE2_NOMIN_NOMAX-NEXT: mad z0.h, p0/m, z1.h, z2.h
+; SVE2_NOMIN_NOMAX-NEXT: ldr q1, [x0]
+; SVE2_NOMIN_NOMAX-NEXT: ldr q2, [x1]
+; SVE2_NOMIN_NOMAX-NEXT: tbl z0.h, { z1.h, z2.h }, z0.h
+; SVE2_NOMIN_NOMAX-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2_NOMIN_NOMAX-NEXT: ret
+;
+; SVE2_MIN_256_NOMAX-LABEL: shuffle_index_indices_from_both_ops_i16:
+; SVE2_MIN_256_NOMAX: // %bb.0:
+; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.h, vl16
+; SVE2_MIN_256_NOMAX-NEXT: rdvl x8, #1
+; SVE2_MIN_256_NOMAX-NEXT: ldr q0, [x0]
+; SVE2_MIN_256_NOMAX-NEXT: lsr x8, x8, #4
+; SVE2_MIN_256_NOMAX-NEXT: adrp x9, .LCPI7_0
+; SVE2_MIN_256_NOMAX-NEXT: add x9, x9, :lo12:.LCPI7_0
+; SVE2_MIN_256_NOMAX-NEXT: adrp x10, .LCPI7_1
+; SVE2_MIN_256_NOMAX-NEXT: add x10, x10, :lo12:.LCPI7_1
+; SVE2_MIN_256_NOMAX-NEXT: ldr q1, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: lsl w8, w8, #3
+; SVE2_MIN_256_NOMAX-NEXT: mov z4.h, w8
+; SVE2_MIN_256_NOMAX-NEXT: ld1h { z2.h }, p0/z, [x9]
+; SVE2_MIN_256_NOMAX-NEXT: ld1h { z3.h }, p0/z, [x10]
+; SVE2_MIN_256_NOMAX-NEXT: mad z2.h, p0/m, z4.h, z3.h
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.h, { z0.h, z1.h }, z2.h
+; SVE2_MIN_256_NOMAX-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2_MIN_256_NOMAX-NEXT: ret
+ %op1 = load <8 x i16>, ptr %a
+ %op2 = load <8 x i16>, ptr %b
+ %1 = shufflevector <8 x i16> %op1, <8 x i16> %op2, <8 x i32> <i32 1, i32 9, i32 10, i32 11, i32 12, i32 12, i32 14, i32 15>
+ ret <8 x i16> %1
+}
>From 681cd397b02a59485d8176b6185aabfd3e7d9581 Mon Sep 17 00:00:00 2001
From: Dinar Temirbulatov <Dinar.Temirbulatov at arm.com>
Date: Mon, 11 Dec 2023 12:02:30 +0000
Subject: [PATCH 2/7] Resolved issue with 64-bit element types incorrect
lowering.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 14 +++++---
.../sve-fixed-length-vector-shuffle-tbl.ll | 34 +++++++++----------
2 files changed, 27 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 2423ef6f8962a5..960db2ef1bdb7a 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -26142,7 +26142,9 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
// TBL mask element needs adjustment.
SmallVector<SDValue, 8> MaskNormalized;
- if (BitsPerElt == 8 && !MinMaxEqual && !IsSingleOp)
+ // Bail out for 8-bits element types, because with 2048-bit SVE register
+ // size we could not repersent index correctly.
+ if (!IsSingleOp && !MinMaxEqual && BitsPerElt == 8)
return SDValue();
for (int Index : ShuffleMask) {
@@ -26196,10 +26198,14 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
Op1, SVEMask);
else if (Subtarget.hasSVE2()) {
if (!MinMaxEqual) {
- SDValue VScale = DAG.getVScale(DL, MVT::i32, APInt(32, 1));
+ SDValue VScale = (BitsPerElt == 64)
+ ? DAG.getVScale(DL, MVT::i64, APInt(64, 1))
+ : DAG.getVScale(DL, MVT::i32, APInt(32, 1));
SDValue Mul =
- DAG.getNode(ISD::MUL, DL, MVT::i32,
- DAG.getConstant(128 / BitsPerElt, DL, MVT::i32), VScale);
+ DAG.getNode(ISD::MUL, DL, (BitsPerElt == 64) ? MVT::i64 : MVT::i32,
+ DAG.getConstant(128 / BitsPerElt, DL,
+ (BitsPerElt == 64) ? MVT::i64 : MVT::i32),
+ VScale);
SDValue VecMask =
DAG.getBuildVector(MaskType, DL, ArrayRef(TBLMask.data(), IndexLen));
SDValue MulMask = DAG.getBuildVector(
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
index 3beccdf278ebd1..4e4b6e4e2ff822 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
@@ -50,9 +50,9 @@ define <8 x i8> @shuffle_index_indices_from_op1(ptr %a, ptr %b) {
; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.b, vl32
; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI0_0
; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI0_0
-; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x0]
-; SVE2_MIN_256_NOMAX-NEXT: ld1b { z1.b }, p0/z, [x8]
-; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_MIN_256_NOMAX-NEXT: ldr d1, [x0]
+; SVE2_MIN_256_NOMAX-NEXT: ld1b { z0.b }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z1.b }, z0.b
; SVE2_MIN_256_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i8>, ptr %a
@@ -105,9 +105,9 @@ define <8 x i8> @shuffle_index_indices_from_op2(ptr %a, ptr %b) {
; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.b, vl32
; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI1_0
; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI1_0
-; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x1]
-; SVE2_MIN_256_NOMAX-NEXT: ld1b { z1.b }, p0/z, [x8]
-; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_MIN_256_NOMAX-NEXT: ldr d1, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: ld1b { z0.b }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z1.b }, z0.b
; SVE2_MIN_256_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i8>, ptr %a
@@ -386,9 +386,9 @@ define <8 x i8> @shuffle_op1_poison(ptr %a, ptr %b) {
; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.b, vl32
; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI4_0
; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI4_0
-; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x1]
-; SVE2_MIN_256_NOMAX-NEXT: ld1b { z1.b }, p0/z, [x8]
-; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
+; SVE2_MIN_256_NOMAX-NEXT: ldr d1, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: ld1b { z0.b }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z1.b }, z0.b
; SVE2_MIN_256_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE2_MIN_256_NOMAX-NEXT: ret
%op2 = load <8 x i8>, ptr %b
@@ -597,19 +597,19 @@ define <8 x i16> @shuffle_index_indices_from_both_ops_i16(ptr %a, ptr %b) {
; SVE2_MIN_256_NOMAX: // %bb.0:
; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.h, vl16
; SVE2_MIN_256_NOMAX-NEXT: rdvl x8, #1
-; SVE2_MIN_256_NOMAX-NEXT: ldr q0, [x0]
-; SVE2_MIN_256_NOMAX-NEXT: lsr x8, x8, #4
; SVE2_MIN_256_NOMAX-NEXT: adrp x9, .LCPI7_0
; SVE2_MIN_256_NOMAX-NEXT: add x9, x9, :lo12:.LCPI7_0
+; SVE2_MIN_256_NOMAX-NEXT: lsr x8, x8, #4
; SVE2_MIN_256_NOMAX-NEXT: adrp x10, .LCPI7_1
; SVE2_MIN_256_NOMAX-NEXT: add x10, x10, :lo12:.LCPI7_1
-; SVE2_MIN_256_NOMAX-NEXT: ldr q1, [x1]
; SVE2_MIN_256_NOMAX-NEXT: lsl w8, w8, #3
-; SVE2_MIN_256_NOMAX-NEXT: mov z4.h, w8
-; SVE2_MIN_256_NOMAX-NEXT: ld1h { z2.h }, p0/z, [x9]
-; SVE2_MIN_256_NOMAX-NEXT: ld1h { z3.h }, p0/z, [x10]
-; SVE2_MIN_256_NOMAX-NEXT: mad z2.h, p0/m, z4.h, z3.h
-; SVE2_MIN_256_NOMAX-NEXT: tbl z0.h, { z0.h, z1.h }, z2.h
+; SVE2_MIN_256_NOMAX-NEXT: mov z2.h, w8
+; SVE2_MIN_256_NOMAX-NEXT: ld1h { z0.h }, p0/z, [x9]
+; SVE2_MIN_256_NOMAX-NEXT: ld1h { z1.h }, p0/z, [x10]
+; SVE2_MIN_256_NOMAX-NEXT: mad z0.h, p0/m, z2.h, z1.h
+; SVE2_MIN_256_NOMAX-NEXT: ldr q1, [x0]
+; SVE2_MIN_256_NOMAX-NEXT: ldr q2, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.h, { z1.h, z2.h }, z0.h
; SVE2_MIN_256_NOMAX-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i16>, ptr %a
>From 8e1ad6454043fb051dfe5a87a3fc16e1be9459ab Mon Sep 17 00:00:00 2001
From: Dinar Temirbulatov <Dinar.Temirbulatov at arm.com>
Date: Thu, 14 Dec 2023 13:40:07 +0000
Subject: [PATCH 3/7] Corrected comment.
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 960db2ef1bdb7a..05c58174c117f1 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -26142,7 +26142,7 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
// TBL mask element needs adjustment.
SmallVector<SDValue, 8> MaskNormalized;
- // Bail out for 8-bits element types, because with 2048-bit SVE register
+ // Avoid if 8-bits element types, since with 2048-bit SVE register
// size we could not repersent index correctly.
if (!IsSingleOp && !MinMaxEqual && BitsPerElt == 8)
return SDValue();
>From a18006c48050c5654be9c921238a6f240893ed77 Mon Sep 17 00:00:00 2001
From: Dinar Temirbulatov <Dinar.Temirbulatov at arm.com>
Date: Thu, 14 Dec 2023 14:30:52 +0000
Subject: [PATCH 4/7] Corrected test.
---
.../CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
index 4e4b6e4e2ff822..43735e0cbe0b3c 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
@@ -453,9 +453,9 @@ define <8 x i8> @shuffle_index_size_op1_maxhw(ptr %a, ptr %b) "target-features"=
; CHECK-NEXT: ptrue p0.b
; CHECK-NEXT: adrp x8, .LCPI6_0
; CHECK-NEXT: add x8, x8, :lo12:.LCPI6_0
-; CHECK-NEXT: ldr d0, [x0]
-; CHECK-NEXT: ld1b { z1.b }, p0/z, [x8]
-; CHECK-NEXT: tbl z0.b, { z0.b }, z1.b
+; CHECK-NEXT: ldr d1, [x0]
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x8]
+; CHECK-NEXT: tbl z0.b, { z1.b }, z0.b
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT: ret
%op1 = load <8 x i8>, ptr %a
>From 92e8e709019312a38bb37440d48bf8a0a5f41a7b Mon Sep 17 00:00:00 2001
From: Dinar Temirbulatov <Dinar.Temirbulatov at arm.com>
Date: Thu, 14 Dec 2023 15:37:14 +0000
Subject: [PATCH 5/7] Corrected test.
---
.../sve-fixed-length-vector-shuffle-tbl.ll | 40 +++++++++----------
1 file changed, 20 insertions(+), 20 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
index 43735e0cbe0b3c..3beccdf278ebd1 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
@@ -50,9 +50,9 @@ define <8 x i8> @shuffle_index_indices_from_op1(ptr %a, ptr %b) {
; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.b, vl32
; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI0_0
; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI0_0
-; SVE2_MIN_256_NOMAX-NEXT: ldr d1, [x0]
-; SVE2_MIN_256_NOMAX-NEXT: ld1b { z0.b }, p0/z, [x8]
-; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z1.b }, z0.b
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x0]
+; SVE2_MIN_256_NOMAX-NEXT: ld1b { z1.b }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
; SVE2_MIN_256_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i8>, ptr %a
@@ -105,9 +105,9 @@ define <8 x i8> @shuffle_index_indices_from_op2(ptr %a, ptr %b) {
; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.b, vl32
; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI1_0
; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI1_0
-; SVE2_MIN_256_NOMAX-NEXT: ldr d1, [x1]
-; SVE2_MIN_256_NOMAX-NEXT: ld1b { z0.b }, p0/z, [x8]
-; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z1.b }, z0.b
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: ld1b { z1.b }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
; SVE2_MIN_256_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i8>, ptr %a
@@ -386,9 +386,9 @@ define <8 x i8> @shuffle_op1_poison(ptr %a, ptr %b) {
; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.b, vl32
; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI4_0
; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI4_0
-; SVE2_MIN_256_NOMAX-NEXT: ldr d1, [x1]
-; SVE2_MIN_256_NOMAX-NEXT: ld1b { z0.b }, p0/z, [x8]
-; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z1.b }, z0.b
+; SVE2_MIN_256_NOMAX-NEXT: ldr d0, [x1]
+; SVE2_MIN_256_NOMAX-NEXT: ld1b { z1.b }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.b, { z0.b }, z1.b
; SVE2_MIN_256_NOMAX-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE2_MIN_256_NOMAX-NEXT: ret
%op2 = load <8 x i8>, ptr %b
@@ -453,9 +453,9 @@ define <8 x i8> @shuffle_index_size_op1_maxhw(ptr %a, ptr %b) "target-features"=
; CHECK-NEXT: ptrue p0.b
; CHECK-NEXT: adrp x8, .LCPI6_0
; CHECK-NEXT: add x8, x8, :lo12:.LCPI6_0
-; CHECK-NEXT: ldr d1, [x0]
-; CHECK-NEXT: ld1b { z0.b }, p0/z, [x8]
-; CHECK-NEXT: tbl z0.b, { z1.b }, z0.b
+; CHECK-NEXT: ldr d0, [x0]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x8]
+; CHECK-NEXT: tbl z0.b, { z0.b }, z1.b
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT: ret
%op1 = load <8 x i8>, ptr %a
@@ -597,19 +597,19 @@ define <8 x i16> @shuffle_index_indices_from_both_ops_i16(ptr %a, ptr %b) {
; SVE2_MIN_256_NOMAX: // %bb.0:
; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.h, vl16
; SVE2_MIN_256_NOMAX-NEXT: rdvl x8, #1
+; SVE2_MIN_256_NOMAX-NEXT: ldr q0, [x0]
+; SVE2_MIN_256_NOMAX-NEXT: lsr x8, x8, #4
; SVE2_MIN_256_NOMAX-NEXT: adrp x9, .LCPI7_0
; SVE2_MIN_256_NOMAX-NEXT: add x9, x9, :lo12:.LCPI7_0
-; SVE2_MIN_256_NOMAX-NEXT: lsr x8, x8, #4
; SVE2_MIN_256_NOMAX-NEXT: adrp x10, .LCPI7_1
; SVE2_MIN_256_NOMAX-NEXT: add x10, x10, :lo12:.LCPI7_1
+; SVE2_MIN_256_NOMAX-NEXT: ldr q1, [x1]
; SVE2_MIN_256_NOMAX-NEXT: lsl w8, w8, #3
-; SVE2_MIN_256_NOMAX-NEXT: mov z2.h, w8
-; SVE2_MIN_256_NOMAX-NEXT: ld1h { z0.h }, p0/z, [x9]
-; SVE2_MIN_256_NOMAX-NEXT: ld1h { z1.h }, p0/z, [x10]
-; SVE2_MIN_256_NOMAX-NEXT: mad z0.h, p0/m, z2.h, z1.h
-; SVE2_MIN_256_NOMAX-NEXT: ldr q1, [x0]
-; SVE2_MIN_256_NOMAX-NEXT: ldr q2, [x1]
-; SVE2_MIN_256_NOMAX-NEXT: tbl z0.h, { z1.h, z2.h }, z0.h
+; SVE2_MIN_256_NOMAX-NEXT: mov z4.h, w8
+; SVE2_MIN_256_NOMAX-NEXT: ld1h { z2.h }, p0/z, [x9]
+; SVE2_MIN_256_NOMAX-NEXT: ld1h { z3.h }, p0/z, [x10]
+; SVE2_MIN_256_NOMAX-NEXT: mad z2.h, p0/m, z4.h, z3.h
+; SVE2_MIN_256_NOMAX-NEXT: tbl z0.h, { z0.h, z1.h }, z2.h
; SVE2_MIN_256_NOMAX-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE2_MIN_256_NOMAX-NEXT: ret
%op1 = load <8 x i16>, ptr %a
>From 3f6091348810091f746e339f50108ea576546a83 Mon Sep 17 00:00:00 2001
From: Dinar Temirbulatov <Dinar.Temirbulatov at arm.com>
Date: Thu, 11 Jan 2024 01:36:19 +0000
Subject: [PATCH 6/7] Resolved remarks.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 58 ++++++++-----------
.../sve-fixed-length-vector-shuffle-tbl.ll | 38 +++++-------
2 files changed, 40 insertions(+), 56 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 05c58174c117f1..867c34b525ea3e 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -26131,7 +26131,7 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
unsigned IndexLen = MinSVESize / BitsPerElt;
unsigned ElementsPerVectorReg = VTOp1.getVectorNumElements();
uint64_t MaxOffset = APInt(BitsPerElt, -1, false).getZExtValue();
- EVT MaskEltType = EVT::getIntegerVT(*DAG.getContext(), BitsPerElt);
+ EVT MaskEltType = VTOp1.getVectorElementType().changeTypeToInteger();
EVT MaskType = EVT::getVectorVT(*DAG.getContext(), MaskEltType, IndexLen);
bool MinMaxEqual = (MinSVESize == MaxSVESize);
assert(ElementsPerVectorReg <= IndexLen && ShuffleMask.size() <= IndexLen &&
@@ -26140,10 +26140,10 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
SmallVector<SDValue, 8> TBLMask;
// If MinSVESize is not equal to MaxSVESize then we need to know which
// TBL mask element needs adjustment.
- SmallVector<SDValue, 8> MaskNormalized;
+ SmallVector<SDValue, 8> MulByVLMask;
- // Avoid if 8-bits element types, since with 2048-bit SVE register
- // size we could not repersent index correctly.
+ // Bail out for 8-bits element types, because with 2048-bit SVE register
+ // size 8 bits is only sufficient to index into the first source vector.
if (!IsSingleOp && !MinMaxEqual && BitsPerElt == 8)
return SDValue();
@@ -26154,18 +26154,18 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
// If we refer to the second operand then we have to add elements
// number in hardware register minus number of elements in a type in
// case if MinSVESize equals to MaxSVESize, otherwise just add normalized
- // value and record this element in MaskNormalized to be adjusted in the
+ // value and record this element in MulByVLMask to be adjusted in the
// runtime.
if ((unsigned)Index >= ElementsPerVectorReg) {
if (!MinMaxEqual) {
Index = Index - ElementsPerVectorReg;
- MaskNormalized.push_back(DAG.getConstant(1, DL, MVT::i64));
+ MulByVLMask.push_back(DAG.getConstant(1, DL, MVT::i64));
} else {
Index += IndexLen - ElementsPerVectorReg;
}
} else {
if (!MinMaxEqual)
- MaskNormalized.push_back(DAG.getConstant(0, DL, MVT::i64));
+ MulByVLMask.push_back(DAG.getConstant(0, DL, MVT::i64));
}
// For 8-bit elements and 1024-bit SVE registers and MaxOffset equals
// to 255, this might point to the last element of in the second operand
@@ -26182,7 +26182,7 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
for (unsigned i = 0; i < IndexLen - ElementsPerVectorReg; ++i) {
TBLMask.push_back(DAG.getConstant((int)MaxOffset, DL, MVT::i64));
if (!MinMaxEqual)
- MaskNormalized.push_back(DAG.getConstant(0, DL, MVT::i64));
+ MulByVLMask.push_back(DAG.getConstant(0, DL, MVT::i64));
}
EVT MaskContainerVT = getContainerForFixedLengthVector(DAG, MaskType);
@@ -26198,36 +26198,26 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
Op1, SVEMask);
else if (Subtarget.hasSVE2()) {
if (!MinMaxEqual) {
- SDValue VScale = (BitsPerElt == 64)
- ? DAG.getVScale(DL, MVT::i64, APInt(64, 1))
- : DAG.getVScale(DL, MVT::i32, APInt(32, 1));
- SDValue Mul =
- DAG.getNode(ISD::MUL, DL, (BitsPerElt == 64) ? MVT::i64 : MVT::i32,
- DAG.getConstant(128 / BitsPerElt, DL,
- (BitsPerElt == 64) ? MVT::i64 : MVT::i32),
- VScale);
+ SDValue VScale =
+ (BitsPerElt == 64)
+ ? DAG.getVScale(DL, MVT::i64, APInt(64, 128 / BitsPerElt))
+ : DAG.getVScale(DL, MVT::i32, APInt(32, 128 / BitsPerElt));
SDValue VecMask =
DAG.getBuildVector(MaskType, DL, ArrayRef(TBLMask.data(), IndexLen));
- SDValue MulMask = DAG.getBuildVector(
- MaskType, DL, ArrayRef(MaskNormalized.data(), IndexLen));
- SDValue SplatPred = DAG.getNode(ISD::SPLAT_VECTOR, DL, MaskType, Mul);
- SDValue MulMaskNormalized =
- DAG.getNode(ISD::MUL, DL, MaskType, SplatPred, MulMask);
+ SDValue MulByMask = DAG.getNode(
+ ISD::MUL, DL, MaskType,
+ DAG.getNode(ISD::SPLAT_VECTOR, DL, MaskType, VScale),
+ DAG.getBuildVector(MaskType, DL,
+ ArrayRef(MulByVLMask.data(), IndexLen)));
SDValue UpdatedVecMask =
- DAG.getNode(ISD::ADD, DL, MaskType, VecMask, MulMaskNormalized);
- EVT MaskContainerVT = getContainerForFixedLengthVector(DAG, MaskType);
- SDValue SVEMask =
- convertToScalableVector(DAG, MaskContainerVT, UpdatedVecMask);
- Shuffle = DAG.getNode(
- ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
- DAG.getConstant(Intrinsic::aarch64_sve_tbl2, DL, MVT::i32), Op1, Op2,
- SVEMask);
- } else {
- Shuffle = DAG.getNode(
- ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
- DAG.getConstant(Intrinsic::aarch64_sve_tbl2, DL, MVT::i32), Op1, Op2,
- SVEMask);
+ DAG.getNode(ISD::ADD, DL, MaskType, VecMask, MulByMask);
+ SVEMask = convertToScalableVector(
+ DAG, getContainerForFixedLengthVector(DAG, MaskType), UpdatedVecMask);
}
+ Shuffle =
+ DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
+ DAG.getConstant(Intrinsic::aarch64_sve_tbl2, DL, MVT::i32),
+ Op1, Op2, SVEMask);
}
Shuffle = convertFromScalableVector(DAG, VT, Shuffle);
return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Shuffle);
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
index 3beccdf278ebd1..57b1ab3f9eacff 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll
@@ -559,15 +559,13 @@ define <8 x i16> @shuffle_index_indices_from_both_ops_i16(ptr %a, ptr %b) {
;
; SVE2_128_NOMAX-LABEL: shuffle_index_indices_from_both_ops_i16:
; SVE2_128_NOMAX: // %bb.0:
-; SVE2_128_NOMAX-NEXT: rdvl x8, #1
; SVE2_128_NOMAX-NEXT: ptrue p0.h, vl8
+; SVE2_128_NOMAX-NEXT: cnth x8
; SVE2_128_NOMAX-NEXT: adrp x9, .LCPI7_0
-; SVE2_128_NOMAX-NEXT: lsr x8, x8, #4
-; SVE2_128_NOMAX-NEXT: ldr q1, [x9, :lo12:.LCPI7_0]
-; SVE2_128_NOMAX-NEXT: lsl w8, w8, #3
+; SVE2_128_NOMAX-NEXT: adrp x10, .LCPI7_1
; SVE2_128_NOMAX-NEXT: mov z0.h, w8
-; SVE2_128_NOMAX-NEXT: adrp x8, .LCPI7_1
-; SVE2_128_NOMAX-NEXT: ldr q2, [x8, :lo12:.LCPI7_1]
+; SVE2_128_NOMAX-NEXT: ldr q1, [x9, :lo12:.LCPI7_0]
+; SVE2_128_NOMAX-NEXT: ldr q2, [x10, :lo12:.LCPI7_1]
; SVE2_128_NOMAX-NEXT: mad z0.h, p0/m, z1.h, z2.h
; SVE2_128_NOMAX-NEXT: ldr q1, [x0]
; SVE2_128_NOMAX-NEXT: ldr q2, [x1]
@@ -577,15 +575,13 @@ define <8 x i16> @shuffle_index_indices_from_both_ops_i16(ptr %a, ptr %b) {
;
; SVE2_NOMIN_NOMAX-LABEL: shuffle_index_indices_from_both_ops_i16:
; SVE2_NOMIN_NOMAX: // %bb.0:
-; SVE2_NOMIN_NOMAX-NEXT: rdvl x8, #1
; SVE2_NOMIN_NOMAX-NEXT: ptrue p0.h, vl8
+; SVE2_NOMIN_NOMAX-NEXT: cnth x8
; SVE2_NOMIN_NOMAX-NEXT: adrp x9, .LCPI7_0
-; SVE2_NOMIN_NOMAX-NEXT: lsr x8, x8, #4
-; SVE2_NOMIN_NOMAX-NEXT: ldr q1, [x9, :lo12:.LCPI7_0]
-; SVE2_NOMIN_NOMAX-NEXT: lsl w8, w8, #3
+; SVE2_NOMIN_NOMAX-NEXT: adrp x10, .LCPI7_1
; SVE2_NOMIN_NOMAX-NEXT: mov z0.h, w8
-; SVE2_NOMIN_NOMAX-NEXT: adrp x8, .LCPI7_1
-; SVE2_NOMIN_NOMAX-NEXT: ldr q2, [x8, :lo12:.LCPI7_1]
+; SVE2_NOMIN_NOMAX-NEXT: ldr q1, [x9, :lo12:.LCPI7_0]
+; SVE2_NOMIN_NOMAX-NEXT: ldr q2, [x10, :lo12:.LCPI7_1]
; SVE2_NOMIN_NOMAX-NEXT: mad z0.h, p0/m, z1.h, z2.h
; SVE2_NOMIN_NOMAX-NEXT: ldr q1, [x0]
; SVE2_NOMIN_NOMAX-NEXT: ldr q2, [x1]
@@ -596,18 +592,16 @@ define <8 x i16> @shuffle_index_indices_from_both_ops_i16(ptr %a, ptr %b) {
; SVE2_MIN_256_NOMAX-LABEL: shuffle_index_indices_from_both_ops_i16:
; SVE2_MIN_256_NOMAX: // %bb.0:
; SVE2_MIN_256_NOMAX-NEXT: ptrue p0.h, vl16
-; SVE2_MIN_256_NOMAX-NEXT: rdvl x8, #1
; SVE2_MIN_256_NOMAX-NEXT: ldr q0, [x0]
-; SVE2_MIN_256_NOMAX-NEXT: lsr x8, x8, #4
-; SVE2_MIN_256_NOMAX-NEXT: adrp x9, .LCPI7_0
-; SVE2_MIN_256_NOMAX-NEXT: add x9, x9, :lo12:.LCPI7_0
-; SVE2_MIN_256_NOMAX-NEXT: adrp x10, .LCPI7_1
-; SVE2_MIN_256_NOMAX-NEXT: add x10, x10, :lo12:.LCPI7_1
+; SVE2_MIN_256_NOMAX-NEXT: adrp x8, .LCPI7_0
+; SVE2_MIN_256_NOMAX-NEXT: add x8, x8, :lo12:.LCPI7_0
+; SVE2_MIN_256_NOMAX-NEXT: adrp x9, .LCPI7_1
+; SVE2_MIN_256_NOMAX-NEXT: add x9, x9, :lo12:.LCPI7_1
+; SVE2_MIN_256_NOMAX-NEXT: cnth x10
; SVE2_MIN_256_NOMAX-NEXT: ldr q1, [x1]
-; SVE2_MIN_256_NOMAX-NEXT: lsl w8, w8, #3
-; SVE2_MIN_256_NOMAX-NEXT: mov z4.h, w8
-; SVE2_MIN_256_NOMAX-NEXT: ld1h { z2.h }, p0/z, [x9]
-; SVE2_MIN_256_NOMAX-NEXT: ld1h { z3.h }, p0/z, [x10]
+; SVE2_MIN_256_NOMAX-NEXT: mov z4.h, w10
+; SVE2_MIN_256_NOMAX-NEXT: ld1h { z2.h }, p0/z, [x8]
+; SVE2_MIN_256_NOMAX-NEXT: ld1h { z3.h }, p0/z, [x9]
; SVE2_MIN_256_NOMAX-NEXT: mad z2.h, p0/m, z4.h, z3.h
; SVE2_MIN_256_NOMAX-NEXT: tbl z0.h, { z0.h, z1.h }, z2.h
; SVE2_MIN_256_NOMAX-NEXT: // kill: def $q0 killed $q0 killed $z0
>From 6e47eb2fb6a99ee9ccfbebead141fa9f7c4a880d Mon Sep 17 00:00:00 2001
From: Dinar Temirbulatov <Dinar.Temirbulatov at arm.com>
Date: Wed, 14 Feb 2024 12:45:19 +0000
Subject: [PATCH 7/7] Resolved comments.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 37 +++++++++----------
1 file changed, 17 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 867c34b525ea3e..d30ff6939affee 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -26140,7 +26140,7 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
SmallVector<SDValue, 8> TBLMask;
// If MinSVESize is not equal to MaxSVESize then we need to know which
// TBL mask element needs adjustment.
- SmallVector<SDValue, 8> MulByVLMask;
+ SmallVector<SDValue, 8> AddRuntimeVLMask;
// Bail out for 8-bits element types, because with 2048-bit SVE register
// size 8 bits is only sufficient to index into the first source vector.
@@ -26151,22 +26151,19 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
// Handling poison index value.
if (Index < 0)
Index = 0;
- // If we refer to the second operand then we have to add elements
- // number in hardware register minus number of elements in a type in
- // case if MinSVESize equals to MaxSVESize, otherwise just add normalized
- // value and record this element in MulByVLMask to be adjusted in the
- // runtime.
+ // If the mask refers to elements in the second operand, then we have to
+ // offset the index by the number of elements in a vector. If this is number
+ // is not known at compile-time, we need to maintain a mask with 'VL' values
+ // to add at runtime.
if ((unsigned)Index >= ElementsPerVectorReg) {
- if (!MinMaxEqual) {
- Index = Index - ElementsPerVectorReg;
- MulByVLMask.push_back(DAG.getConstant(1, DL, MVT::i64));
- } else {
+ if (MinMaxEqual) {
Index += IndexLen - ElementsPerVectorReg;
+ } else {
+ Index = Index - ElementsPerVectorReg;
+ AddRuntimeVLMask.push_back(DAG.getConstant(1, DL, MVT::i64));
}
- } else {
- if (!MinMaxEqual)
- MulByVLMask.push_back(DAG.getConstant(0, DL, MVT::i64));
- }
+ } else if (!MinMaxEqual)
+ AddRuntimeVLMask.push_back(DAG.getConstant(0, DL, MVT::i64));
// For 8-bit elements and 1024-bit SVE registers and MaxOffset equals
// to 255, this might point to the last element of in the second operand
// of the shufflevector, thus we are rejecting this transform.
@@ -26182,7 +26179,7 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
for (unsigned i = 0; i < IndexLen - ElementsPerVectorReg; ++i) {
TBLMask.push_back(DAG.getConstant((int)MaxOffset, DL, MVT::i64));
if (!MinMaxEqual)
- MulByVLMask.push_back(DAG.getConstant(0, DL, MVT::i64));
+ AddRuntimeVLMask.push_back(DAG.getConstant(0, DL, MVT::i64));
}
EVT MaskContainerVT = getContainerForFixedLengthVector(DAG, MaskType);
@@ -26198,17 +26195,17 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
Op1, SVEMask);
else if (Subtarget.hasSVE2()) {
if (!MinMaxEqual) {
- SDValue VScale =
- (BitsPerElt == 64)
- ? DAG.getVScale(DL, MVT::i64, APInt(64, 128 / BitsPerElt))
- : DAG.getVScale(DL, MVT::i32, APInt(32, 128 / BitsPerElt));
+ unsigned MinNumElts = AArch64::SVEBitsPerBlock / BitsPerElt;
+ SDValue VScale = (BitsPerElt == 64)
+ ? DAG.getVScale(DL, MVT::i64, APInt(64, MinNumElts))
+ : DAG.getVScale(DL, MVT::i32, APInt(32, MinNumElts));
SDValue VecMask =
DAG.getBuildVector(MaskType, DL, ArrayRef(TBLMask.data(), IndexLen));
SDValue MulByMask = DAG.getNode(
ISD::MUL, DL, MaskType,
DAG.getNode(ISD::SPLAT_VECTOR, DL, MaskType, VScale),
DAG.getBuildVector(MaskType, DL,
- ArrayRef(MulByVLMask.data(), IndexLen)));
+ ArrayRef(AddRuntimeVLMask.data(), IndexLen)));
SDValue UpdatedVecMask =
DAG.getNode(ISD::ADD, DL, MaskType, VecMask, MulByMask);
SVEMask = convertToScalableVector(
More information about the llvm-commits
mailing list