[llvm] [AArch64][SDAG] Generate TBL for shuffles in single block loops. (PR #227593)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 30 00:55:56 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: David Green (davemgreen)
<details>
<summary>Changes</summary>
When lowering shuffles we can either generate multiple zip/trn/ins/etc, or a single tbl with a mask generated from a constant pool. The sequence of 2 instructions will be quicker and smaller for a one-off instruction, the tbl will be quicker if the mask load is loop invarinant and can be loaded once and used many times.
This and several other places are examples of where the lowering should be dependant on whether the current basic block is inside a relatively small loop. (Very large loops act like standard straight-line code).
This patch adds a simple heiristic that makes use of the current BasicBlock being selected, changing the behaviour of shuffle generation if it is a single self-looping block. This keeps the types of block where this would apply constrained and ensures it will only trigger for small loops that can reliably pull the constant pool out of the loop. If this is acceptable there are a few other optimizations that could reuse the same pattern.
---
Patch is 28.35 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/227593.diff
5 Files Affected:
- (modified) llvm/include/llvm/CodeGen/SelectionDAG.h (+2)
- (modified) llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp (+4)
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.cpp (+15-4)
- (modified) llvm/test/CodeGen/AArch64/tbl-loops.ll (+141-78)
- (modified) llvm/test/CodeGen/AArch64/vldn_shuffle.ll (+92-86)
``````````diff
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 764858702b732..61f3100ba38ae 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -2711,6 +2711,8 @@ class SelectionDAG {
LLVM_ABI bool shouldOptForSize() const;
+ LLVM_ABI const BasicBlock *getBasicBlock() const;
+
/// Get the (commutative) identity element for the given opcode, if it exists.
LLVM_ABI SDValue getIdentityElement(unsigned Opcode, const SDLoc &DL, EVT VT,
SDNodeFlags Flags);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index f0b1613e73f7b..3715673542ed2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -1486,6 +1486,10 @@ bool SelectionDAG::shouldOptForSize() const {
return llvm::shouldOptimizeForSize(FLI->MBB->getBasicBlock(), PSI, BFI);
}
+const BasicBlock *SelectionDAG::getBasicBlock() const {
+ return FLI->MBB->getBasicBlock();
+}
+
void SelectionDAG::allnodes_clear() {
assert(&*AllNodes.begin() == &EntryNode);
AllNodes.remove(AllNodes.begin());
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 2ddc7fe934430..3e1eb7a5e59bc 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -16063,6 +16063,14 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
DAG.getNode(AArch64ISD::NVCAST, DL, BSVT, V1)));
}
+ auto preferTBLToMultiInstrShuffle = [&]() {
+ if (DAG.shouldOptForSize())
+ return false;
+ // Look for a simple single block loop.
+ const BasicBlock *Block = DAG.getBasicBlock();
+ return is_contained(successors(Block), Block);
+ };
+
if (((NumElts == 8 && EltSize == 16) || (NumElts == 16 && EltSize == 8) ||
(NumElts == 4 && EltSize == 32)) &&
ShuffleVectorInst::isReverseMask(ShuffleMask, ShuffleMask.size())) {
@@ -16075,9 +16083,11 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
return convertFromScalableVector(DAG, VT, Rev);
}
- SDValue Rev = DAG.getNode(AArch64ISD::REV64, DL, VT, V1);
- return DAG.getNode(AArch64ISD::EXT, DL, VT, Rev, Rev,
- DAG.getConstant(8, DL, MVT::i32));
+ if (!preferTBLToMultiInstrShuffle()) {
+ SDValue Rev = DAG.getNode(AArch64ISD::REV64, DL, VT, V1);
+ return DAG.getNode(AArch64ISD::EXT, DL, VT, Rev, Rev,
+ DAG.getConstant(8, DL, MVT::i32));
+ }
}
// Check for slide-with-zeros pattern before EXT (slide is also valid EXT)
@@ -16196,7 +16206,8 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
// If the shuffle is not directly supported and it has 4 elements, use
// the PerfectShuffle-generated table to synthesize it from other shuffles.
- if (NumElts == 4) {
+ if (NumElts == 4 && (getPerfectShuffleCost(ShuffleMask) == 1 ||
+ !preferTBLToMultiInstrShuffle())) {
SmallVector<ShuffleEntry> Entries;
if (generatePerfectShuffle(ShuffleMask, NumElts, Entries)) {
SmallVector<SDValue> Vals;
diff --git a/llvm/test/CodeGen/AArch64/tbl-loops.ll b/llvm/test/CodeGen/AArch64/tbl-loops.ll
index 4870243e0e4a4..6817123ccb3c4 100644
--- a/llvm/test/CodeGen/AArch64/tbl-loops.ll
+++ b/llvm/test/CodeGen/AArch64/tbl-loops.ll
@@ -544,53 +544,55 @@ define void @loop3(ptr noalias nocapture noundef writeonly %dst, ptr nocapture n
; CHECK-IADISABLED-NEXT: ret
; CHECK-IADISABLED-NEXT: .LBB2_7: // %vector.ph
; CHECK-IADISABLED-NEXT: add x11, x8, #1
-; CHECK-IADISABLED-NEXT: mov w8, #1132396544 // =0x437f0000
-; CHECK-IADISABLED-NEXT: adrp x12, .LCPI2_0
+; CHECK-IADISABLED-NEXT: adrp x8, .LCPI2_0
+; CHECK-IADISABLED-NEXT: adrp x9, .LCPI2_1
; CHECK-IADISABLED-NEXT: and x10, x11, #0x1fffffffc
-; CHECK-IADISABLED-NEXT: dup v0.4s, w8
-; CHECK-IADISABLED-NEXT: ldr q1, [x12, :lo12:.LCPI2_0]
-; CHECK-IADISABLED-NEXT: add x9, x10, x10, lsl #1
+; CHECK-IADISABLED-NEXT: ldr q0, [x8, :lo12:.LCPI2_0]
+; CHECK-IADISABLED-NEXT: mov w8, #1132396544 // =0x437f0000
+; CHECK-IADISABLED-NEXT: add x12, x10, x10, lsl #1
+; CHECK-IADISABLED-NEXT: ldr q1, [x9, :lo12:.LCPI2_1]
+; CHECK-IADISABLED-NEXT: adrp x9, .LCPI2_2
+; CHECK-IADISABLED-NEXT: dup v2.4s, w8
+; CHECK-IADISABLED-NEXT: adrp x13, .LCPI2_3
+; CHECK-IADISABLED-NEXT: ldr q3, [x9, :lo12:.LCPI2_2]
+; CHECK-IADISABLED-NEXT: add x8, x1, x12, lsl #2
+; CHECK-IADISABLED-NEXT: ldr q4, [x13, :lo12:.LCPI2_3]
+; CHECK-IADISABLED-NEXT: add x9, x0, x12
; CHECK-IADISABLED-NEXT: and x12, x11, #0x1fffffffc
-; CHECK-IADISABLED-NEXT: add x8, x1, x9, lsl #2
-; CHECK-IADISABLED-NEXT: add x9, x0, x9
; CHECK-IADISABLED-NEXT: .LBB2_8: // %vector.body
; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-IADISABLED-NEXT: ldp q2, q4, [x1]
+; CHECK-IADISABLED-NEXT: ldp q5, q6, [x1]
; CHECK-IADISABLED-NEXT: subs x12, x12, #4
-; CHECK-IADISABLED-NEXT: ldr q6, [x1, #32]
+; CHECK-IADISABLED-NEXT: ldr q7, [x1, #32]
+; CHECK-IADISABLED-NEXT: tbl v16.16b, { v5.16b, v6.16b }, v0.16b
+; CHECK-IADISABLED-NEXT: tbl v17.16b, { v5.16b, v6.16b }, v1.16b
; CHECK-IADISABLED-NEXT: add x1, x1, #48
-; CHECK-IADISABLED-NEXT: mov v3.16b, v2.16b
-; CHECK-IADISABLED-NEXT: rev64 v5.4s, v4.4s
-; CHECK-IADISABLED-NEXT: mov v3.s[1], v2.s[3]
-; CHECK-IADISABLED-NEXT: mov v5.s[0], v2.s[1]
-; CHECK-IADISABLED-NEXT: mov v3.s[2], v4.s[2]
-; CHECK-IADISABLED-NEXT: mov v4.s[0], v2.s[2]
-; CHECK-IADISABLED-NEXT: mov v5.s[3], v6.s[2]
-; CHECK-IADISABLED-NEXT: mov v4.s[2], v6.s[0]
-; CHECK-IADISABLED-NEXT: mov v3.s[3], v6.s[1]
-; CHECK-IADISABLED-NEXT: mov v4.s[3], v6.s[3]
-; CHECK-IADISABLED-NEXT: fcmgt v2.4s, v3.4s, v0.4s
-; CHECK-IADISABLED-NEXT: fcmgt v6.4s, v5.4s, v0.4s
-; CHECK-IADISABLED-NEXT: fcmgt v7.4s, v4.4s, v0.4s
-; CHECK-IADISABLED-NEXT: bsl v2.16b, v0.16b, v3.16b
-; CHECK-IADISABLED-NEXT: fcmlt v3.4s, v3.4s, #0.0
-; CHECK-IADISABLED-NEXT: bsl v6.16b, v0.16b, v5.16b
+; CHECK-IADISABLED-NEXT: mov v16.s[3], v7.s[1]
+; CHECK-IADISABLED-NEXT: mov v17.s[3], v7.s[2]
+; CHECK-IADISABLED-NEXT: mov v6.s[0], v5.s[2]
+; CHECK-IADISABLED-NEXT: tbl v5.16b, { v6.16b, v7.16b }, v3.16b
+; CHECK-IADISABLED-NEXT: fcmgt v6.4s, v16.4s, v2.4s
+; CHECK-IADISABLED-NEXT: fcmgt v7.4s, v17.4s, v2.4s
+; CHECK-IADISABLED-NEXT: fcmgt v18.4s, v5.4s, v2.4s
+; CHECK-IADISABLED-NEXT: bsl v6.16b, v2.16b, v16.16b
+; CHECK-IADISABLED-NEXT: fcmlt v16.4s, v16.4s, #0.0
+; CHECK-IADISABLED-NEXT: bsl v7.16b, v2.16b, v17.16b
+; CHECK-IADISABLED-NEXT: fcmlt v17.4s, v17.4s, #0.0
+; CHECK-IADISABLED-NEXT: bsl v18.16b, v2.16b, v5.16b
; CHECK-IADISABLED-NEXT: fcmlt v5.4s, v5.4s, #0.0
-; CHECK-IADISABLED-NEXT: bsl v7.16b, v0.16b, v4.16b
-; CHECK-IADISABLED-NEXT: fcmlt v4.4s, v4.4s, #0.0
-; CHECK-IADISABLED-NEXT: bic v2.16b, v2.16b, v3.16b
-; CHECK-IADISABLED-NEXT: bic v3.16b, v6.16b, v5.16b
-; CHECK-IADISABLED-NEXT: fcvtzs v2.4s, v2.4s
-; CHECK-IADISABLED-NEXT: bic v4.16b, v7.16b, v4.16b
-; CHECK-IADISABLED-NEXT: fcvtzs v3.4s, v3.4s
-; CHECK-IADISABLED-NEXT: fcvtzs v4.4s, v4.4s
-; CHECK-IADISABLED-NEXT: xtn v5.4h, v2.4s
-; CHECK-IADISABLED-NEXT: xtn v6.4h, v3.4s
-; CHECK-IADISABLED-NEXT: xtn v7.4h, v4.4s
-; CHECK-IADISABLED-NEXT: tbl v2.16b, { v5.16b, v6.16b, v7.16b }, v1.16b
-; CHECK-IADISABLED-NEXT: mov s3, v2.s[2]
-; CHECK-IADISABLED-NEXT: str d2, [x0]
-; CHECK-IADISABLED-NEXT: str s3, [x0, #8]
+; CHECK-IADISABLED-NEXT: bic v6.16b, v6.16b, v16.16b
+; CHECK-IADISABLED-NEXT: bic v7.16b, v7.16b, v17.16b
+; CHECK-IADISABLED-NEXT: fcvtzs v6.4s, v6.4s
+; CHECK-IADISABLED-NEXT: bic v5.16b, v18.16b, v5.16b
+; CHECK-IADISABLED-NEXT: fcvtzs v7.4s, v7.4s
+; CHECK-IADISABLED-NEXT: fcvtzs v5.4s, v5.4s
+; CHECK-IADISABLED-NEXT: xtn v16.4h, v6.4s
+; CHECK-IADISABLED-NEXT: xtn v17.4h, v7.4s
+; CHECK-IADISABLED-NEXT: xtn v18.4h, v5.4s
+; CHECK-IADISABLED-NEXT: tbl v5.16b, { v16.16b, v17.16b, v18.16b }, v4.16b
+; CHECK-IADISABLED-NEXT: mov s6, v5.s[2]
+; CHECK-IADISABLED-NEXT: str d5, [x0]
+; CHECK-IADISABLED-NEXT: str s6, [x0, #8]
; CHECK-IADISABLED-NEXT: add x0, x0, #12
; CHECK-IADISABLED-NEXT: b.ne .LBB2_8
; CHECK-IADISABLED-NEXT: // %bb.9: // %middle.block
@@ -878,56 +880,59 @@ define void @loop4(ptr noalias nocapture noundef writeonly %dst, ptr nocapture n
; CHECK-IADISABLED-NEXT: add x11, x8, #1
; CHECK-IADISABLED-NEXT: mov w8, #1132396544 // =0x437f0000
; CHECK-IADISABLED-NEXT: adrp x12, .LCPI3_0
+; CHECK-IADISABLED-NEXT: adrp x13, .LCPI3_1
+; CHECK-IADISABLED-NEXT: adrp x14, .LCPI3_2
; CHECK-IADISABLED-NEXT: and x10, x11, #0x1fffffffc
; CHECK-IADISABLED-NEXT: dup v0.4s, w8
-; CHECK-IADISABLED-NEXT: ldr q1, [x12, :lo12:.LCPI3_0]
; CHECK-IADISABLED-NEXT: add x8, x1, x10, lsl #4
; CHECK-IADISABLED-NEXT: add x9, x0, x10, lsl #2
+; CHECK-IADISABLED-NEXT: ldr q1, [x12, :lo12:.LCPI3_0]
+; CHECK-IADISABLED-NEXT: ldr q2, [x13, :lo12:.LCPI3_1]
+; CHECK-IADISABLED-NEXT: ldr q3, [x14, :lo12:.LCPI3_2]
; CHECK-IADISABLED-NEXT: and x12, x11, #0x1fffffffc
; CHECK-IADISABLED-NEXT: .LBB3_8: // %vector.body
; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-IADISABLED-NEXT: ldp q3, q2, [x1, #32]
+; CHECK-IADISABLED-NEXT: ldp q4, q5, [x1, #32]
; CHECK-IADISABLED-NEXT: subs x12, x12, #4
-; CHECK-IADISABLED-NEXT: ldp q6, q5, [x1], #64
-; CHECK-IADISABLED-NEXT: zip1 v4.4s, v3.4s, v2.4s
-; CHECK-IADISABLED-NEXT: trn1 v19.4s, v3.4s, v2.4s
-; CHECK-IADISABLED-NEXT: zip2 v2.4s, v3.4s, v2.4s
-; CHECK-IADISABLED-NEXT: uzp2 v7.4s, v6.4s, v5.4s
-; CHECK-IADISABLED-NEXT: zip1 v16.4s, v6.4s, v5.4s
-; CHECK-IADISABLED-NEXT: trn2 v18.4s, v6.4s, v5.4s
-; CHECK-IADISABLED-NEXT: zip2 v5.4s, v6.4s, v5.4s
-; CHECK-IADISABLED-NEXT: ext v17.16b, v3.16b, v4.16b, #8
-; CHECK-IADISABLED-NEXT: uzp2 v3.4s, v7.4s, v6.4s
-; CHECK-IADISABLED-NEXT: mov v18.d[1], v4.d[1]
-; CHECK-IADISABLED-NEXT: mov v5.d[1], v19.d[1]
+; CHECK-IADISABLED-NEXT: ldp q6, q7, [x1], #64
+; CHECK-IADISABLED-NEXT: tbl v17.16b, { v4.16b, v5.16b }, v1.16b
+; CHECK-IADISABLED-NEXT: zip1 v18.4s, v4.4s, v5.4s
+; CHECK-IADISABLED-NEXT: trn1 v20.4s, v4.4s, v5.4s
+; CHECK-IADISABLED-NEXT: zip1 v16.4s, v6.4s, v7.4s
+; CHECK-IADISABLED-NEXT: trn2 v19.4s, v6.4s, v7.4s
+; CHECK-IADISABLED-NEXT: zip2 v21.4s, v6.4s, v7.4s
+; CHECK-IADISABLED-NEXT: zip2 v4.4s, v4.4s, v5.4s
+; CHECK-IADISABLED-NEXT: tbl v5.16b, { v6.16b, v7.16b }, v2.16b
; CHECK-IADISABLED-NEXT: mov v16.d[1], v17.d[1]
-; CHECK-IADISABLED-NEXT: mov v3.d[1], v2.d[1]
-; CHECK-IADISABLED-NEXT: fcmgt v4.4s, v18.4s, v0.4s
-; CHECK-IADISABLED-NEXT: fcmlt v17.4s, v18.4s, #0.0
-; CHECK-IADISABLED-NEXT: fcmgt v6.4s, v5.4s, v0.4s
-; CHECK-IADISABLED-NEXT: fcmgt v2.4s, v16.4s, v0.4s
-; CHECK-IADISABLED-NEXT: fcmgt v7.4s, v3.4s, v0.4s
-; CHECK-IADISABLED-NEXT: bsl v4.16b, v0.16b, v18.16b
-; CHECK-IADISABLED-NEXT: bsl v6.16b, v0.16b, v5.16b
-; CHECK-IADISABLED-NEXT: fcmlt v5.4s, v5.4s, #0.0
-; CHECK-IADISABLED-NEXT: bsl v2.16b, v0.16b, v16.16b
+; CHECK-IADISABLED-NEXT: mov v19.d[1], v18.d[1]
+; CHECK-IADISABLED-NEXT: mov v21.d[1], v20.d[1]
+; CHECK-IADISABLED-NEXT: mov v5.d[1], v4.d[1]
+; CHECK-IADISABLED-NEXT: fcmgt v4.4s, v16.4s, v0.4s
+; CHECK-IADISABLED-NEXT: fcmgt v6.4s, v19.4s, v0.4s
+; CHECK-IADISABLED-NEXT: fcmgt v7.4s, v21.4s, v0.4s
+; CHECK-IADISABLED-NEXT: fcmgt v17.4s, v5.4s, v0.4s
+; CHECK-IADISABLED-NEXT: fcmlt v18.4s, v19.4s, #0.0
+; CHECK-IADISABLED-NEXT: bsl v4.16b, v0.16b, v16.16b
; CHECK-IADISABLED-NEXT: fcmlt v16.4s, v16.4s, #0.0
-; CHECK-IADISABLED-NEXT: bsl v7.16b, v0.16b, v3.16b
-; CHECK-IADISABLED-NEXT: fcmlt v3.4s, v3.4s, #0.0
-; CHECK-IADISABLED-NEXT: bic v4.16b, v4.16b, v17.16b
-; CHECK-IADISABLED-NEXT: bic v5.16b, v6.16b, v5.16b
-; CHECK-IADISABLED-NEXT: bic v2.16b, v2.16b, v16.16b
+; CHECK-IADISABLED-NEXT: bsl v6.16b, v0.16b, v19.16b
+; CHECK-IADISABLED-NEXT: bsl v7.16b, v0.16b, v21.16b
+; CHECK-IADISABLED-NEXT: fcmlt v19.4s, v21.4s, #0.0
+; CHECK-IADISABLED-NEXT: bsl v17.16b, v0.16b, v5.16b
+; CHECK-IADISABLED-NEXT: fcmlt v5.4s, v5.4s, #0.0
+; CHECK-IADISABLED-NEXT: bic v4.16b, v4.16b, v16.16b
+; CHECK-IADISABLED-NEXT: bic v6.16b, v6.16b, v18.16b
+; CHECK-IADISABLED-NEXT: bic v7.16b, v7.16b, v19.16b
+; CHECK-IADISABLED-NEXT: bic v5.16b, v17.16b, v5.16b
; CHECK-IADISABLED-NEXT: fcvtzs v4.4s, v4.4s
-; CHECK-IADISABLED-NEXT: bic v3.16b, v7.16b, v3.16b
+; CHECK-IADISABLED-NEXT: fcvtzs v6.4s, v6.4s
+; CHECK-IADISABLED-NEXT: fcvtzs v7.4s, v7.4s
; CHECK-IADISABLED-NEXT: fcvtzs v5.4s, v5.4s
-; CHECK-IADISABLED-NEXT: fcvtzs v2.4s, v2.4s
-; CHECK-IADISABLED-NEXT: fcvtzs v3.4s, v3.4s
-; CHECK-IADISABLED-NEXT: xtn v16.4h, v2.4s
-; CHECK-IADISABLED-NEXT: xtn v17.4h, v4.4s
-; CHECK-IADISABLED-NEXT: xtn v18.4h, v5.4s
-; CHECK-IADISABLED-NEXT: xtn v19.4h, v3.4s
-; CHECK-IADISABLED-NEXT: tbl v2.16b, { v16.16b, v17.16b, v18.16b, v19.16b }, v1.16b
-; CHECK-IADISABLED-NEXT: str q2, [x0], #16
+; CHECK-IADISABLED-NEXT: xtn v16.4h, v4.4s
+; CHECK-IADISABLED-NEXT: xtn v17.4h, v6.4s
+; CHECK-IADISABLED-NEXT: xtn v18.4h, v7.4s
+; CHECK-IADISABLED-NEXT: xtn v19.4h, v5.4s
+; CHECK-IADISABLED-NEXT: tbl v4.16b, { v16.16b, v17.16b, v18.16b, v19.16b }, v3.16b
+; CHECK-IADISABLED-NEXT: str q4, [x0], #16
; CHECK-IADISABLED-NEXT: b.ne .LBB3_8
; CHECK-IADISABLED-NEXT: // %bb.9: // %middle.block
; CHECK-IADISABLED-NEXT: cmp x11, x10
@@ -1769,3 +1774,61 @@ for.body: ; preds = %for.body.preheader5
%exitcond.not = icmp eq i32 %inc, %width
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
+
+define void @reverse4_shuffle_1024(ptr noalias %out, ptr noalias readonly %in) {
+; CHECK-LABEL: reverse4_shuffle_1024:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: adrp x8, .LCPI7_0
+; CHECK-NEXT: add x9, x0, #32
+; CHECK-NEXT: mov w10, #1024 // =0x400
+; CHECK-NEXT: ldr q0, [x8, :lo12:.LCPI7_0]
+; CHECK-NEXT: add x8, x1, #32
+; CHECK-NEXT: .LBB7_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldp q1, q2, [x8, #-32]
+; CHECK-NEXT: subs x10, x10, #1
+; CHECK-NEXT: ldp q3, q4, [x8], #64
+; CHECK-NEXT: tbl v1.16b, { v1.16b }, v0.16b
+; CHECK-NEXT: tbl v2.16b, { v2.16b }, v0.16b
+; CHECK-NEXT: tbl v3.16b, { v3.16b }, v0.16b
+; CHECK-NEXT: tbl v4.16b, { v4.16b }, v0.16b
+; CHECK-NEXT: stp q1, q2, [x9, #-32]
+; CHECK-NEXT: stp q3, q4, [x9], #64
+; CHECK-NEXT: b.ne .LBB7_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: ret
+entry:
+ br label %loop
+
+loop:
+ %i = phi i64 [ 0, %entry ], [ %next, %loop ]
+ %offset = shl i64 %i, 6
+ %src = getelementptr i8, ptr %in, i64 %offset
+ %dst = getelementptr i8, ptr %out, i64 %offset
+ %p0 = getelementptr i8, ptr %src, i64 0
+ %x0 = load <16 x i8>, ptr %p0, align 1
+ %p1 = getelementptr i8, ptr %src, i64 16
+ %x1 = load <16 x i8>, ptr %p1, align 1
+ %p2 = getelementptr i8, ptr %src, i64 32
+ %x2 = load <16 x i8>, ptr %p2, align 1
+ %p3 = getelementptr i8, ptr %src, i64 48
+ %x3 = load <16 x i8>, ptr %p3, align 1
+ %r0 = shufflevector <16 x i8> %x0, <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+ %r1 = shufflevector <16 x i8> %x1, <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+ %r2 = shufflevector <16 x i8> %x2, <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+ %r3 = shufflevector <16 x i8> %x3, <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+ %q0 = getelementptr i8, ptr %dst, i64 0
+ store <16 x i8> %r0, ptr %q0, align 1
+ %q1 = getelementptr i8, ptr %dst, i64 16
+ store <16 x i8> %r1, ptr %q1, align 1
+ %q2 = getelementptr i8, ptr %dst, i64 32
+ store <16 x i8> %r2, ptr %q2, align 1
+ %q3 = getelementptr i8, ptr %dst, i64 48
+ store <16 x i8> %r3, ptr %q3, align 1
+ %next = add nuw nsw i64 %i, 1
+ %done = icmp eq i64 %next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/vldn_shuffle.ll b/llvm/test/CodeGen/AArch64/vldn_shuffle.ll
index 963c28aeba32b..7b04218a78051 100644
--- a/llvm/test/CodeGen/AArch64/vldn_shuffle.ll
+++ b/llvm/test/CodeGen/AArch64/vldn_shuffle.ll
@@ -83,28 +83,29 @@ define void @vld3(ptr nocapture readonly %pSrc, ptr noalias nocapture %pDst, i32
; CHECK-IADISABLED: .Lfunc_begin1:
; CHECK-IADISABLED-NEXT: .cfi_startproc
; CHECK-IADISABLED-NEXT: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: adrp x8, .LCPI1_0
+; CHECK-IADISABLED-NEXT: adrp x9, .LCPI1_1
+; CHECK-IADISABLED-NEXT: adrp x10, .LCPI1_2
+; CHECK-IADISABLED-NEXT: ldr q0, [x8, :lo12:.LCPI1_0]
+; CHECK-IADISABLED-NEXT: ldr q1, [x9, :lo12:.LCPI1_1]
+; CHECK-IADISABLED-NEXT: ldr q2, [x10, :lo12:.LCPI1_2]
; CHECK-IADISABLED-NEXT: mov x8, xzr
; CHECK-IADISABLED-NEXT: .LBB1_1: // %vector.body
; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-IADISABLED-NEXT: ldp q0, q1, [x0]
-; CHECK-IADISABLED-NEXT: ldr q4, [x0, #32]
-; CHECK-IADISABLED-NEXT: add x0, x0, #48
-; CHECK-IADISABLED-NEXT: fmul v4.4s, v4.4s, v4.4s
-; CHECK-IADISABLED-NEXT: fmul v0.4s, v0.4s, v0.4s
-; CHECK-IADISABLED-NEXT: fmul v1.4s, v1.4s, v1.4s
-; CHECK-IADISABLED-NEXT: mov v2.16b, v0.16b
-; CHECK-IADISABLED-NEXT: rev64 v3.4s, v1.4s
-; CHECK-IADISABLED-NEXT: mov v2.s[1], v0.s[3]
-; CHECK-IADISABLED-NEXT: mov v3.s[0], v0.s[1]
-; CHECK-IADISABLED-NEXT: mov v2.s[2], v1.s[2]
-; CHECK-IADISABLED-NEXT: mov v1.s[0], v0.s[2]
-; CHECK-IADISABLED-NEXT: mov v3.s[3], v4.s[2]
-; CHECK-IADISABLED-NEXT: mov v2.s[3], v4.s[1]
-; CHECK-IADISABLED-NEXT: mov v1.s[2], v4.s[0]
-; CHECK-IADISABLED-NEXT: fadd v0.4s, v3.4s, v2.4s
-; CHECK-IADISABLED-NEXT: mov v1.s[3], v4.s[3]
-; CHECK-IADISABLED-NEXT: fadd v0.4s, v0.4s, v1.4s
-; CHECK-IADISABLED-NEXT: str q0, [x1, x8]
+; CHECK-IADISABLED-NEXT: ldp q4, q3, [x0, #16]
+; CHECK-IADISABLED-NEXT: ldr q5, [x0], #48
+; CHECK-IADISABLED-NEXT: fmul v18.4s, v3.4s, v3.4s
+; CHECK-IADISABLED-NEXT: fmul v17.4s, v4.4s, v4.4s
+; CHECK-IADISABLED-NEXT: fmul v16.4s, v5.4s, v5.4s
+; CHECK-IADISABLED-NEXT: tbl v3.16b, { v16.16b, v17.16b }, v0.16b
+; CHECK-IADISABLED-NEXT: tbl v4.16b, { v16.16b, v17.16b }, v1.16b
+; CHECK-IADISABLED-NEXT: mov v3.s[3], v18.s[1]
+; CHECK-IADISABLED-NEXT: mov v4.s[3], v18.s[2]
+; CHECK-IADISABLED-NEXT: mov v17.s[0], v16.s[2]
+; CHECK-IADISABLED-NEXT: fadd v3.4s, v4.4s, v3.4s
+; CHECK-IADISABLED-NEXT: tbl v4.16b, { v17.16b, v18.16b }, v2.16b
+; CHECK-IADISABLED-NEXT: fadd v3.4s, v3.4s, v4.4s
+; CHECK-IADISABLED-NEXT: str q3, [x1, x8]
; CHECK-IADISABLED-NEXT: add x8, x8, #16
; CHECK-IADISABLED-NEXT: cmp x8, #1, lsl #12 // =4096
; CHECK-IADISABLED-NEXT: b.ne .LBB1_1
@@ -161,37 +162,39 @@ define void @vld4(ptr nocapture readonly %pSrc, ptr noalias nocapture %pDst, i32
; CHECK-IADISABLED: .Lfunc_begin2:
; CHECK-IADISABLED-NEXT: .cfi_startproc
; CHECK-IADISABLED-NEXT: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: adrp x8, .LCPI2_0
+; CHECK-IADISABLED-NEXT: adrp x9, .LCPI2_1
+; CHECK-IADISABLED-NEXT: ldr q0, [x8, :lo12:.LCPI2_0]
+; CHECK-IADISABLED-NEXT: ldr q1, [x9, :lo12:.LCPI2_1]
; CHECK-IADISABLED-NEXT: mov x8, xzr
; CHECK-IADISABLED-NEXT: .LBB2_1: // %vector.body
; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-IADISABLED-NEXT: ldp q0, q1, [x0]
+; CHECK-IADISABLED-NEXT: ldp q5, q2, [x0, #32]
; CHECK-IADISABLED-NEXT: add x9, x1, x8
-; CHECK-IADISABLED-NEXT: ldp q2, q3, [x0, #32]
+; CHECK-I...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/227593
More information about the llvm-commits
mailing list