[llvm] [AArch64] Add SVE shuffle optimization pass (PR #193951)
Graham Hunter via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 10 05:05:14 PDT 2026
https://github.com/huntergr-arm updated https://github.com/llvm/llvm-project/pull/193951
>From 4f263341fdcdc60729311edf70c76017ab32113d Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Thu, 23 Apr 2026 14:05:53 +0000
Subject: [PATCH 1/7] [AArch64] Initial tests for SVE tbl-based deinterleaving
---
.../CodeGen/AArch64/sve-tbl-folding-opts.ll | 619 ++++++++++++++++++
1 file changed, 619 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/sve-tbl-folding-opts.ll
diff --git a/llvm/test/CodeGen/AArch64/sve-tbl-folding-opts.ll b/llvm/test/CodeGen/AArch64/sve-tbl-folding-opts.ll
new file mode 100644
index 0000000000000..9b9fbcc0cc904
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-tbl-folding-opts.ll
@@ -0,0 +1,619 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -O3 -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+define void @zext_nxv8i16_to_nxv8i64_deinterleave_in_loop(ptr %src, ptr %dst, <vscale x 8 x i1> %mask) #0 {
+; CHECK-LABEL: zext_nxv8i16_to_nxv8i64_deinterleave_in_loop:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: mov w8, #2048 // =0x800
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: movi v3.2d, #0000000000000000
+; CHECK-NEXT: cntd x9
+; CHECK-NEXT: rdvl x10, #1
+; CHECK-NEXT: .LBB0_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ld1h { z4.h }, p0/z, [x0]
+; CHECK-NEXT: subs x8, x8, x9
+; CHECK-NEXT: add x0, x0, x10
+; CHECK-NEXT: uunpkhi z5.s, z4.h
+; CHECK-NEXT: uunpklo z4.s, z4.h
+; CHECK-NEXT: uunpkhi z6.d, z5.s
+; CHECK-NEXT: uunpklo z5.d, z5.s
+; CHECK-NEXT: uunpkhi z7.d, z4.s
+; CHECK-NEXT: uunpklo z4.d, z4.s
+; CHECK-NEXT: uzp1 z24.d, z5.d, z6.d
+; CHECK-NEXT: uzp2 z5.d, z5.d, z6.d
+; CHECK-NEXT: uzp1 z25.d, z4.d, z7.d
+; CHECK-NEXT: uzp2 z4.d, z4.d, z7.d
+; CHECK-NEXT: uzp1 z6.d, z25.d, z24.d
+; CHECK-NEXT: uzp2 z7.d, z4.d, z5.d
+; CHECK-NEXT: uzp2 z24.d, z25.d, z24.d
+; CHECK-NEXT: uzp1 z4.d, z4.d, z5.d
+; CHECK-NEXT: and z6.d, z6.d, #0xffff
+; CHECK-NEXT: and z24.d, z24.d, #0xffff
+; CHECK-NEXT: and z7.d, z7.d, #0xffff
+; CHECK-NEXT: and z4.d, z4.d, #0xffff
+; CHECK-NEXT: add z0.d, z0.d, z6.d
+; CHECK-NEXT: add z2.d, z2.d, z24.d
+; CHECK-NEXT: add z3.d, z3.d, z7.d
+; CHECK-NEXT: add z1.d, z1.d, z4.d
+; CHECK-NEXT: b.ne .LBB0_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: str z0, [x1]
+; CHECK-NEXT: str z1, [x1, #1, mul vl]
+; CHECK-NEXT: str z2, [x1, #2, mul vl]
+; CHECK-NEXT: str z3, [x1, #3, mul vl]
+; CHECK-NEXT: ret
+entry:
+ %vscale = tail call i64 @llvm.vscale.i64()
+ %stride = shl nuw nsw i64 %vscale, 1
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc.b.i64 = phi <vscale x 2 x i64> [ splat(i64 0), %entry ], [ %add.b.i64, %loop ]
+ %acc.g.i64 = phi <vscale x 2 x i64> [ splat(i64 0), %entry ], [ %add.g.i64, %loop ]
+ %acc.r.i64 = phi <vscale x 2 x i64> [ splat(i64 0), %entry ], [ %add.r.i64, %loop ]
+ %acc.a.i64 = phi <vscale x 2 x i64> [ splat(i64 0), %entry ], [ %add.a.i64, %loop ]
+ %src.gep = getelementptr inbounds nuw [4 x i16], ptr %src, i64 %iv
+ %bgra = call <vscale x 8 x i16> @llvm.masked.load(ptr %src.gep, <vscale x 8 x i1> %mask, <vscale x 8 x i16> zeroinitializer)
+ %deinterleave = tail call { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } @llvm.vector.deinterleave4(<vscale x 8 x i16> %bgra)
+ %b.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 0
+ %g.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 1
+ %r.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 2
+ %a.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 3
+ %b.i64 = zext <vscale x 2 x i16> %b.i16 to <vscale x 2 x i64>
+ %g.i64 = zext <vscale x 2 x i16> %g.i16 to <vscale x 2 x i64>
+ %r.i64 = zext <vscale x 2 x i16> %r.i16 to <vscale x 2 x i64>
+ %a.i64 = zext <vscale x 2 x i16> %a.i16 to <vscale x 2 x i64>
+ %add.b.i64 = add <vscale x 2 x i64> %acc.b.i64, %b.i64
+ %add.g.i64 = add <vscale x 2 x i64> %acc.g.i64, %g.i64
+ %add.r.i64 = add <vscale x 2 x i64> %acc.r.i64, %r.i64
+ %add.a.i64 = add <vscale x 2 x i64> %acc.a.i64, %a.i64
+ %iv.next = add nuw i64 %iv, %stride
+ %ec = icmp eq i64 %iv.next, 2048
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ store <vscale x 2 x i64> %add.b.i64, ptr %dst
+ %g.i64.gep = getelementptr <vscale x 2 x i64>, ptr %dst, i64 1
+ store <vscale x 2 x i64> %add.g.i64, ptr %g.i64.gep
+ %r.i64.gep = getelementptr <vscale x 2 x i64>, ptr %dst, i64 2
+ store <vscale x 2 x i64> %add.r.i64, ptr %r.i64.gep
+ %a.i64.gep = getelementptr <vscale x 2 x i64>, ptr %dst, i64 3
+ store <vscale x 2 x i64> %add.a.i64, ptr %a.i64.gep
+ ret void
+}
+
+;; TODO: Do we want to perform the sext equivalent? Requires a splat of the
+;; sign bits into another register (using asr) and a more complex tbl
+;; mask to choose; more instructions, but may still be worthwhile if
+;; we find cases in real code.
+define void @sext_nxv8i16_to_nxv8i64_deinterleave_in_loop(ptr %src, ptr %dst, <vscale x 8 x i1> %mask) #0 {
+; CHECK-LABEL: sext_nxv8i16_to_nxv8i64_deinterleave_in_loop:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: mov w8, #2048 // =0x800
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: movi v3.2d, #0000000000000000
+; CHECK-NEXT: cntd x9
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: rdvl x10, #1
+; CHECK-NEXT: .LBB1_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ld1h { z4.h }, p0/z, [x0]
+; CHECK-NEXT: subs x8, x8, x9
+; CHECK-NEXT: add x0, x0, x10
+; CHECK-NEXT: uunpkhi z5.s, z4.h
+; CHECK-NEXT: uunpklo z4.s, z4.h
+; CHECK-NEXT: uunpkhi z6.d, z5.s
+; CHECK-NEXT: uunpklo z5.d, z5.s
+; CHECK-NEXT: uunpkhi z7.d, z4.s
+; CHECK-NEXT: uunpklo z4.d, z4.s
+; CHECK-NEXT: uzp1 z24.d, z5.d, z6.d
+; CHECK-NEXT: uzp2 z5.d, z5.d, z6.d
+; CHECK-NEXT: uzp1 z25.d, z4.d, z7.d
+; CHECK-NEXT: uzp2 z4.d, z4.d, z7.d
+; CHECK-NEXT: uzp1 z6.d, z25.d, z24.d
+; CHECK-NEXT: uzp2 z7.d, z4.d, z5.d
+; CHECK-NEXT: uzp2 z24.d, z25.d, z24.d
+; CHECK-NEXT: uzp1 z4.d, z4.d, z5.d
+; CHECK-NEXT: sxth z6.d, p1/m, z6.d
+; CHECK-NEXT: sxth z24.d, p1/m, z24.d
+; CHECK-NEXT: sxth z7.d, p1/m, z7.d
+; CHECK-NEXT: sxth z4.d, p1/m, z4.d
+; CHECK-NEXT: add z0.d, z0.d, z6.d
+; CHECK-NEXT: add z2.d, z2.d, z24.d
+; CHECK-NEXT: add z3.d, z3.d, z7.d
+; CHECK-NEXT: add z1.d, z1.d, z4.d
+; CHECK-NEXT: b.ne .LBB1_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: str z0, [x1]
+; CHECK-NEXT: str z1, [x1, #1, mul vl]
+; CHECK-NEXT: str z2, [x1, #2, mul vl]
+; CHECK-NEXT: str z3, [x1, #3, mul vl]
+; CHECK-NEXT: ret
+entry:
+ %vscale = tail call i64 @llvm.vscale.i64()
+ %stride = shl nuw nsw i64 %vscale, 1
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc.b.i64 = phi <vscale x 2 x i64> [ splat(i64 0), %entry ], [ %add.b.i64, %loop ]
+ %acc.g.i64 = phi <vscale x 2 x i64> [ splat(i64 0), %entry ], [ %add.g.i64, %loop ]
+ %acc.r.i64 = phi <vscale x 2 x i64> [ splat(i64 0), %entry ], [ %add.r.i64, %loop ]
+ %acc.a.i64 = phi <vscale x 2 x i64> [ splat(i64 0), %entry ], [ %add.a.i64, %loop ]
+ %src.gep = getelementptr inbounds nuw [4 x i16], ptr %src, i64 %iv
+ %bgra = call <vscale x 8 x i16> @llvm.masked.load(ptr %src.gep, <vscale x 8 x i1> %mask, <vscale x 8 x i16> zeroinitializer)
+ %deinterleave = tail call { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } @llvm.vector.deinterleave4(<vscale x 8 x i16> %bgra)
+ %b.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 0
+ %g.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 1
+ %r.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 2
+ %a.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 3
+ %b.i64 = sext <vscale x 2 x i16> %b.i16 to <vscale x 2 x i64>
+ %g.i64 = sext <vscale x 2 x i16> %g.i16 to <vscale x 2 x i64>
+ %r.i64 = sext <vscale x 2 x i16> %r.i16 to <vscale x 2 x i64>
+ %a.i64 = sext <vscale x 2 x i16> %a.i16 to <vscale x 2 x i64>
+ %add.b.i64 = add <vscale x 2 x i64> %acc.b.i64, %b.i64
+ %add.g.i64 = add <vscale x 2 x i64> %acc.g.i64, %g.i64
+ %add.r.i64 = add <vscale x 2 x i64> %acc.r.i64, %r.i64
+ %add.a.i64 = add <vscale x 2 x i64> %acc.a.i64, %a.i64
+ %iv.next = add nuw i64 %iv, %stride
+ %ec = icmp eq i64 %iv.next, 2048
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ store <vscale x 2 x i64> %add.b.i64, ptr %dst
+ %g.i64.gep = getelementptr <vscale x 2 x i64>, ptr %dst, i64 1
+ store <vscale x 2 x i64> %add.g.i64, ptr %g.i64.gep
+ %r.i64.gep = getelementptr <vscale x 2 x i64>, ptr %dst, i64 2
+ store <vscale x 2 x i64> %add.r.i64, ptr %r.i64.gep
+ %a.i64.gep = getelementptr <vscale x 2 x i64>, ptr %dst, i64 3
+ store <vscale x 2 x i64> %add.a.i64, ptr %a.i64.gep
+ ret void
+}
+
+;; Check that we reuse tbl masks for the same shuffle type.
+define void @zext_2x_nxv8i16_to_nxv8i64_deinterleave_in_loop(ptr %src, ptr %dst, <vscale x 8 x i1> %mask) #0 {
+; CHECK-LABEL: zext_2x_nxv8i16_to_nxv8i64_deinterleave_in_loop:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: mov w8, #2048 // =0x800
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: movi v3.2d, #0000000000000000
+; CHECK-NEXT: cntw x9
+; CHECK-NEXT: rdvl x10, #2
+; CHECK-NEXT: .LBB2_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ld1h { z4.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z5.h }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT: subs x8, x8, x9
+; CHECK-NEXT: add x0, x0, x10
+; CHECK-NEXT: uunpkhi z6.s, z4.h
+; CHECK-NEXT: uunpklo z4.s, z4.h
+; CHECK-NEXT: uunpkhi z7.s, z5.h
+; CHECK-NEXT: uunpklo z5.s, z5.h
+; CHECK-NEXT: uunpkhi z24.d, z6.s
+; CHECK-NEXT: uunpklo z6.d, z6.s
+; CHECK-NEXT: uunpkhi z25.d, z4.s
+; CHECK-NEXT: uunpklo z4.d, z4.s
+; CHECK-NEXT: uunpkhi z26.d, z7.s
+; CHECK-NEXT: uunpklo z7.d, z7.s
+; CHECK-NEXT: uunpkhi z27.d, z5.s
+; CHECK-NEXT: uunpklo z5.d, z5.s
+; CHECK-NEXT: uzp1 z28.d, z6.d, z24.d
+; CHECK-NEXT: uzp2 z6.d, z6.d, z24.d
+; CHECK-NEXT: uzp1 z29.d, z4.d, z25.d
+; CHECK-NEXT: uzp2 z4.d, z4.d, z25.d
+; CHECK-NEXT: uzp1 z24.d, z7.d, z26.d
+; CHECK-NEXT: uzp1 z25.d, z5.d, z27.d
+; CHECK-NEXT: uzp2 z7.d, z7.d, z26.d
+; CHECK-NEXT: uzp2 z5.d, z5.d, z27.d
+; CHECK-NEXT: uzp1 z30.d, z29.d, z28.d
+; CHECK-NEXT: uzp2 z26.d, z29.d, z28.d
+; CHECK-NEXT: uzp2 z27.d, z4.d, z6.d
+; CHECK-NEXT: uzp1 z4.d, z4.d, z6.d
+; CHECK-NEXT: uzp1 z6.d, z25.d, z24.d
+; CHECK-NEXT: uzp2 z24.d, z25.d, z24.d
+; CHECK-NEXT: uzp2 z25.d, z5.d, z7.d
+; CHECK-NEXT: uzp1 z5.d, z5.d, z7.d
+; CHECK-NEXT: and z30.d, z30.d, #0xffff
+; CHECK-NEXT: and z26.d, z26.d, #0xffff
+; CHECK-NEXT: and z27.d, z27.d, #0xffff
+; CHECK-NEXT: movprfx z7, z4
+; CHECK-NEXT: and z7.d, z7.d, #0xffff
+; CHECK-NEXT: and z24.d, z24.d, #0xffff
+; CHECK-NEXT: movprfx z28, z5
+; CHECK-NEXT: and z28.d, z28.d, #0xffff
+; CHECK-NEXT: and z25.d, z25.d, #0xffff
+; CHECK-NEXT: add z4.d, z0.d, z30.d
+; CHECK-NEXT: movprfx z0, z6
+; CHECK-NEXT: and z0.d, z0.d, #0xffff
+; CHECK-NEXT: add z6.d, z2.d, z26.d
+; CHECK-NEXT: add z5.d, z1.d, z7.d
+; CHECK-NEXT: add z7.d, z3.d, z27.d
+; CHECK-NEXT: add z0.d, z4.d, z0.d
+; CHECK-NEXT: add z2.d, z6.d, z24.d
+; CHECK-NEXT: add z1.d, z5.d, z28.d
+; CHECK-NEXT: add z3.d, z7.d, z25.d
+; CHECK-NEXT: b.ne .LBB2_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: str z4, [x1]
+; CHECK-NEXT: str z5, [x1, #1, mul vl]
+; CHECK-NEXT: str z6, [x1, #2, mul vl]
+; CHECK-NEXT: str z7, [x1, #3, mul vl]
+; CHECK-NEXT: ret
+entry:
+ %vscale = tail call i64 @llvm.vscale.i64()
+ %stride = shl nuw nsw i64 %vscale, 2
+ %midpoint = shl nuw nsw i64 %vscale, 1
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc.b.i64 = phi <vscale x 2 x i64> [ splat(i64 0), %entry ], [ %add.b.i64.2, %loop ]
+ %acc.g.i64 = phi <vscale x 2 x i64> [ splat(i64 0), %entry ], [ %add.g.i64.2, %loop ]
+ %acc.r.i64 = phi <vscale x 2 x i64> [ splat(i64 0), %entry ], [ %add.r.i64.2, %loop ]
+ %acc.a.i64 = phi <vscale x 2 x i64> [ splat(i64 0), %entry ], [ %add.a.i64.2, %loop ]
+ %src.gep = getelementptr inbounds nuw [4 x i16], ptr %src, i64 %iv
+ %bgra = call <vscale x 8 x i16> @llvm.masked.load(ptr %src.gep, <vscale x 8 x i1> %mask, <vscale x 8 x i16> zeroinitializer)
+ %deinterleave = tail call { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } @llvm.vector.deinterleave4(<vscale x 8 x i16> %bgra)
+ %b.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 0
+ %g.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 1
+ %r.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 2
+ %a.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 3
+ %b.i64 = zext <vscale x 2 x i16> %b.i16 to <vscale x 2 x i64>
+ %g.i64 = zext <vscale x 2 x i16> %g.i16 to <vscale x 2 x i64>
+ %r.i64 = zext <vscale x 2 x i16> %r.i16 to <vscale x 2 x i64>
+ %a.i64 = zext <vscale x 2 x i16> %a.i16 to <vscale x 2 x i64>
+ %add.b.i64 = add <vscale x 2 x i64> %acc.b.i64, %b.i64
+ %add.g.i64 = add <vscale x 2 x i64> %acc.g.i64, %g.i64
+ %add.r.i64 = add <vscale x 2 x i64> %acc.r.i64, %r.i64
+ %add.a.i64 = add <vscale x 2 x i64> %acc.a.i64, %a.i64
+ %src.gep.2 = getelementptr inbounds nuw [4 x i16], ptr %src.gep, i64 %midpoint
+ %bgra.2 = call <vscale x 8 x i16> @llvm.masked.load(ptr %src.gep.2, <vscale x 8 x i1> %mask, <vscale x 8 x i16> zeroinitializer)
+ %deinterleave.2 = tail call { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } @llvm.vector.deinterleave4(<vscale x 8 x i16> %bgra.2)
+ %b.i16.2 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave.2, 0
+ %g.i16.2 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave.2, 1
+ %r.i16.2 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave.2, 2
+ %a.i16.2 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave.2, 3
+ %b.i64.2 = zext <vscale x 2 x i16> %b.i16.2 to <vscale x 2 x i64>
+ %g.i64.2 = zext <vscale x 2 x i16> %g.i16.2 to <vscale x 2 x i64>
+ %r.i64.2 = zext <vscale x 2 x i16> %r.i16.2 to <vscale x 2 x i64>
+ %a.i64.2 = zext <vscale x 2 x i16> %a.i16.2 to <vscale x 2 x i64>
+ %add.b.i64.2 = add <vscale x 2 x i64> %add.b.i64, %b.i64.2
+ %add.g.i64.2 = add <vscale x 2 x i64> %add.g.i64, %g.i64.2
+ %add.r.i64.2 = add <vscale x 2 x i64> %add.r.i64, %r.i64.2
+ %add.a.i64.2 = add <vscale x 2 x i64> %add.a.i64, %a.i64.2
+ %iv.next = add nuw i64 %iv, %stride
+ %ec = icmp eq i64 %iv.next, 2048
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ store <vscale x 2 x i64> %add.b.i64, ptr %dst
+ %g.i64.gep = getelementptr <vscale x 2 x i64>, ptr %dst, i64 1
+ store <vscale x 2 x i64> %add.g.i64, ptr %g.i64.gep
+ %r.i64.gep = getelementptr <vscale x 2 x i64>, ptr %dst, i64 2
+ store <vscale x 2 x i64> %add.r.i64, ptr %r.i64.gep
+ %a.i64.gep = getelementptr <vscale x 2 x i64>, ptr %dst, i64 3
+ store <vscale x 2 x i64> %add.a.i64, ptr %a.i64.gep
+ ret void
+}
+
+define void @uitofp_nxv8i16_to_nxv8f64_deinterleave_in_loop(ptr %src, ptr %dst, <vscale x 8 x i1> %mask) #0 {
+; CHECK-LABEL: uitofp_nxv8i16_to_nxv8f64_deinterleave_in_loop:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: mov w8, #2048 // =0x800
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: movi v3.2d, #0000000000000000
+; CHECK-NEXT: cntw x9
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: rdvl x10, #2
+; CHECK-NEXT: .LBB3_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ld1h { z4.h }, p0/z, [x0]
+; CHECK-NEXT: subs x8, x8, x9
+; CHECK-NEXT: add x0, x0, x10
+; CHECK-NEXT: uunpkhi z5.s, z4.h
+; CHECK-NEXT: uunpklo z4.s, z4.h
+; CHECK-NEXT: uunpkhi z6.d, z5.s
+; CHECK-NEXT: uunpklo z5.d, z5.s
+; CHECK-NEXT: uunpkhi z7.d, z4.s
+; CHECK-NEXT: uunpklo z4.d, z4.s
+; CHECK-NEXT: uzp1 z24.d, z5.d, z6.d
+; CHECK-NEXT: uzp2 z5.d, z5.d, z6.d
+; CHECK-NEXT: uzp1 z25.d, z4.d, z7.d
+; CHECK-NEXT: uzp2 z4.d, z4.d, z7.d
+; CHECK-NEXT: uzp1 z6.d, z25.d, z24.d
+; CHECK-NEXT: uzp2 z7.d, z4.d, z5.d
+; CHECK-NEXT: uzp2 z24.d, z25.d, z24.d
+; CHECK-NEXT: uzp1 z4.d, z4.d, z5.d
+; CHECK-NEXT: and z6.d, z6.d, #0xffff
+; CHECK-NEXT: and z7.d, z7.d, #0xffff
+; CHECK-NEXT: and z24.d, z24.d, #0xffff
+; CHECK-NEXT: and z4.d, z4.d, #0xffff
+; CHECK-NEXT: ucvtf z6.d, p1/m, z6.d
+; CHECK-NEXT: ucvtf z24.d, p1/m, z24.d
+; CHECK-NEXT: ucvtf z7.d, p1/m, z7.d
+; CHECK-NEXT: ucvtf z4.d, p1/m, z4.d
+; CHECK-NEXT: fadd z0.d, z0.d, z6.d
+; CHECK-NEXT: fadd z2.d, z2.d, z24.d
+; CHECK-NEXT: fadd z3.d, z3.d, z7.d
+; CHECK-NEXT: fadd z1.d, z1.d, z4.d
+; CHECK-NEXT: b.ne .LBB3_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: str z0, [x1]
+; CHECK-NEXT: str z1, [x1, #1, mul vl]
+; CHECK-NEXT: str z2, [x1, #2, mul vl]
+; CHECK-NEXT: str z3, [x1, #3, mul vl]
+; CHECK-NEXT: ret
+entry:
+ %vscale = tail call i64 @llvm.vscale.i64()
+ %stride = shl nuw nsw i64 %vscale, 2
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc.b.f64 = phi <vscale x 2 x double> [ splat(double 0.000000e+00), %entry ], [ %fadd.b.f64, %loop ]
+ %acc.g.f64 = phi <vscale x 2 x double> [ splat(double 0.000000e+00), %entry ], [ %fadd.g.f64, %loop ]
+ %acc.r.f64 = phi <vscale x 2 x double> [ splat(double 0.000000e+00), %entry ], [ %fadd.r.f64, %loop ]
+ %acc.a.f64 = phi <vscale x 2 x double> [ splat(double 0.000000e+00), %entry ], [ %fadd.a.f64, %loop ]
+ %src.gep = getelementptr inbounds nuw [4 x i16], ptr %src, i64 %iv
+ %bgra = call <vscale x 8 x i16> @llvm.masked.load(ptr %src.gep, <vscale x 8 x i1> %mask, <vscale x 8 x i16> zeroinitializer)
+ %deinterleave = tail call { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } @llvm.vector.deinterleave4(<vscale x 8 x i16> %bgra)
+ %b.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 0
+ %g.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 1
+ %r.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 2
+ %a.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 3
+ %b.f64 = uitofp <vscale x 2 x i16> %b.i16 to <vscale x 2 x double>
+ %g.f64 = uitofp <vscale x 2 x i16> %g.i16 to <vscale x 2 x double>
+ %r.f64 = uitofp <vscale x 2 x i16> %r.i16 to <vscale x 2 x double>
+ %a.f64 = uitofp <vscale x 2 x i16> %a.i16 to <vscale x 2 x double>
+ %fadd.b.f64 = fadd <vscale x 2 x double> %acc.b.f64, %b.f64
+ %fadd.g.f64 = fadd <vscale x 2 x double> %acc.g.f64, %g.f64
+ %fadd.r.f64 = fadd <vscale x 2 x double> %acc.r.f64, %r.f64
+ %fadd.a.f64 = fadd <vscale x 2 x double> %acc.a.f64, %a.f64
+ %iv.next = add nuw i64 %iv, %stride
+ %ec = icmp eq i64 %iv.next, 2048
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ store <vscale x 2 x double> %fadd.b.f64, ptr %dst
+ %g.f64.gep = getelementptr <vscale x 2 x double>, ptr %dst, i64 1
+ store <vscale x 2 x double> %fadd.g.f64, ptr %g.f64.gep
+ %r.f64.gep = getelementptr <vscale x 2 x double>, ptr %dst, i64 2
+ store <vscale x 2 x double> %fadd.r.f64, ptr %r.f64.gep
+ %a.f64.gep = getelementptr <vscale x 2 x double>, ptr %dst, i64 3
+ store <vscale x 2 x double> %fadd.a.f64, ptr %a.f64.gep
+ ret void
+}
+
+define void @zext_nxv16i8_to_nxv16i32_deinterleave_in_loop(ptr %src, ptr %dst, <vscale x 16 x i1> %mask) #0 {
+; CHECK-LABEL: zext_nxv16i8_to_nxv16i32_deinterleave_in_loop:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: mov w8, #2048 // =0x800
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: movi v3.2d, #0000000000000000
+; CHECK-NEXT: cntd x9
+; CHECK-NEXT: cnth x10
+; CHECK-NEXT: .LBB4_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ld1b { z4.b }, p0/z, [x0]
+; CHECK-NEXT: subs x8, x8, x9
+; CHECK-NEXT: add x0, x0, x10
+; CHECK-NEXT: uunpkhi z5.h, z4.b
+; CHECK-NEXT: uunpklo z4.h, z4.b
+; CHECK-NEXT: uunpkhi z6.s, z5.h
+; CHECK-NEXT: uunpklo z5.s, z5.h
+; CHECK-NEXT: uunpkhi z7.s, z4.h
+; CHECK-NEXT: uunpklo z4.s, z4.h
+; CHECK-NEXT: uzp1 z24.s, z5.s, z6.s
+; CHECK-NEXT: uzp2 z5.s, z5.s, z6.s
+; CHECK-NEXT: uzp1 z25.s, z4.s, z7.s
+; CHECK-NEXT: uzp2 z4.s, z4.s, z7.s
+; CHECK-NEXT: uzp1 z6.s, z25.s, z24.s
+; CHECK-NEXT: uzp2 z7.s, z4.s, z5.s
+; CHECK-NEXT: uzp2 z24.s, z25.s, z24.s
+; CHECK-NEXT: uzp1 z4.s, z4.s, z5.s
+; CHECK-NEXT: and z6.s, z6.s, #0xff
+; CHECK-NEXT: and z24.s, z24.s, #0xff
+; CHECK-NEXT: and z7.s, z7.s, #0xff
+; CHECK-NEXT: and z4.s, z4.s, #0xff
+; CHECK-NEXT: add z0.s, z0.s, z6.s
+; CHECK-NEXT: add z2.s, z2.s, z24.s
+; CHECK-NEXT: add z3.s, z3.s, z7.s
+; CHECK-NEXT: add z1.s, z1.s, z4.s
+; CHECK-NEXT: b.ne .LBB4_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: str z0, [x1]
+; CHECK-NEXT: str z1, [x1, #1, mul vl]
+; CHECK-NEXT: str z2, [x1, #2, mul vl]
+; CHECK-NEXT: str z3, [x1, #3, mul vl]
+; CHECK-NEXT: ret
+entry:
+ %vscale = tail call i64 @llvm.vscale.i64()
+ %stride = shl nuw nsw i64 %vscale, 1
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc.b.i64 = phi <vscale x 4 x i32> [ splat(i32 0), %entry ], [ %add.b.i64, %loop ]
+ %acc.g.i64 = phi <vscale x 4 x i32> [ splat(i32 0), %entry ], [ %add.g.i64, %loop ]
+ %acc.r.i64 = phi <vscale x 4 x i32> [ splat(i32 0), %entry ], [ %add.r.i64, %loop ]
+ %acc.a.i64 = phi <vscale x 4 x i32> [ splat(i32 0), %entry ], [ %add.a.i64, %loop ]
+ %src.gep = getelementptr inbounds nuw [4 x i8], ptr %src, i64 %iv
+ %bgra = call <vscale x 16 x i8> @llvm.masked.load(ptr %src.gep, <vscale x 16 x i1> %mask, <vscale x 16 x i8> zeroinitializer)
+ %deinterleave = tail call { <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8> } @llvm.vector.deinterleave4(<vscale x 16 x i8> %bgra)
+ %b.i16 = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8> } %deinterleave, 0
+ %g.i16 = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8> } %deinterleave, 1
+ %r.i16 = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8> } %deinterleave, 2
+ %a.i16 = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8> } %deinterleave, 3
+ %b.i64 = zext <vscale x 4 x i8> %b.i16 to <vscale x 4 x i32>
+ %g.i64 = zext <vscale x 4 x i8> %g.i16 to <vscale x 4 x i32>
+ %r.i64 = zext <vscale x 4 x i8> %r.i16 to <vscale x 4 x i32>
+ %a.i64 = zext <vscale x 4 x i8> %a.i16 to <vscale x 4 x i32>
+ %add.b.i64 = add <vscale x 4 x i32> %acc.b.i64, %b.i64
+ %add.g.i64 = add <vscale x 4 x i32> %acc.g.i64, %g.i64
+ %add.r.i64 = add <vscale x 4 x i32> %acc.r.i64, %r.i64
+ %add.a.i64 = add <vscale x 4 x i32> %acc.a.i64, %a.i64
+ %iv.next = add nuw i64 %iv, %stride
+ %ec = icmp eq i64 %iv.next, 2048
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ store <vscale x 4 x i32> %add.b.i64, ptr %dst
+ %g.i64.gep = getelementptr <vscale x 4 x i32>, ptr %dst, i64 1
+ store <vscale x 4 x i32> %add.g.i64, ptr %g.i64.gep
+ %r.i64.gep = getelementptr <vscale x 4 x i32>, ptr %dst, i64 2
+ store <vscale x 4 x i32> %add.r.i64, ptr %r.i64.gep
+ %a.i64.gep = getelementptr <vscale x 4 x i32>, ptr %dst, i64 3
+ store <vscale x 4 x i32> %add.a.i64, ptr %a.i64.gep
+ ret void
+}
+
+;; TODO: If we know vscale is small enough, then we can do extensions via tbl
+;; even for 8b elements.
+define void @zext_nxv16i8_to_nxv16i32_deinterleave_in_loop_max_vscale_8(ptr %src, ptr %dst, <vscale x 16 x i1> %mask) #1 {
+; CHECK-LABEL: zext_nxv16i8_to_nxv16i32_deinterleave_in_loop_max_vscale_8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: mov x8, xzr
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: movi v3.2d, #0000000000000000
+; CHECK-NEXT: cnth x9
+; CHECK-NEXT: .LBB5_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ld1b { z4.b }, p0/z, [x0, x8]
+; CHECK-NEXT: add x8, x8, x9
+; CHECK-NEXT: cmp x8, #2, lsl #12 // =8192
+; CHECK-NEXT: uunpkhi z5.h, z4.b
+; CHECK-NEXT: uunpklo z4.h, z4.b
+; CHECK-NEXT: uunpkhi z6.s, z5.h
+; CHECK-NEXT: uunpklo z5.s, z5.h
+; CHECK-NEXT: uunpkhi z7.s, z4.h
+; CHECK-NEXT: uunpklo z4.s, z4.h
+; CHECK-NEXT: uzp1 z24.s, z5.s, z6.s
+; CHECK-NEXT: uzp2 z5.s, z5.s, z6.s
+; CHECK-NEXT: uzp1 z25.s, z4.s, z7.s
+; CHECK-NEXT: uzp2 z4.s, z4.s, z7.s
+; CHECK-NEXT: uzp1 z6.s, z25.s, z24.s
+; CHECK-NEXT: uzp2 z7.s, z4.s, z5.s
+; CHECK-NEXT: uzp2 z24.s, z25.s, z24.s
+; CHECK-NEXT: uzp1 z4.s, z4.s, z5.s
+; CHECK-NEXT: and z6.s, z6.s, #0xff
+; CHECK-NEXT: and z24.s, z24.s, #0xff
+; CHECK-NEXT: and z7.s, z7.s, #0xff
+; CHECK-NEXT: and z4.s, z4.s, #0xff
+; CHECK-NEXT: add z0.s, z0.s, z6.s
+; CHECK-NEXT: add z2.s, z2.s, z24.s
+; CHECK-NEXT: add z3.s, z3.s, z7.s
+; CHECK-NEXT: add z1.s, z1.s, z4.s
+; CHECK-NEXT: b.ne .LBB5_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: str z0, [x1]
+; CHECK-NEXT: str z1, [x1, #1, mul vl]
+; CHECK-NEXT: str z2, [x1, #2, mul vl]
+; CHECK-NEXT: str z3, [x1, #3, mul vl]
+; CHECK-NEXT: ret
+entry:
+ %vscale = tail call i64 @llvm.vscale.i64()
+ %stride = shl nuw nsw i64 %vscale, 1
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc.b.i64 = phi <vscale x 4 x i32> [ splat(i32 0), %entry ], [ %add.b.i64, %loop ]
+ %acc.g.i64 = phi <vscale x 4 x i32> [ splat(i32 0), %entry ], [ %add.g.i64, %loop ]
+ %acc.r.i64 = phi <vscale x 4 x i32> [ splat(i32 0), %entry ], [ %add.r.i64, %loop ]
+ %acc.a.i64 = phi <vscale x 4 x i32> [ splat(i32 0), %entry ], [ %add.a.i64, %loop ]
+ %src.gep = getelementptr inbounds nuw [4 x i8], ptr %src, i64 %iv
+ %bgra = call <vscale x 16 x i8> @llvm.masked.load(ptr %src.gep, <vscale x 16 x i1> %mask, <vscale x 16 x i8> zeroinitializer)
+ %deinterleave = tail call { <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8> } @llvm.vector.deinterleave4(<vscale x 16 x i8> %bgra)
+ %b.i16 = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8> } %deinterleave, 0
+ %g.i16 = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8> } %deinterleave, 1
+ %r.i16 = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8> } %deinterleave, 2
+ %a.i16 = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8> } %deinterleave, 3
+ %b.i64 = zext <vscale x 4 x i8> %b.i16 to <vscale x 4 x i32>
+ %g.i64 = zext <vscale x 4 x i8> %g.i16 to <vscale x 4 x i32>
+ %r.i64 = zext <vscale x 4 x i8> %r.i16 to <vscale x 4 x i32>
+ %a.i64 = zext <vscale x 4 x i8> %a.i16 to <vscale x 4 x i32>
+ %add.b.i64 = add <vscale x 4 x i32> %acc.b.i64, %b.i64
+ %add.g.i64 = add <vscale x 4 x i32> %acc.g.i64, %g.i64
+ %add.r.i64 = add <vscale x 4 x i32> %acc.r.i64, %r.i64
+ %add.a.i64 = add <vscale x 4 x i32> %acc.a.i64, %a.i64
+ %iv.next = add nuw i64 %iv, %stride
+ %ec = icmp eq i64 %iv.next, 2048
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ store <vscale x 4 x i32> %add.b.i64, ptr %dst
+ %g.i64.gep = getelementptr <vscale x 4 x i32>, ptr %dst, i64 1
+ store <vscale x 4 x i32> %add.g.i64, ptr %g.i64.gep
+ %r.i64.gep = getelementptr <vscale x 4 x i32>, ptr %dst, i64 2
+ store <vscale x 4 x i32> %add.r.i64, ptr %r.i64.gep
+ %a.i64.gep = getelementptr <vscale x 4 x i32>, ptr %dst, i64 3
+ store <vscale x 4 x i32> %add.a.i64, ptr %a.i64.gep
+ ret void
+}
+
+;; If not in a loop, don't perform the transform, since the setup costs more
+;; than the unpacks.
+define void @zext_nxv8i16_to_nxv8i64_out_of_loop(ptr %src, ptr %dst, <vscale x 8 x i1> %mask) #0 {
+; CHECK-LABEL: zext_nxv8i16_to_nxv8i64_out_of_loop:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: uunpkhi z1.s, z0.h
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: uunpkhi z2.d, z1.s
+; CHECK-NEXT: uunpklo z1.d, z1.s
+; CHECK-NEXT: uunpkhi z3.d, z0.s
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: uzp1 z4.d, z1.d, z2.d
+; CHECK-NEXT: uzp2 z1.d, z1.d, z2.d
+; CHECK-NEXT: uzp1 z5.d, z0.d, z3.d
+; CHECK-NEXT: uzp2 z0.d, z0.d, z3.d
+; CHECK-NEXT: uzp1 z2.d, z5.d, z4.d
+; CHECK-NEXT: uzp1 z3.d, z0.d, z1.d
+; CHECK-NEXT: uzp2 z4.d, z5.d, z4.d
+; CHECK-NEXT: uzp2 z0.d, z0.d, z1.d
+; CHECK-NEXT: and z2.d, z2.d, #0xffff
+; CHECK-NEXT: and z3.d, z3.d, #0xffff
+; CHECK-NEXT: and z4.d, z4.d, #0xffff
+; CHECK-NEXT: and z0.d, z0.d, #0xffff
+; CHECK-NEXT: str z2, [x1]
+; CHECK-NEXT: str z3, [x1, #1, mul vl]
+; CHECK-NEXT: str z4, [x1, #2, mul vl]
+; CHECK-NEXT: str z0, [x1, #3, mul vl]
+; CHECK-NEXT: ret
+entry:
+ %bgra = call <vscale x 8 x i16> @llvm.masked.load(ptr %src, <vscale x 8 x i1> %mask, <vscale x 8 x i16> zeroinitializer)
+ %deinterleave = tail call { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } @llvm.vector.deinterleave4(<vscale x 8 x i16> %bgra)
+ %b.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 0
+ %g.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 1
+ %r.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 2
+ %a.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 3
+ %b.i64 = zext <vscale x 2 x i16> %b.i16 to <vscale x 2 x i64>
+ %g.i64 = zext <vscale x 2 x i16> %g.i16 to <vscale x 2 x i64>
+ %r.i64 = zext <vscale x 2 x i16> %r.i16 to <vscale x 2 x i64>
+ %a.i64 = zext <vscale x 2 x i16> %a.i16 to <vscale x 2 x i64>
+ store <vscale x 2 x i64> %b.i64, ptr %dst
+ %g.i64.gep = getelementptr <vscale x 2 x i64>, ptr %dst, i64 1
+ store <vscale x 2 x i64> %g.i64, ptr %g.i64.gep
+ %r.i64.gep = getelementptr <vscale x 2 x i64>, ptr %dst, i64 2
+ store <vscale x 2 x i64> %r.i64, ptr %r.i64.gep
+ %a.i64.gep = getelementptr <vscale x 2 x i64>, ptr %dst, i64 3
+ store <vscale x 2 x i64> %a.i64, ptr %a.i64.gep
+ ret void
+}
+
+attributes #0 = { "target-features"="+sve" }
+attributes #1 = { "target-features"="+sve" vscale_range(1, 8) }
>From b5bf9bd654197076025d51d5fd2953c74c4583e4 Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Thu, 23 Apr 2026 14:07:07 +0000
Subject: [PATCH 2/7] [AArch64] Add SVE shuffle optimization pass
In some cases, it can be preferable to use tbl instructions in place of
multiple shuffling or extension instructions.
The initial commit is focused on a legal-sized load followed by a 4-way
deinterleave, then a zext or uitofp conversion.
Later on we can introduce shuffles in some cases to allow for the use
of SVE's bottom/top instructions.
---
llvm/lib/Target/AArch64/AArch64.h | 2 +
.../Target/AArch64/AArch64TargetMachine.cpp | 9 +
llvm/lib/Target/AArch64/CMakeLists.txt | 1 +
llvm/lib/Target/AArch64/SVEShuffleOpts.cpp | 300 ++++++++++++++++++
llvm/test/CodeGen/AArch64/O3-pipeline.ll | 2 +
.../CodeGen/AArch64/sve-tbl-folding-opts.ll | 195 ++++++------
6 files changed, 403 insertions(+), 106 deletions(-)
create mode 100644 llvm/lib/Target/AArch64/SVEShuffleOpts.cpp
diff --git a/llvm/lib/Target/AArch64/AArch64.h b/llvm/lib/Target/AArch64/AArch64.h
index 3ef6e693a4076..2a3e8d4294f34 100644
--- a/llvm/lib/Target/AArch64/AArch64.h
+++ b/llvm/lib/Target/AArch64/AArch64.h
@@ -75,6 +75,7 @@ FunctionPass *createSMEPeepholeOptPass();
FunctionPass *createMachineSMEABIPass(CodeGenOptLevel);
FunctionPass *createAArch64SRLTDefineSuperRegsPass();
ModulePass *createSVEIntrinsicOptsPass();
+FunctionPass *createSVEShuffleOptsPass();
InstructionSelector *
createAArch64InstructionSelector(const AArch64TargetMachine &,
const AArch64Subtarget &,
@@ -180,6 +181,7 @@ void initializeSMEPeepholeOptPass(PassRegistry &);
void initializeMachineSMEABIPass(PassRegistry &);
void initializeAArch64SRLTDefineSuperRegsPass(PassRegistry &);
void initializeSVEIntrinsicOptsPass(PassRegistry &);
+void initializeSVEShuffleOptsPass(PassRegistry &);
void initializeAArch64Arm64ECCallLoweringPass(PassRegistry &);
class AArch64StackTaggingPreRAPass
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index b8cc69f7569d6..546c00371911c 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -229,6 +229,12 @@ static cl::opt<bool> EnableSRLTSubregToRegMitigation(
"super-regs when using Subreg Liveness Tracking"),
cl::init(true), cl::Hidden);
+static cl::opt<bool> EnableSVETblOpt(
+ "aarch64-sve-tbl-opt",
+ cl::desc("Enable the use of SVE tbls instructions to replace other kinds"
+ "shuffles, particularly combining them into one operation."),
+ cl::init(true), cl::Hidden);
+
extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
LLVMInitializeAArch64Target() {
// Register the target.
@@ -682,6 +688,9 @@ void AArch64PassConfig::addIRPasses() {
addPass(createAArch64StackTaggingPass(
/*IsOptNone=*/TM->getOptLevel() == CodeGenOptLevel::None));
+ if (getOptLevel() == CodeGenOptLevel::Aggressive && EnableSVETblOpt)
+ addPass(createSVEShuffleOptsPass());
+
// Match complex arithmetic patterns
if (TM->getOptLevel() >= CodeGenOptLevel::Default)
addPass(createComplexDeinterleavingPass(TM));
diff --git a/llvm/lib/Target/AArch64/CMakeLists.txt b/llvm/lib/Target/AArch64/CMakeLists.txt
index 7be0ca38ef4f3..0df2719d67ca0 100644
--- a/llvm/lib/Target/AArch64/CMakeLists.txt
+++ b/llvm/lib/Target/AArch64/CMakeLists.txt
@@ -91,6 +91,7 @@ add_llvm_target(AArch64CodeGen
AArch64TargetTransformInfo.cpp
SMEPeepholeOpt.cpp
SVEIntrinsicOpts.cpp
+ SVEShuffleOpts.cpp
MachineSMEABIPass.cpp
AArch64SRLTDefineSuperRegs.cpp
AArch64SIMDInstrOpt.cpp
diff --git a/llvm/lib/Target/AArch64/SVEShuffleOpts.cpp b/llvm/lib/Target/AArch64/SVEShuffleOpts.cpp
new file mode 100644
index 0000000000000..ab90c6eebb3d8
--- /dev/null
+++ b/llvm/lib/Target/AArch64/SVEShuffleOpts.cpp
@@ -0,0 +1,300 @@
+//===------- SVEShuffleOpts - SVE Shuffle Optimization --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Tries to pattern match and combine scalable vector shuffles that could
+// be more efficiently performed by tbl instructions.
+//
+// An example would be a loop with 4 multiply-accumulate reductions, where the
+// new data in each vector iterations comes from a 4-way deinterleaving of
+// smaller datatypes loaded from memory, which are then extended and multiplied
+// by a common term loaded in reverse order from memory before being added to
+// the accumulator.
+//
+// If the initial load is a legal vector rather than 4x the size (generating a
+// structured ld4 instead), we would see multiple uunpkhi/lo instructions for
+// the extensions, followed by uzp1/2 instructions for the deinterleave, and rev
+// instructions for the common terms. Instead, we can replace all of those with
+// 4 tbl instructions. The tradeoff, of course, is that we now have 4 mask
+// values to maintain which increases register pressure.
+//
+// We should also be able to introduce new shuffles in order to balance out
+// SVE's bottom/top instruction pairs, which act on even/odd lanes instead of
+// the high or low half of a register.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AArch64.h"
+#include "AArch64Subtarget.h"
+#include "AArch64TargetMachine.h"
+#include "Utils/AArch64BaseInfo.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/CodeGen/TargetSubtargetInfo.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IntrinsicsAArch64.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/IR/PatternMatch.h"
+#include "llvm/InitializePasses.h"
+#include <optional>
+
+using namespace llvm;
+using namespace llvm::PatternMatch;
+
+#define DEBUG_TYPE "aarch64-sve-shuffle-opts"
+
+namespace {
+
+class SVEShuffleImpl {
+ const AArch64TargetMachine *TM = nullptr;
+ const LoopInfo *LI = nullptr;
+
+public:
+ SVEShuffleImpl() {};
+ SVEShuffleImpl(const AArch64TargetMachine *TM) : TM(TM) {};
+
+ PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
+ bool runOnFunction(Function &F, Pass &P);
+
+private:
+ bool processLoop(Loop &L);
+};
+
+struct SVEShuffleOpts : public FunctionPass {
+ SVEShuffleImpl Impl;
+ static char ID; // Pass identification, replacement for typeid
+ SVEShuffleOpts() : FunctionPass(ID) {}
+
+ bool runOnFunction(Function &F) override {
+ if (skipFunction(F))
+ return false;
+
+ return Impl.runOnFunction(F, *this);
+ }
+ void getAnalysisUsage(AnalysisUsage &AU) const override;
+
+ StringRef getPassName() const override { return "SVE Tbl Folding Opts"; }
+
+private:
+};
+} // end anonymous namespace
+
+/// A mapping between a vector_deinterleaveN intrinsic and extending cast
+/// instructions used on the resulting subvectors.
+using DeinterleaveMap =
+ SmallDenseMap<CallInst *, SmallVector<std::pair<CastInst *, unsigned>, 4>>;
+
+static void evaluateDeinterleave(IntrinsicInst *I, DeinterleaveMap &Candidates,
+ Loop &L) {
+ // TODO: 'Legalize' if the input is wider than nx128b but not wide enough
+ // to match a structured load?
+ if (I->getOperand(0)
+ ->getType()
+ ->getPrimitiveSizeInBits()
+ .getKnownMinValue() != AArch64::SVEBitsPerBlock)
+ return;
+
+ unsigned IntId = I->getIntrinsicID();
+ assert(IntId == Intrinsic::vector_deinterleave4 &&
+ "Only deinterleave4 supported currently");
+ SmallVector<std::pair<CastInst *, unsigned>, 4> Extends;
+ unsigned Opcode = 0;
+ Type *DestTy = nullptr;
+ for (User *U : I->users()) {
+ auto *Extract = dyn_cast<ExtractValueInst>(U);
+ // We expect only a single cast instruction as a user.
+ if (!Extract || Extract->getNumIndices() != 1)
+ return;
+
+ auto *Extend = dyn_cast<CastInst>(Extract->getUniqueUndroppableUser());
+ if (!Extend || (!isa<ZExtInst>(Extend) && !isa<UIToFPInst>(Extend)))
+ return;
+
+ // We're only interested if the uses are in the loop. This is almost
+ // certainly the case.
+ if (!L.contains(Extract) || !L.contains(Extend))
+ return;
+
+ Opcode = Extend->getOpcode();
+ DestTy = Extend->getDestTy();
+ Type *SrcTy = Extend->getSrcTy();
+
+ // For now, we only want to handle scalable vectors here.
+ if (!DestTy->isScalableTy())
+ return;
+
+ unsigned SrcBits = SrcTy->getScalarSizeInBits();
+ unsigned DestBits = DestTy->getScalarSizeInBits();
+
+ // Looking to match the deinterleave factor.
+ if (DestBits / SrcBits != 4)
+ return;
+
+ // We can't abuse the invalid index trick for tbls of bytes, since the
+ // largest possible SVE vector (2048b) would have 256 bytes, leaving no
+ // way of zeroing.
+ // TODO: If we know vscale is 8 or less, then we could use tbls for bytes.
+ if (SrcBits <= 8)
+ return;
+
+ Extends.push_back({Extend, Extract->getIndices()[0]});
+ }
+
+ // Check that all extracted values are being extended the same way, and that
+ // we have the expected number of extensions.
+ if (Extends.size() != 4 ||
+ !all_of(Extends, [&](std::pair<CastInst *, unsigned> Ext) {
+ CastInst *CI = Ext.first;
+ return CI->getDestTy() == DestTy && CI->getOpcode() == Opcode;
+ }))
+ return;
+
+ Candidates.insert({I, Extends});
+}
+
+// Optimize zext and uitofp from a 4-way deinterleaved load.
+static void optimizeSVEDeinterleavedExtends(DeinterleaveMap Deinterleaves) {
+ // TODO: Cache tbl patterns and reuse, and abandon transforms for a particular
+ // deinterleave if it would introduce too many. We probably want a
+ // hardcoded number of tbls to start with, but if we can estimate
+ // register pressure then we could make better decisions.
+ for (auto &[Deinterleave, Extends] : Deinterleaves) {
+ VectorType *DestTy = cast<VectorType>(Extends[0].first->getDestTy());
+ VectorType *SrcTy = cast<VectorType>(Extends[0].first->getSrcTy());
+ unsigned DstBits = DestTy->getScalarSizeInBits();
+ unsigned SrcBits = SrcTy->getScalarSizeInBits();
+ bool IsUIToFP = isa<UIToFPInst>(Extends[0].first);
+ VectorType *StepVecTy = VectorType::getInteger(DestTy);
+ Type *StepTy = StepVecTy->getScalarType();
+ Value *Input = Deinterleave->getOperand(0);
+ Type *InputTy = Input->getType();
+
+ APInt Invalid = APInt::getAllOnes(DstBits);
+ for (auto &[Extend, Idx] : Extends) {
+ // Build mask
+ APInt StartIdx = Invalid << SrcBits;
+ StartIdx += Idx;
+ IRBuilder<> Builder(Extend);
+ Value *StepVector = Builder.CreateStepVector(StepVecTy);
+ Value *ScaledSteps = Builder.CreateMul(
+ StepVector, Builder.CreateVectorSplat(StepVecTy->getElementCount(),
+ ConstantInt::get(StepTy, 4)));
+ Value *Start = ConstantInt::get(StepTy, StartIdx);
+ Value *ZextTbl = Builder.CreateAdd(
+ ScaledSteps,
+ Builder.CreateVectorSplat(StepVecTy->getElementCount(), Start));
+ Value *FinalMask = Builder.CreateBitCast(ZextTbl, InputTy);
+
+ // Replace the deinterleave, extractvalue, and extension chain with
+ // a tbl directly on the input value.
+ Value *Tbl = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_tbl,
+ {InputTy}, {Input, FinalMask});
+ Value *Widen = Builder.CreateBitCast(Tbl, StepVecTy);
+ if (IsUIToFP)
+ Widen = Builder.CreateUIToFP(Widen, DestTy);
+ LLVM_DEBUG(dbgs() << "SVETBLOPT: Replaced " << *Extend << " with "
+ << *Widen << "\n");
+ Extend->replaceAllUsesWith(Widen);
+ Extend->eraseFromParent();
+ }
+ }
+}
+
+bool SVEShuffleImpl::processLoop(Loop &L) {
+ // TODO: Pull other shuffles into the tbl where possible.
+ // TODO: Add more advanced cases, such as introducing shuffles so that
+ // the SVE odd/even BT narrowing instructions can be used.
+ // TODO: Support other deinterleaves.
+ DeinterleaveMap Candidates;
+ for (auto *BB : L.blocks())
+ for (auto &I : *BB)
+ if (match(&I, m_Intrinsic<Intrinsic::vector_deinterleave4>(m_Value())))
+ evaluateDeinterleave(cast<IntrinsicInst>(&I), Candidates, L);
+
+ if (Candidates.empty())
+ return false;
+
+ optimizeSVEDeinterleavedExtends(Candidates);
+ return true;
+}
+
+void SVEShuffleOpts::getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.addRequired<LoopInfoWrapperPass>();
+ AU.addRequired<TargetPassConfig>();
+ AU.setPreservesCFG();
+}
+
+char SVEShuffleOpts::ID = 0;
+static const char *name = "SVE VLA shuffle optimizations";
+INITIALIZE_PASS_BEGIN(SVEShuffleOpts, DEBUG_TYPE, name, false, false)
+INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
+INITIALIZE_PASS_END(SVEShuffleOpts, DEBUG_TYPE, name, false, false)
+
+FunctionPass *llvm::createSVEShuffleOptsPass() { return new SVEShuffleOpts(); }
+
+namespace llvm {
+class SVEShuffleOptsPass : public PassInfoMixin<SVEShuffleOptsPass> {
+ const AArch64TargetMachine *TM;
+
+public:
+ explicit SVEShuffleOptsPass(const AArch64TargetMachine &TM) : TM(&TM) {}
+ PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM) {
+ SVEShuffleImpl Impl(TM);
+ return Impl.run(F, FAM);
+ }
+};
+} // end namespace llvm
+
+bool SVEShuffleImpl::runOnFunction(Function &F, Pass &P) {
+ // Make sure we can use SVE
+ TargetPassConfig &TPC = P.getAnalysis<TargetPassConfig>();
+ TM = &TPC.getTM<AArch64TargetMachine>();
+ const AArch64Subtarget *ST = TM->getSubtargetImpl(F);
+ if (!ST->isSVEorStreamingSVEAvailable())
+ return false;
+
+ LI = &P.getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
+
+ bool Changed = false;
+ // Only looking to tranform innermost loops, given the increase in
+ // register usage.
+ for (Loop *L : LI->getLoopsInPreorder()) {
+ if (L->isInnermost())
+ Changed |= processLoop(*L);
+ }
+
+ return Changed;
+}
+
+PreservedAnalyses SVEShuffleImpl::run(Function &F,
+ FunctionAnalysisManager &FAM) {
+ const AArch64Subtarget *ST = TM->getSubtargetImpl(F);
+ if (!ST->isSVEorStreamingSVEAvailable())
+ return PreservedAnalyses::all();
+
+ LI = &FAM.getResult<LoopAnalysis>(F);
+
+ bool Changed = false;
+ // Only looking to tranform innermost loops, given the increase in
+ // register usage.
+ for (Loop *L : LI->getLoopsInPreorder()) {
+ if (L->isInnermost())
+ Changed |= processLoop(*L);
+ }
+
+ // Can we do better than 'none'?
+ // We're not actually using the new pass manager though.
+ return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
+}
diff --git a/llvm/test/CodeGen/AArch64/O3-pipeline.ll b/llvm/test/CodeGen/AArch64/O3-pipeline.ll
index 1a0ffe234a236..5990b1664556e 100644
--- a/llvm/test/CodeGen/AArch64/O3-pipeline.ll
+++ b/llvm/test/CodeGen/AArch64/O3-pipeline.ll
@@ -84,6 +84,8 @@
; CHECK-NEXT: Basic Alias Analysis (stateless AA impl)
; CHECK-NEXT: Function Alias Analysis Results
; CHECK-NEXT: AArch64 Stack Tagging
+; CHECK-NEXT: Natural Loop Information
+; CHECK-NEXT: SVE Tbl Folding Opts
; CHECK-NEXT: Complex Deinterleaving Pass
; CHECK-NEXT: Function Alias Analysis Results
; CHECK-NEXT: Memory SSA
diff --git a/llvm/test/CodeGen/AArch64/sve-tbl-folding-opts.ll b/llvm/test/CodeGen/AArch64/sve-tbl-folding-opts.ll
index 9b9fbcc0cc904..e9de3f0b3e4d9 100644
--- a/llvm/test/CodeGen/AArch64/sve-tbl-folding-opts.ll
+++ b/llvm/test/CodeGen/AArch64/sve-tbl-folding-opts.ll
@@ -4,40 +4,36 @@
define void @zext_nxv8i16_to_nxv8i64_deinterleave_in_loop(ptr %src, ptr %dst, <vscale x 8 x i1> %mask) #0 {
; CHECK-LABEL: zext_nxv8i16_to_nxv8i64_deinterleave_in_loop:
; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: index z7.d, #0, #4
+; CHECK-NEXT: mov z4.d, #0xffffffffffff0000
+; CHECK-NEXT: mov z5.d, #0xffffffffffff0001
+; CHECK-NEXT: mov x8, #-65534 // =0xffffffffffff0002
+; CHECK-NEXT: mov z24.d, #0xffffffffffff0003
; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z6.d, x8
; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: mov w8, #2048 // =0x800
; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: movi v3.2d, #0000000000000000
; CHECK-NEXT: cntd x9
+; CHECK-NEXT: add z4.d, z7.d, z4.d
+; CHECK-NEXT: add z5.d, z7.d, z5.d
; CHECK-NEXT: rdvl x10, #1
+; CHECK-NEXT: add z6.d, z7.d, z6.d
+; CHECK-NEXT: add z7.d, z7.d, z24.d
; CHECK-NEXT: .LBB0_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ld1h { z4.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z24.h }, p0/z, [x0]
; CHECK-NEXT: subs x8, x8, x9
; CHECK-NEXT: add x0, x0, x10
-; CHECK-NEXT: uunpkhi z5.s, z4.h
-; CHECK-NEXT: uunpklo z4.s, z4.h
-; CHECK-NEXT: uunpkhi z6.d, z5.s
-; CHECK-NEXT: uunpklo z5.d, z5.s
-; CHECK-NEXT: uunpkhi z7.d, z4.s
-; CHECK-NEXT: uunpklo z4.d, z4.s
-; CHECK-NEXT: uzp1 z24.d, z5.d, z6.d
-; CHECK-NEXT: uzp2 z5.d, z5.d, z6.d
-; CHECK-NEXT: uzp1 z25.d, z4.d, z7.d
-; CHECK-NEXT: uzp2 z4.d, z4.d, z7.d
-; CHECK-NEXT: uzp1 z6.d, z25.d, z24.d
-; CHECK-NEXT: uzp2 z7.d, z4.d, z5.d
-; CHECK-NEXT: uzp2 z24.d, z25.d, z24.d
-; CHECK-NEXT: uzp1 z4.d, z4.d, z5.d
-; CHECK-NEXT: and z6.d, z6.d, #0xffff
-; CHECK-NEXT: and z24.d, z24.d, #0xffff
-; CHECK-NEXT: and z7.d, z7.d, #0xffff
-; CHECK-NEXT: and z4.d, z4.d, #0xffff
-; CHECK-NEXT: add z0.d, z0.d, z6.d
-; CHECK-NEXT: add z2.d, z2.d, z24.d
-; CHECK-NEXT: add z3.d, z3.d, z7.d
-; CHECK-NEXT: add z1.d, z1.d, z4.d
+; CHECK-NEXT: tbl z25.h, { z24.h }, z4.h
+; CHECK-NEXT: tbl z26.h, { z24.h }, z5.h
+; CHECK-NEXT: tbl z27.h, { z24.h }, z6.h
+; CHECK-NEXT: tbl z24.h, { z24.h }, z7.h
+; CHECK-NEXT: add z0.d, z0.d, z25.d
+; CHECK-NEXT: add z1.d, z1.d, z26.d
+; CHECK-NEXT: add z2.d, z2.d, z27.d
+; CHECK-NEXT: add z3.d, z3.d, z24.d
; CHECK-NEXT: b.ne .LBB0_1
; CHECK-NEXT: // %bb.2: // %exit
; CHECK-NEXT: str z0, [x1]
@@ -180,72 +176,63 @@ exit:
define void @zext_2x_nxv8i16_to_nxv8i64_deinterleave_in_loop(ptr %src, ptr %dst, <vscale x 8 x i1> %mask) #0 {
; CHECK-LABEL: zext_2x_nxv8i16_to_nxv8i64_deinterleave_in_loop:
; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-2
+; CHECK-NEXT: str z9, [sp] // 16-byte Folded Spill
+; CHECK-NEXT: str z8, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16
+; CHECK-NEXT: .cfi_escape 0x10, 0x49, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x40, 0x1c // $d9 @ cfa - 16 * VG - 16
+; CHECK-NEXT: index z6.d, #0, #4
+; CHECK-NEXT: mov z1.d, #0xffffffffffff0000
+; CHECK-NEXT: mov z2.d, #0xffffffffffff0001
+; CHECK-NEXT: mov x8, #-65534 // =0xffffffffffff0002
+; CHECK-NEXT: mov z24.d, #0xffffffffffff0003
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: mov w8, #2048 // =0x800
-; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: mov z4.d, x8
; CHECK-NEXT: movi v3.2d, #0000000000000000
+; CHECK-NEXT: mov w8, #2048 // =0x800
+; CHECK-NEXT: movi v5.2d, #0000000000000000
+; CHECK-NEXT: movi v7.2d, #0000000000000000
; CHECK-NEXT: cntw x9
+; CHECK-NEXT: add z1.d, z6.d, z1.d
+; CHECK-NEXT: add z2.d, z6.d, z2.d
; CHECK-NEXT: rdvl x10, #2
+; CHECK-NEXT: add z4.d, z6.d, z4.d
+; CHECK-NEXT: add z6.d, z6.d, z24.d
; CHECK-NEXT: .LBB2_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ld1h { z4.h }, p0/z, [x0]
-; CHECK-NEXT: ld1h { z5.h }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT: ld1h { z24.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z25.h }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: subs x8, x8, x9
; CHECK-NEXT: add x0, x0, x10
-; CHECK-NEXT: uunpkhi z6.s, z4.h
-; CHECK-NEXT: uunpklo z4.s, z4.h
-; CHECK-NEXT: uunpkhi z7.s, z5.h
-; CHECK-NEXT: uunpklo z5.s, z5.h
-; CHECK-NEXT: uunpkhi z24.d, z6.s
-; CHECK-NEXT: uunpklo z6.d, z6.s
-; CHECK-NEXT: uunpkhi z25.d, z4.s
-; CHECK-NEXT: uunpklo z4.d, z4.s
-; CHECK-NEXT: uunpkhi z26.d, z7.s
-; CHECK-NEXT: uunpklo z7.d, z7.s
-; CHECK-NEXT: uunpkhi z27.d, z5.s
-; CHECK-NEXT: uunpklo z5.d, z5.s
-; CHECK-NEXT: uzp1 z28.d, z6.d, z24.d
-; CHECK-NEXT: uzp2 z6.d, z6.d, z24.d
-; CHECK-NEXT: uzp1 z29.d, z4.d, z25.d
-; CHECK-NEXT: uzp2 z4.d, z4.d, z25.d
-; CHECK-NEXT: uzp1 z24.d, z7.d, z26.d
-; CHECK-NEXT: uzp1 z25.d, z5.d, z27.d
-; CHECK-NEXT: uzp2 z7.d, z7.d, z26.d
-; CHECK-NEXT: uzp2 z5.d, z5.d, z27.d
-; CHECK-NEXT: uzp1 z30.d, z29.d, z28.d
-; CHECK-NEXT: uzp2 z26.d, z29.d, z28.d
-; CHECK-NEXT: uzp2 z27.d, z4.d, z6.d
-; CHECK-NEXT: uzp1 z4.d, z4.d, z6.d
-; CHECK-NEXT: uzp1 z6.d, z25.d, z24.d
-; CHECK-NEXT: uzp2 z24.d, z25.d, z24.d
-; CHECK-NEXT: uzp2 z25.d, z5.d, z7.d
-; CHECK-NEXT: uzp1 z5.d, z5.d, z7.d
-; CHECK-NEXT: and z30.d, z30.d, #0xffff
-; CHECK-NEXT: and z26.d, z26.d, #0xffff
-; CHECK-NEXT: and z27.d, z27.d, #0xffff
-; CHECK-NEXT: movprfx z7, z4
-; CHECK-NEXT: and z7.d, z7.d, #0xffff
-; CHECK-NEXT: and z24.d, z24.d, #0xffff
-; CHECK-NEXT: movprfx z28, z5
-; CHECK-NEXT: and z28.d, z28.d, #0xffff
-; CHECK-NEXT: and z25.d, z25.d, #0xffff
-; CHECK-NEXT: add z4.d, z0.d, z30.d
-; CHECK-NEXT: movprfx z0, z6
-; CHECK-NEXT: and z0.d, z0.d, #0xffff
-; CHECK-NEXT: add z6.d, z2.d, z26.d
-; CHECK-NEXT: add z5.d, z1.d, z7.d
-; CHECK-NEXT: add z7.d, z3.d, z27.d
-; CHECK-NEXT: add z0.d, z4.d, z0.d
-; CHECK-NEXT: add z2.d, z6.d, z24.d
-; CHECK-NEXT: add z1.d, z5.d, z28.d
-; CHECK-NEXT: add z3.d, z7.d, z25.d
+; CHECK-NEXT: tbl z26.h, { z24.h }, z1.h
+; CHECK-NEXT: tbl z27.h, { z24.h }, z2.h
+; CHECK-NEXT: tbl z28.h, { z24.h }, z4.h
+; CHECK-NEXT: tbl z29.h, { z24.h }, z6.h
+; CHECK-NEXT: tbl z30.h, { z25.h }, z1.h
+; CHECK-NEXT: tbl z31.h, { z25.h }, z2.h
+; CHECK-NEXT: tbl z8.h, { z25.h }, z4.h
+; CHECK-NEXT: tbl z9.h, { z25.h }, z6.h
+; CHECK-NEXT: add z24.d, z0.d, z26.d
+; CHECK-NEXT: add z25.d, z3.d, z27.d
+; CHECK-NEXT: add z26.d, z5.d, z28.d
+; CHECK-NEXT: add z27.d, z7.d, z29.d
+; CHECK-NEXT: add z0.d, z24.d, z30.d
+; CHECK-NEXT: add z3.d, z25.d, z31.d
+; CHECK-NEXT: add z5.d, z26.d, z8.d
+; CHECK-NEXT: add z7.d, z27.d, z9.d
; CHECK-NEXT: b.ne .LBB2_1
; CHECK-NEXT: // %bb.2: // %exit
-; CHECK-NEXT: str z4, [x1]
-; CHECK-NEXT: str z5, [x1, #1, mul vl]
-; CHECK-NEXT: str z6, [x1, #2, mul vl]
-; CHECK-NEXT: str z7, [x1, #3, mul vl]
+; CHECK-NEXT: ldr z9, [sp] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z8, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: str z24, [x1]
+; CHECK-NEXT: str z25, [x1, #1, mul vl]
+; CHECK-NEXT: str z26, [x1, #2, mul vl]
+; CHECK-NEXT: str z27, [x1, #3, mul vl]
+; CHECK-NEXT: addvl sp, sp, #2
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
entry:
%vscale = tail call i64 @llvm.vscale.i64()
@@ -307,45 +294,41 @@ exit:
define void @uitofp_nxv8i16_to_nxv8f64_deinterleave_in_loop(ptr %src, ptr %dst, <vscale x 8 x i1> %mask) #0 {
; CHECK-LABEL: uitofp_nxv8i16_to_nxv8f64_deinterleave_in_loop:
; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: index z7.d, #0, #4
+; CHECK-NEXT: mov z4.d, #0xffffffffffff0000
+; CHECK-NEXT: mov z5.d, #0xffffffffffff0001
+; CHECK-NEXT: mov x8, #-65534 // =0xffffffffffff0002
+; CHECK-NEXT: mov z24.d, #0xffffffffffff0003
; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z6.d, x8
; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: mov w8, #2048 // =0x800
; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: movi v3.2d, #0000000000000000
; CHECK-NEXT: cntw x9
-; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: add z4.d, z7.d, z4.d
+; CHECK-NEXT: add z5.d, z7.d, z5.d
; CHECK-NEXT: rdvl x10, #2
+; CHECK-NEXT: add z6.d, z7.d, z6.d
+; CHECK-NEXT: add z7.d, z7.d, z24.d
+; CHECK-NEXT: ptrue p1.d
; CHECK-NEXT: .LBB3_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ld1h { z4.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z24.h }, p0/z, [x0]
; CHECK-NEXT: subs x8, x8, x9
; CHECK-NEXT: add x0, x0, x10
-; CHECK-NEXT: uunpkhi z5.s, z4.h
-; CHECK-NEXT: uunpklo z4.s, z4.h
-; CHECK-NEXT: uunpkhi z6.d, z5.s
-; CHECK-NEXT: uunpklo z5.d, z5.s
-; CHECK-NEXT: uunpkhi z7.d, z4.s
-; CHECK-NEXT: uunpklo z4.d, z4.s
-; CHECK-NEXT: uzp1 z24.d, z5.d, z6.d
-; CHECK-NEXT: uzp2 z5.d, z5.d, z6.d
-; CHECK-NEXT: uzp1 z25.d, z4.d, z7.d
-; CHECK-NEXT: uzp2 z4.d, z4.d, z7.d
-; CHECK-NEXT: uzp1 z6.d, z25.d, z24.d
-; CHECK-NEXT: uzp2 z7.d, z4.d, z5.d
-; CHECK-NEXT: uzp2 z24.d, z25.d, z24.d
-; CHECK-NEXT: uzp1 z4.d, z4.d, z5.d
-; CHECK-NEXT: and z6.d, z6.d, #0xffff
-; CHECK-NEXT: and z7.d, z7.d, #0xffff
-; CHECK-NEXT: and z24.d, z24.d, #0xffff
-; CHECK-NEXT: and z4.d, z4.d, #0xffff
-; CHECK-NEXT: ucvtf z6.d, p1/m, z6.d
+; CHECK-NEXT: tbl z25.h, { z24.h }, z4.h
+; CHECK-NEXT: tbl z26.h, { z24.h }, z5.h
+; CHECK-NEXT: tbl z27.h, { z24.h }, z6.h
+; CHECK-NEXT: tbl z24.h, { z24.h }, z7.h
+; CHECK-NEXT: ucvtf z25.d, p1/m, z25.d
+; CHECK-NEXT: ucvtf z26.d, p1/m, z26.d
+; CHECK-NEXT: ucvtf z27.d, p1/m, z27.d
; CHECK-NEXT: ucvtf z24.d, p1/m, z24.d
-; CHECK-NEXT: ucvtf z7.d, p1/m, z7.d
-; CHECK-NEXT: ucvtf z4.d, p1/m, z4.d
-; CHECK-NEXT: fadd z0.d, z0.d, z6.d
-; CHECK-NEXT: fadd z2.d, z2.d, z24.d
-; CHECK-NEXT: fadd z3.d, z3.d, z7.d
-; CHECK-NEXT: fadd z1.d, z1.d, z4.d
+; CHECK-NEXT: fadd z0.d, z0.d, z25.d
+; CHECK-NEXT: fadd z1.d, z1.d, z26.d
+; CHECK-NEXT: fadd z2.d, z2.d, z27.d
+; CHECK-NEXT: fadd z3.d, z3.d, z24.d
; CHECK-NEXT: b.ne .LBB3_1
; CHECK-NEXT: // %bb.2: // %exit
; CHECK-NEXT: str z0, [x1]
>From b124e77fcc61e026bff724011e5940676539852d Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Tue, 12 May 2026 10:43:56 +0000
Subject: [PATCH 3/7] Use dyn_cast_if_present
---
llvm/lib/Target/AArch64/SVEShuffleOpts.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AArch64/SVEShuffleOpts.cpp b/llvm/lib/Target/AArch64/SVEShuffleOpts.cpp
index ab90c6eebb3d8..8b557a95dee7f 100644
--- a/llvm/lib/Target/AArch64/SVEShuffleOpts.cpp
+++ b/llvm/lib/Target/AArch64/SVEShuffleOpts.cpp
@@ -117,7 +117,8 @@ static void evaluateDeinterleave(IntrinsicInst *I, DeinterleaveMap &Candidates,
if (!Extract || Extract->getNumIndices() != 1)
return;
- auto *Extend = dyn_cast<CastInst>(Extract->getUniqueUndroppableUser());
+ auto *Extend =
+ dyn_cast_if_present<CastInst>(Extract->getUniqueUndroppableUser());
if (!Extend || (!isa<ZExtInst>(Extend) && !isa<UIToFPInst>(Extend)))
return;
>From db5d1b6636444174e8eda7b7acfe72da9c55fb56 Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Tue, 2 Jun 2026 11:20:03 +0000
Subject: [PATCH 4/7] Update comments, register pass, use array instead of
vector of pairs
---
llvm/lib/Target/AArch64/AArch64.h | 8 ++
.../Target/AArch64/AArch64PassRegistry.def | 1 +
.../Target/AArch64/AArch64TargetMachine.cpp | 11 +-
llvm/lib/Target/AArch64/SVEShuffleOpts.cpp | 123 ++++++++++++------
4 files changed, 95 insertions(+), 48 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64.h b/llvm/lib/Target/AArch64/AArch64.h
index 2a3e8d4294f34..9aae1dde6e886 100644
--- a/llvm/lib/Target/AArch64/AArch64.h
+++ b/llvm/lib/Target/AArch64/AArch64.h
@@ -184,6 +184,14 @@ void initializeSVEIntrinsicOptsPass(PassRegistry &);
void initializeSVEShuffleOptsPass(PassRegistry &);
void initializeAArch64Arm64ECCallLoweringPass(PassRegistry &);
+class SVEShuffleOptsPass : public PassInfoMixin<SVEShuffleOptsPass> {
+ const AArch64TargetMachine &TM;
+
+public:
+ explicit SVEShuffleOptsPass(const AArch64TargetMachine &TM) : TM(TM) {}
+ PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
+};
+
class AArch64StackTaggingPreRAPass
: public OptionalPassInfoMixin<AArch64StackTaggingPreRAPass> {
public:
diff --git a/llvm/lib/Target/AArch64/AArch64PassRegistry.def b/llvm/lib/Target/AArch64/AArch64PassRegistry.def
index e3e47b12a9ec3..137cdae846d06 100644
--- a/llvm/lib/Target/AArch64/AArch64PassRegistry.def
+++ b/llvm/lib/Target/AArch64/AArch64PassRegistry.def
@@ -22,6 +22,7 @@ MODULE_PASS("aarch64-lower-homogeneous-prolog-epilog", AArch64LowerHomogeneousPr
#ifndef FUNCTION_PASS
#define FUNCTION_PASS(NAME, CREATE_PASS)
#endif
+FUNCTION_PASS("aarch64-sve-shuffle-opts", SVEShuffleOptsPass(*static_cast<const AArch64TargetMachine *>(this)))
#undef FUNCTION_PASS
#ifndef MACHINE_FUNCTION_PASS
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index 546c00371911c..e919fc52f02aa 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -229,10 +229,10 @@ static cl::opt<bool> EnableSRLTSubregToRegMitigation(
"super-regs when using Subreg Liveness Tracking"),
cl::init(true), cl::Hidden);
-static cl::opt<bool> EnableSVETblOpt(
- "aarch64-sve-tbl-opt",
- cl::desc("Enable the use of SVE tbls instructions to replace other kinds"
- "shuffles, particularly combining them into one operation."),
+static cl::opt<bool> EnableSVEShuffleOpt(
+ "aarch64-enable-sve-shuffle-opts",
+ cl::desc("Enable pattern matching of shuffles that could make use of SVE "
+ "instructions like tbl or the bottom/top variants"),
cl::init(true), cl::Hidden);
extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
@@ -287,6 +287,7 @@ LLVMInitializeAArch64Target() {
initializeAArch64DAGToDAGISelLegacyPass(PR);
initializeAArch64CondBrTuningPass(PR);
initializeAArch64Arm64ECCallLoweringPass(PR);
+ initializeSVEShuffleOptsPass(PR);
}
bool AArch64TargetMachine::isGlobalISelOptNone() const {
@@ -688,7 +689,7 @@ void AArch64PassConfig::addIRPasses() {
addPass(createAArch64StackTaggingPass(
/*IsOptNone=*/TM->getOptLevel() == CodeGenOptLevel::None));
- if (getOptLevel() == CodeGenOptLevel::Aggressive && EnableSVETblOpt)
+ if (getOptLevel() == CodeGenOptLevel::Aggressive && EnableSVEShuffleOpt)
addPass(createSVEShuffleOptsPass());
// Match complex arithmetic patterns
diff --git a/llvm/lib/Target/AArch64/SVEShuffleOpts.cpp b/llvm/lib/Target/AArch64/SVEShuffleOpts.cpp
index 8b557a95dee7f..b39c416851771 100644
--- a/llvm/lib/Target/AArch64/SVEShuffleOpts.cpp
+++ b/llvm/lib/Target/AArch64/SVEShuffleOpts.cpp
@@ -11,21 +11,56 @@
//
// An example would be a loop with 4 multiply-accumulate reductions, where the
// new data in each vector iterations comes from a 4-way deinterleaving of
-// smaller datatypes loaded from memory, which are then extended and multiplied
-// by a common term loaded in reverse order from memory before being added to
-// the accumulator.
+// smaller datatypes loaded from memory which are then zero extended.
+//
+// Something like the following:
+// %bgra = call ... @llvm.masked.load
+// %deinterleave = call ... @llvm.vector.deinterleave4(%bgra)
+// If the load was of a <vscale x 8 x i16>, we now have 4 deinterleaved
+// <vscale x 2 x i16> values.
+// %b.i16 = extractvalue %deinterleave, 0
+// %b.i64 = zext <vscale x 2 x i16> %b.i16 to <vscale x 2 x i64>
+// %acc.b.next = add <vscale x 2 x i64> %acc.b, %b.i64
+// <repeat for the other 3 subvectors>
//
// If the initial load is a legal vector rather than 4x the size (generating a
// structured ld4 instead), we would see multiple uunpkhi/lo instructions for
-// the extensions, followed by uzp1/2 instructions for the deinterleave, and rev
-// instructions for the common terms. Instead, we can replace all of those with
-// 4 tbl instructions. The tradeoff, of course, is that we now have 4 mask
-// values to maintain which increases register pressure.
+// the extensions, followed by uzp1/2 instructions for the deinterleave.
+// Instead, we can replace all of those with 4 tbl instructions. The tradeoff,
+// of course, is that we now have 4 mask values to maintain which increases
+// register pressure.
+//
+// This basic transform could be performed in CodeGenPrepare (as the equivalent
+// for NEON is), or in a DAG Combine. However, we hope to extend it to detect
+// other shuffles that we can fold into the tbl. Extending the above example,
+// if instead of directly adding to the accumulator we multiplied it by a
+// common term for all 4 components that had been reversed:
+// %common.load = call @llvm.masked.load
+// %common.reverse = call @llvm.vector.reverse
+// These would be loaded at the extended size, <vscale x 2 x i64> in our
+// example.
+// %b.mul = mul <vscale x 2 x i64> %b.i64, %common.reverse
+// %acc.b.next = add <vscale x 2 x i64> %acc.b, %b.mul
+// <repeat for the other 3 subvectors, using %common.reverse for each)
+//
+// In this case, the reverse isn't applied to the deinterleaved data in the
+// original IR, but to the common term multiplied by the individual bgra
+// elements. If the order of the elements in the accumulator is important, we
+// cannot change that. If, however, we know that the accumulator is reduced to
+// a single scalar after the loop and the data is either integers or floating
+// point with reassociation allowed, we could instead choose a different mask
+// for the tbls to reverse the individual bgra elements instead, removing an
+// additional instruction from the loop. This does require looking beyond the
+// blocks in the loop, so DAGCombine won't help.
//
// We should also be able to introduce new shuffles in order to balance out
// SVE's bottom/top instruction pairs, which act on even/odd lanes instead of
// the high or low half of a register.
//
+// This pass may end up being a temporary solution that is removed if we can
+// create a generic vector shuffle intrinsic and move this feature to
+// LoopVectorize itself, as that would allow for better cost modelling.
+//
//===----------------------------------------------------------------------===//
#include "AArch64.h"
@@ -47,6 +82,7 @@
#include "llvm/IR/PassManager.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/InitializePasses.h"
+#include <array>
#include <optional>
using namespace llvm;
@@ -92,13 +128,14 @@ struct SVEShuffleOpts : public FunctionPass {
/// A mapping between a vector_deinterleaveN intrinsic and extending cast
/// instructions used on the resulting subvectors.
-using DeinterleaveMap =
- SmallDenseMap<CallInst *, SmallVector<std::pair<CastInst *, unsigned>, 4>>;
+using DeinterleaveMap = SmallDenseMap<CallInst *, std::array<CastInst *, 4>>;
+/// Evaluate a deinterleave and see what the uses are. If we find other
+/// operations that we can combine into a tbl shuffle, add the deinterleave and
+/// the operations (currently only zext or uitofp) to the candidates map.
static void evaluateDeinterleave(IntrinsicInst *I, DeinterleaveMap &Candidates,
Loop &L) {
- // TODO: 'Legalize' if the input is wider than nx128b but not wide enough
- // to match a structured load?
+ // This pass currently only handles legal SVE vector types.
if (I->getOperand(0)
->getType()
->getPrimitiveSizeInBits()
@@ -108,15 +145,15 @@ static void evaluateDeinterleave(IntrinsicInst *I, DeinterleaveMap &Candidates,
unsigned IntId = I->getIntrinsicID();
assert(IntId == Intrinsic::vector_deinterleave4 &&
"Only deinterleave4 supported currently");
- SmallVector<std::pair<CastInst *, unsigned>, 4> Extends;
+ std::array<CastInst *, 4> Extends = {};
unsigned Opcode = 0;
Type *DestTy = nullptr;
for (User *U : I->users()) {
auto *Extract = dyn_cast<ExtractValueInst>(U);
- // We expect only a single cast instruction as a user.
- if (!Extract || Extract->getNumIndices() != 1)
+ if (!Extract)
return;
+ // We expect only a single cast instruction as a user for the extract.
auto *Extend =
dyn_cast_if_present<CastInst>(Extract->getUniqueUndroppableUser());
if (!Extend || (!isa<ZExtInst>(Extend) && !isa<UIToFPInst>(Extend)))
@@ -142,48 +179,56 @@ static void evaluateDeinterleave(IntrinsicInst *I, DeinterleaveMap &Candidates,
if (DestBits / SrcBits != 4)
return;
- // We can't abuse the invalid index trick for tbls of bytes, since the
- // largest possible SVE vector (2048b) would have 256 bytes, leaving no
- // way of zeroing.
+ // TBL zeroes elements with an out-of-bounds index, but for the largest
+ // possible SVE vector (2048b) a maximum value for i8 elements (256) is not
+ // large enough to encode an 'out of bounds' value.
// TODO: If we know vscale is 8 or less, then we could use tbls for bytes.
if (SrcBits <= 8)
return;
- Extends.push_back({Extend, Extract->getIndices()[0]});
+ Extends[Extract->getIndices().front()] = Extend;
}
// Check that all extracted values are being extended the same way, and that
// we have the expected number of extensions.
- if (Extends.size() != 4 ||
- !all_of(Extends, [&](std::pair<CastInst *, unsigned> Ext) {
- CastInst *CI = Ext.first;
- return CI->getDestTy() == DestTy && CI->getOpcode() == Opcode;
+ if (!all_of(Extends, [DestTy, Opcode](CastInst *CI) {
+ return CI && CI->getDestTy() == DestTy && CI->getOpcode() == Opcode;
}))
return;
- Candidates.insert({I, Extends});
+ Candidates.try_emplace(I, Extends);
}
-// Optimize zext and uitofp from a 4-way deinterleaved load.
+/// Given a map of deinterleaves to zext or uitofp casts, remove the operations
+/// and replace them with tbl shuffles.
static void optimizeSVEDeinterleavedExtends(DeinterleaveMap Deinterleaves) {
// TODO: Cache tbl patterns and reuse, and abandon transforms for a particular
// deinterleave if it would introduce too many. We probably want a
// hardcoded number of tbls to start with, but if we can estimate
// register pressure then we could make better decisions.
for (auto &[Deinterleave, Extends] : Deinterleaves) {
- VectorType *DestTy = cast<VectorType>(Extends[0].first->getDestTy());
- VectorType *SrcTy = cast<VectorType>(Extends[0].first->getSrcTy());
+ VectorType *DestTy = cast<VectorType>(Extends[0]->getDestTy());
+ VectorType *SrcTy = cast<VectorType>(Extends[0]->getSrcTy());
unsigned DstBits = DestTy->getScalarSizeInBits();
unsigned SrcBits = SrcTy->getScalarSizeInBits();
- bool IsUIToFP = isa<UIToFPInst>(Extends[0].first);
+ bool IsUIToFP = isa<UIToFPInst>(Extends[0]);
VectorType *StepVecTy = VectorType::getInteger(DestTy);
Type *StepTy = StepVecTy->getScalarType();
Value *Input = Deinterleave->getOperand(0);
Type *InputTy = Input->getType();
APInt Invalid = APInt::getAllOnes(DstBits);
- for (auto &[Extend, Idx] : Extends) {
- // Build mask
+ for (auto [Idx, Extend] : enumerate(Extends)) {
+ // Build the mask using stepvectors and casting.
+ // We want to select the Idx'th element, and every 4 elements after that.
+ // Each element needs to be zero extended; we can do that by providing
+ // tbl index values that are out of range. We can't do that nicely with
+ // a stepvector of the same element type as the input type, but we can
+ // do it with elements the size of the output type.
+ // E.g. for element 0 of a 16b -> 64b zext, we would start with a mask of
+ // 0xFFFF_FFFF_FFFF_0000 + Idx for the start of the stepvector, and use a
+ // step of 4. We then cast that back to an element size of 16b, yielding
+ // <0x0000 + Idx, 0xFFFF, 0xFFFF, 0xFFFF, 0x0004 + Idx, 0xFFFF...>.
APInt StartIdx = Invalid << SrcBits;
StartIdx += Idx;
IRBuilder<> Builder(Extend);
@@ -245,18 +290,11 @@ INITIALIZE_PASS_END(SVEShuffleOpts, DEBUG_TYPE, name, false, false)
FunctionPass *llvm::createSVEShuffleOptsPass() { return new SVEShuffleOpts(); }
-namespace llvm {
-class SVEShuffleOptsPass : public PassInfoMixin<SVEShuffleOptsPass> {
- const AArch64TargetMachine *TM;
-
-public:
- explicit SVEShuffleOptsPass(const AArch64TargetMachine &TM) : TM(&TM) {}
- PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM) {
- SVEShuffleImpl Impl(TM);
- return Impl.run(F, FAM);
- }
-};
-} // end namespace llvm
+PreservedAnalyses SVEShuffleOptsPass::run(Function &F,
+ FunctionAnalysisManager &FAM) {
+ SVEShuffleImpl Impl(&TM);
+ return Impl.run(F, FAM);
+}
bool SVEShuffleImpl::runOnFunction(Function &F, Pass &P) {
// Make sure we can use SVE
@@ -290,10 +328,9 @@ PreservedAnalyses SVEShuffleImpl::run(Function &F,
bool Changed = false;
// Only looking to tranform innermost loops, given the increase in
// register usage.
- for (Loop *L : LI->getLoopsInPreorder()) {
+ for (Loop *L : LI->getLoopsInPreorder())
if (L->isInnermost())
Changed |= processLoop(*L);
- }
// Can we do better than 'none'?
// We're not actually using the new pass manager though.
>From e10af5a9c64f107f91e86429fd19fea08b303e81 Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Tue, 2 Jun 2026 12:49:03 +0000
Subject: [PATCH 5/7] Test exercising new pm
---
.../CodeGen/AArch64/sve-tbl-folding-new-pm.ll | 107 ++++++++++++++++++
1 file changed, 107 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/sve-tbl-folding-new-pm.ll
diff --git a/llvm/test/CodeGen/AArch64/sve-tbl-folding-new-pm.ll b/llvm/test/CodeGen/AArch64/sve-tbl-folding-new-pm.ll
new file mode 100644
index 0000000000000..8b62ca157e5ce
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-tbl-folding-new-pm.ll
@@ -0,0 +1,107 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes='aarch64-sve-shuffle-opts,instcombine' -mtriple=aarch64-linux-gnu -mattr=+sve -S < %s | FileCheck %s
+
+define void @zext_nxv8i16_to_nxv8i64_deinterleave_in_loop(ptr %src, ptr %dst, <vscale x 8 x i1> %mask) #0 {
+; CHECK-LABEL: define void @zext_nxv8i16_to_nxv8i64_deinterleave_in_loop(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], <vscale x 8 x i1> [[MASK:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[VSCALE:%.*]] = tail call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[STRIDE:%.*]] = shl nuw nsw i64 [[VSCALE]], 1
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ACC_B_I64:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD_B_I64:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ACC_G_I64:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD_G_I64:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ACC_R_I64:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD_R_I64:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ACC_A_I64:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD_A_I64:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[SRC_GEP:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[BGRA:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr [[SRC_GEP]], <vscale x 8 x i1> [[MASK]], <vscale x 8 x i16> zeroinitializer)
+; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl <vscale x 2 x i64> [[TMP0]], splat (i64 2)
+; CHECK-NEXT: [[TMP2:%.*]] = add <vscale x 2 x i64> [[TMP1]], splat (i64 -65536)
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <vscale x 2 x i64> [[TMP2]] to <vscale x 8 x i16>
+; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.tbl.nxv8i16(<vscale x 8 x i16> [[BGRA]], <vscale x 8 x i16> [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast <vscale x 8 x i16> [[TMP4]] to <vscale x 2 x i64>
+; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; CHECK-NEXT: [[TMP7:%.*]] = shl <vscale x 2 x i64> [[TMP6]], splat (i64 2)
+; CHECK-NEXT: [[TMP8:%.*]] = add <vscale x 2 x i64> [[TMP7]], splat (i64 -65535)
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast <vscale x 2 x i64> [[TMP8]] to <vscale x 8 x i16>
+; CHECK-NEXT: [[TMP10:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.tbl.nxv8i16(<vscale x 8 x i16> [[BGRA]], <vscale x 8 x i16> [[TMP9]])
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast <vscale x 8 x i16> [[TMP10]] to <vscale x 2 x i64>
+; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; CHECK-NEXT: [[TMP13:%.*]] = shl <vscale x 2 x i64> [[TMP12]], splat (i64 2)
+; CHECK-NEXT: [[TMP14:%.*]] = add <vscale x 2 x i64> [[TMP13]], splat (i64 -65534)
+; CHECK-NEXT: [[TMP15:%.*]] = bitcast <vscale x 2 x i64> [[TMP14]] to <vscale x 8 x i16>
+; CHECK-NEXT: [[TMP16:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.tbl.nxv8i16(<vscale x 8 x i16> [[BGRA]], <vscale x 8 x i16> [[TMP15]])
+; CHECK-NEXT: [[TMP17:%.*]] = bitcast <vscale x 8 x i16> [[TMP16]] to <vscale x 2 x i64>
+; CHECK-NEXT: [[TMP18:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; CHECK-NEXT: [[TMP19:%.*]] = shl <vscale x 2 x i64> [[TMP18]], splat (i64 2)
+; CHECK-NEXT: [[TMP20:%.*]] = add <vscale x 2 x i64> [[TMP19]], splat (i64 -65533)
+; CHECK-NEXT: [[TMP21:%.*]] = bitcast <vscale x 2 x i64> [[TMP20]] to <vscale x 8 x i16>
+; CHECK-NEXT: [[TMP22:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.tbl.nxv8i16(<vscale x 8 x i16> [[BGRA]], <vscale x 8 x i16> [[TMP21]])
+; CHECK-NEXT: [[TMP23:%.*]] = bitcast <vscale x 8 x i16> [[TMP22]] to <vscale x 2 x i64>
+; CHECK-NEXT: [[ADD_B_I64]] = add <vscale x 2 x i64> [[ACC_B_I64]], [[TMP5]]
+; CHECK-NEXT: [[ADD_G_I64]] = add <vscale x 2 x i64> [[ACC_G_I64]], [[TMP11]]
+; CHECK-NEXT: [[ADD_R_I64]] = add <vscale x 2 x i64> [[ACC_R_I64]], [[TMP17]]
+; CHECK-NEXT: [[ADD_A_I64]] = add <vscale x 2 x i64> [[ACC_A_I64]], [[TMP23]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], [[STRIDE]]
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 2048
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: store <vscale x 2 x i64> [[ADD_B_I64]], ptr [[DST]], align 16
+; CHECK-NEXT: [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP25:%.*]] = shl nuw i64 [[TMP24]], 4
+; CHECK-NEXT: [[G_I64_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP25]]
+; CHECK-NEXT: store <vscale x 2 x i64> [[ADD_G_I64]], ptr [[G_I64_GEP]], align 16
+; CHECK-NEXT: [[TMP26:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[R_I64_GEP_IDX:%.*]] = shl i64 [[TMP26]], 5
+; CHECK-NEXT: [[R_I64_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[R_I64_GEP_IDX]]
+; CHECK-NEXT: store <vscale x 2 x i64> [[ADD_R_I64]], ptr [[R_I64_GEP]], align 16
+; CHECK-NEXT: [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[A_I64_GEP_IDX:%.*]] = mul i64 [[TMP27]], 48
+; CHECK-NEXT: [[A_I64_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[A_I64_GEP_IDX]]
+; CHECK-NEXT: store <vscale x 2 x i64> [[ADD_A_I64]], ptr [[A_I64_GEP]], align 16
+; CHECK-NEXT: ret void
+;
+entry:
+ %vscale = tail call i64 @llvm.vscale.i64()
+ %stride = shl nuw nsw i64 %vscale, 1
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc.b.i64 = phi <vscale x 2 x i64> [ splat(i64 0), %entry ], [ %add.b.i64, %loop ]
+ %acc.g.i64 = phi <vscale x 2 x i64> [ splat(i64 0), %entry ], [ %add.g.i64, %loop ]
+ %acc.r.i64 = phi <vscale x 2 x i64> [ splat(i64 0), %entry ], [ %add.r.i64, %loop ]
+ %acc.a.i64 = phi <vscale x 2 x i64> [ splat(i64 0), %entry ], [ %add.a.i64, %loop ]
+ %src.gep = getelementptr inbounds nuw [4 x i16], ptr %src, i64 %iv
+ %bgra = call <vscale x 8 x i16> @llvm.masked.load(ptr %src.gep, <vscale x 8 x i1> %mask, <vscale x 8 x i16> zeroinitializer)
+ %deinterleave = tail call { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } @llvm.vector.deinterleave4(<vscale x 8 x i16> %bgra)
+ %b.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 0
+ %g.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 1
+ %r.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 2
+ %a.i16 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %deinterleave, 3
+ %b.i64 = zext <vscale x 2 x i16> %b.i16 to <vscale x 2 x i64>
+ %g.i64 = zext <vscale x 2 x i16> %g.i16 to <vscale x 2 x i64>
+ %r.i64 = zext <vscale x 2 x i16> %r.i16 to <vscale x 2 x i64>
+ %a.i64 = zext <vscale x 2 x i16> %a.i16 to <vscale x 2 x i64>
+ %add.b.i64 = add <vscale x 2 x i64> %acc.b.i64, %b.i64
+ %add.g.i64 = add <vscale x 2 x i64> %acc.g.i64, %g.i64
+ %add.r.i64 = add <vscale x 2 x i64> %acc.r.i64, %r.i64
+ %add.a.i64 = add <vscale x 2 x i64> %acc.a.i64, %a.i64
+ %iv.next = add nuw i64 %iv, %stride
+ %ec = icmp eq i64 %iv.next, 2048
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ store <vscale x 2 x i64> %add.b.i64, ptr %dst
+ %g.i64.gep = getelementptr <vscale x 2 x i64>, ptr %dst, i64 1
+ store <vscale x 2 x i64> %add.g.i64, ptr %g.i64.gep
+ %r.i64.gep = getelementptr <vscale x 2 x i64>, ptr %dst, i64 2
+ store <vscale x 2 x i64> %add.r.i64, ptr %r.i64.gep
+ %a.i64.gep = getelementptr <vscale x 2 x i64>, ptr %dst, i64 3
+ store <vscale x 2 x i64> %add.a.i64, ptr %a.i64.gep
+ ret void
+}
+
+attributes #0 = { "target-features"="+sve" }
>From de7d4bc6a66a6d5298e50143c0b6569df1182650 Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Tue, 2 Jun 2026 13:02:30 +0000
Subject: [PATCH 6/7] Improve preserved analysis set
---
llvm/lib/Target/AArch64/SVEShuffleOpts.cpp | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AArch64/SVEShuffleOpts.cpp b/llvm/lib/Target/AArch64/SVEShuffleOpts.cpp
index b39c416851771..936ecc5795161 100644
--- a/llvm/lib/Target/AArch64/SVEShuffleOpts.cpp
+++ b/llvm/lib/Target/AArch64/SVEShuffleOpts.cpp
@@ -12,7 +12,7 @@
// An example would be a loop with 4 multiply-accumulate reductions, where the
// new data in each vector iterations comes from a 4-way deinterleaving of
// smaller datatypes loaded from memory which are then zero extended.
-//
+//
// Something like the following:
// %bgra = call ... @llvm.masked.load
// %deinterleave = call ... @llvm.vector.deinterleave4(%bgra)
@@ -68,7 +68,9 @@
#include "AArch64TargetMachine.h"
#include "Utils/AArch64BaseInfo.h"
#include "llvm/ADT/SetVector.h"
+#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/MemorySSA.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
@@ -332,7 +334,11 @@ PreservedAnalyses SVEShuffleImpl::run(Function &F,
if (L->isInnermost())
Changed |= processLoop(*L);
- // Can we do better than 'none'?
- // We're not actually using the new pass manager though.
- return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
+ PreservedAnalyses PA;
+ PA.preserveSet<CFGAnalyses>();
+ PA.preserve<TargetIRAnalysis>();
+ PA.preserve<AssumptionAnalysis>();
+ PA.preserve<MemorySSAAnalysis>();
+
+ return Changed ? PA : PreservedAnalyses::all();
}
>From f2694c86a30f6135e8c85a5bfbbf25771c17290a Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Wed, 10 Jun 2026 10:30:41 +0000
Subject: [PATCH 7/7] * Enable shuffle opts at O2 or higher * Change type
checks * Exclude big endian targets * Add NUW flags to arithmetic * Erased
extracts and deinterleave
---
.../Target/AArch64/AArch64TargetMachine.cpp | 2 +-
llvm/lib/Target/AArch64/SVEShuffleOpts.cpp | 70 ++++++++++---------
.../CodeGen/AArch64/sve-tbl-folding-new-pm.ll | 32 ++++-----
3 files changed, 52 insertions(+), 52 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index e919fc52f02aa..b9acf00cf6608 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -689,7 +689,7 @@ void AArch64PassConfig::addIRPasses() {
addPass(createAArch64StackTaggingPass(
/*IsOptNone=*/TM->getOptLevel() == CodeGenOptLevel::None));
- if (getOptLevel() == CodeGenOptLevel::Aggressive && EnableSVEShuffleOpt)
+ if (getOptLevel() >= CodeGenOptLevel::Default && EnableSVEShuffleOpt)
addPass(createSVEShuffleOptsPass());
// Match complex arithmetic patterns
diff --git a/llvm/lib/Target/AArch64/SVEShuffleOpts.cpp b/llvm/lib/Target/AArch64/SVEShuffleOpts.cpp
index 936ecc5795161..6622a34790ef6 100644
--- a/llvm/lib/Target/AArch64/SVEShuffleOpts.cpp
+++ b/llvm/lib/Target/AArch64/SVEShuffleOpts.cpp
@@ -66,12 +66,11 @@
#include "AArch64.h"
#include "AArch64Subtarget.h"
#include "AArch64TargetMachine.h"
-#include "Utils/AArch64BaseInfo.h"
-#include "llvm/ADT/SetVector.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemorySSA.h"
#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/IR/Constants.h"
@@ -80,12 +79,10 @@
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAArch64.h"
#include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/Module.h"
#include "llvm/IR/PassManager.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/InitializePasses.h"
#include <array>
-#include <optional>
using namespace llvm;
using namespace llvm::PatternMatch;
@@ -96,6 +93,7 @@ namespace {
class SVEShuffleImpl {
const AArch64TargetMachine *TM = nullptr;
+ const AArch64Subtarget *ST = nullptr;
const LoopInfo *LI = nullptr;
public:
@@ -136,17 +134,29 @@ using DeinterleaveMap = SmallDenseMap<CallInst *, std::array<CastInst *, 4>>;
/// operations that we can combine into a tbl shuffle, add the deinterleave and
/// the operations (currently only zext or uitofp) to the candidates map.
static void evaluateDeinterleave(IntrinsicInst *I, DeinterleaveMap &Candidates,
- Loop &L) {
- // This pass currently only handles legal SVE vector types.
- if (I->getOperand(0)
- ->getType()
- ->getPrimitiveSizeInBits()
- .getKnownMinValue() != AArch64::SVEBitsPerBlock)
- return;
-
+ Loop &L, const AArch64TargetLowering &TL,
+ const DataLayout DL) {
unsigned IntId = I->getIntrinsicID();
assert(IntId == Intrinsic::vector_deinterleave4 &&
"Only deinterleave4 supported currently");
+
+ // If our user count doesn't match the interleave factor, bail out.
+ if (I->getNumUses() != 4)
+ return;
+
+ // This pass currently only handles legal scalable vector types with an
+ // element type that is at least 16 bits. TBL zeroes elements with an
+ // out-of-bounds index, but for the largest possible SVE vector (2048b) a
+ // maximum value for i8 elements (256) is not large enough to encode an
+ // 'out of bounds' value.
+ // TODO: If we know vscale is 8 or less, then we could use tbls for bytes.
+ EVT InputVT = TL.getValueType(DL, I->getOperand(0)->getType());
+ if (!InputVT.isScalableVector() ||
+ InputVT.getVectorElementType().getFixedSizeInBits() < 16 ||
+ TL.getTypeConversion(I->getContext(), InputVT).first !=
+ TargetLoweringBase::TypeLegal)
+ return;
+
std::array<CastInst *, 4> Extends = {};
unsigned Opcode = 0;
Type *DestTy = nullptr;
@@ -170,10 +180,6 @@ static void evaluateDeinterleave(IntrinsicInst *I, DeinterleaveMap &Candidates,
DestTy = Extend->getDestTy();
Type *SrcTy = Extend->getSrcTy();
- // For now, we only want to handle scalable vectors here.
- if (!DestTy->isScalableTy())
- return;
-
unsigned SrcBits = SrcTy->getScalarSizeInBits();
unsigned DestBits = DestTy->getScalarSizeInBits();
@@ -181,13 +187,6 @@ static void evaluateDeinterleave(IntrinsicInst *I, DeinterleaveMap &Candidates,
if (DestBits / SrcBits != 4)
return;
- // TBL zeroes elements with an out-of-bounds index, but for the largest
- // possible SVE vector (2048b) a maximum value for i8 elements (256) is not
- // large enough to encode an 'out of bounds' value.
- // TODO: If we know vscale is 8 or less, then we could use tbls for bytes.
- if (SrcBits <= 8)
- return;
-
Extends[Extract->getIndices().front()] = Extend;
}
@@ -235,11 +234,10 @@ static void optimizeSVEDeinterleavedExtends(DeinterleaveMap Deinterleaves) {
StartIdx += Idx;
IRBuilder<> Builder(Extend);
Value *StepVector = Builder.CreateStepVector(StepVecTy);
- Value *ScaledSteps = Builder.CreateMul(
- StepVector, Builder.CreateVectorSplat(StepVecTy->getElementCount(),
- ConstantInt::get(StepTy, 4)));
+ Value *ScaledSteps =
+ Builder.CreateNUWMul(StepVector, ConstantInt::get(StepVecTy, 4));
Value *Start = ConstantInt::get(StepTy, StartIdx);
- Value *ZextTbl = Builder.CreateAdd(
+ Value *ZextTbl = Builder.CreateNUWAdd(
ScaledSteps,
Builder.CreateVectorSplat(StepVecTy->getElementCount(), Start));
Value *FinalMask = Builder.CreateBitCast(ZextTbl, InputTy);
@@ -256,6 +254,11 @@ static void optimizeSVEDeinterleavedExtends(DeinterleaveMap Deinterleaves) {
Extend->replaceAllUsesWith(Widen);
Extend->eraseFromParent();
}
+
+ // Delete the unused extracts and deinterleave.
+ for (User *U : make_early_inc_range(Deinterleave->users()))
+ cast<Instruction>(U)->eraseFromParent();
+ Deinterleave->eraseFromParent();
}
}
@@ -268,7 +271,8 @@ bool SVEShuffleImpl::processLoop(Loop &L) {
for (auto *BB : L.blocks())
for (auto &I : *BB)
if (match(&I, m_Intrinsic<Intrinsic::vector_deinterleave4>(m_Value())))
- evaluateDeinterleave(cast<IntrinsicInst>(&I), Candidates, L);
+ evaluateDeinterleave(cast<IntrinsicInst>(&I), Candidates, L,
+ *ST->getTargetLowering(), TM->createDataLayout());
if (Candidates.empty())
return false;
@@ -302,8 +306,9 @@ bool SVEShuffleImpl::runOnFunction(Function &F, Pass &P) {
// Make sure we can use SVE
TargetPassConfig &TPC = P.getAnalysis<TargetPassConfig>();
TM = &TPC.getTM<AArch64TargetMachine>();
- const AArch64Subtarget *ST = TM->getSubtargetImpl(F);
- if (!ST->isSVEorStreamingSVEAvailable())
+ ST = TM->getSubtargetImpl(F);
+ if (!ST->isSVEorStreamingSVEAvailable() ||
+ TM->createDataLayout().isBigEndian())
return false;
LI = &P.getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
@@ -321,8 +326,9 @@ bool SVEShuffleImpl::runOnFunction(Function &F, Pass &P) {
PreservedAnalyses SVEShuffleImpl::run(Function &F,
FunctionAnalysisManager &FAM) {
- const AArch64Subtarget *ST = TM->getSubtargetImpl(F);
- if (!ST->isSVEorStreamingSVEAvailable())
+ ST = TM->getSubtargetImpl(F);
+ if (!ST->isSVEorStreamingSVEAvailable() ||
+ TM->createDataLayout().isBigEndian())
return PreservedAnalyses::all();
LI = &FAM.getResult<LoopAnalysis>(F);
diff --git a/llvm/test/CodeGen/AArch64/sve-tbl-folding-new-pm.ll b/llvm/test/CodeGen/AArch64/sve-tbl-folding-new-pm.ll
index 8b62ca157e5ce..e897264e5b7a4 100644
--- a/llvm/test/CodeGen/AArch64/sve-tbl-folding-new-pm.ll
+++ b/llvm/test/CodeGen/AArch64/sve-tbl-folding-new-pm.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
-; RUN: opt -passes='aarch64-sve-shuffle-opts,instcombine' -mtriple=aarch64-linux-gnu -mattr=+sve -S < %s | FileCheck %s
+; RUN: opt -passes=aarch64-sve-shuffle-opts -mtriple=aarch64-linux-gnu -mattr=+sve -S < %s | FileCheck %s
define void @zext_nxv8i16_to_nxv8i64_deinterleave_in_loop(ptr %src, ptr %dst, <vscale x 8 x i1> %mask) #0 {
; CHECK-LABEL: define void @zext_nxv8i16_to_nxv8i64_deinterleave_in_loop(
@@ -14,29 +14,29 @@ define void @zext_nxv8i16_to_nxv8i64_deinterleave_in_loop(ptr %src, ptr %dst, <v
; CHECK-NEXT: [[ACC_G_I64:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD_G_I64:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[ACC_R_I64:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD_R_I64:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[ACC_A_I64:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD_A_I64:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[SRC_GEP:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[SRC_GEP:%.*]] = getelementptr inbounds nuw [4 x i16], ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[BGRA:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr [[SRC_GEP]], <vscale x 8 x i1> [[MASK]], <vscale x 8 x i16> zeroinitializer)
; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl <vscale x 2 x i64> [[TMP0]], splat (i64 2)
-; CHECK-NEXT: [[TMP2:%.*]] = add <vscale x 2 x i64> [[TMP1]], splat (i64 -65536)
+; CHECK-NEXT: [[TMP1:%.*]] = mul nuw <vscale x 2 x i64> [[TMP0]], splat (i64 4)
+; CHECK-NEXT: [[TMP2:%.*]] = add nuw <vscale x 2 x i64> [[TMP1]], splat (i64 -65536)
; CHECK-NEXT: [[TMP3:%.*]] = bitcast <vscale x 2 x i64> [[TMP2]] to <vscale x 8 x i16>
; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.tbl.nxv8i16(<vscale x 8 x i16> [[BGRA]], <vscale x 8 x i16> [[TMP3]])
; CHECK-NEXT: [[TMP5:%.*]] = bitcast <vscale x 8 x i16> [[TMP4]] to <vscale x 2 x i64>
; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
-; CHECK-NEXT: [[TMP7:%.*]] = shl <vscale x 2 x i64> [[TMP6]], splat (i64 2)
-; CHECK-NEXT: [[TMP8:%.*]] = add <vscale x 2 x i64> [[TMP7]], splat (i64 -65535)
+; CHECK-NEXT: [[TMP7:%.*]] = mul nuw <vscale x 2 x i64> [[TMP6]], splat (i64 4)
+; CHECK-NEXT: [[TMP8:%.*]] = add nuw <vscale x 2 x i64> [[TMP7]], splat (i64 -65535)
; CHECK-NEXT: [[TMP9:%.*]] = bitcast <vscale x 2 x i64> [[TMP8]] to <vscale x 8 x i16>
; CHECK-NEXT: [[TMP10:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.tbl.nxv8i16(<vscale x 8 x i16> [[BGRA]], <vscale x 8 x i16> [[TMP9]])
; CHECK-NEXT: [[TMP11:%.*]] = bitcast <vscale x 8 x i16> [[TMP10]] to <vscale x 2 x i64>
; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
-; CHECK-NEXT: [[TMP13:%.*]] = shl <vscale x 2 x i64> [[TMP12]], splat (i64 2)
-; CHECK-NEXT: [[TMP14:%.*]] = add <vscale x 2 x i64> [[TMP13]], splat (i64 -65534)
+; CHECK-NEXT: [[TMP13:%.*]] = mul nuw <vscale x 2 x i64> [[TMP12]], splat (i64 4)
+; CHECK-NEXT: [[TMP14:%.*]] = add nuw <vscale x 2 x i64> [[TMP13]], splat (i64 -65534)
; CHECK-NEXT: [[TMP15:%.*]] = bitcast <vscale x 2 x i64> [[TMP14]] to <vscale x 8 x i16>
; CHECK-NEXT: [[TMP16:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.tbl.nxv8i16(<vscale x 8 x i16> [[BGRA]], <vscale x 8 x i16> [[TMP15]])
; CHECK-NEXT: [[TMP17:%.*]] = bitcast <vscale x 8 x i16> [[TMP16]] to <vscale x 2 x i64>
; CHECK-NEXT: [[TMP18:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
-; CHECK-NEXT: [[TMP19:%.*]] = shl <vscale x 2 x i64> [[TMP18]], splat (i64 2)
-; CHECK-NEXT: [[TMP20:%.*]] = add <vscale x 2 x i64> [[TMP19]], splat (i64 -65533)
+; CHECK-NEXT: [[TMP19:%.*]] = mul nuw <vscale x 2 x i64> [[TMP18]], splat (i64 4)
+; CHECK-NEXT: [[TMP20:%.*]] = add nuw <vscale x 2 x i64> [[TMP19]], splat (i64 -65533)
; CHECK-NEXT: [[TMP21:%.*]] = bitcast <vscale x 2 x i64> [[TMP20]] to <vscale x 8 x i16>
; CHECK-NEXT: [[TMP22:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.tbl.nxv8i16(<vscale x 8 x i16> [[BGRA]], <vscale x 8 x i16> [[TMP21]])
; CHECK-NEXT: [[TMP23:%.*]] = bitcast <vscale x 8 x i16> [[TMP22]] to <vscale x 2 x i64>
@@ -49,17 +49,11 @@ define void @zext_nxv8i16_to_nxv8i64_deinterleave_in_loop(ptr %src, ptr %dst, <v
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: store <vscale x 2 x i64> [[ADD_B_I64]], ptr [[DST]], align 16
-; CHECK-NEXT: [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP25:%.*]] = shl nuw i64 [[TMP24]], 4
-; CHECK-NEXT: [[G_I64_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP25]]
+; CHECK-NEXT: [[G_I64_GEP:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[DST]], i64 1
; CHECK-NEXT: store <vscale x 2 x i64> [[ADD_G_I64]], ptr [[G_I64_GEP]], align 16
-; CHECK-NEXT: [[TMP26:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[R_I64_GEP_IDX:%.*]] = shl i64 [[TMP26]], 5
-; CHECK-NEXT: [[R_I64_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[R_I64_GEP_IDX]]
+; CHECK-NEXT: [[R_I64_GEP:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[DST]], i64 2
; CHECK-NEXT: store <vscale x 2 x i64> [[ADD_R_I64]], ptr [[R_I64_GEP]], align 16
-; CHECK-NEXT: [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[A_I64_GEP_IDX:%.*]] = mul i64 [[TMP27]], 48
-; CHECK-NEXT: [[A_I64_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[A_I64_GEP_IDX]]
+; CHECK-NEXT: [[A_I64_GEP:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[DST]], i64 3
; CHECK-NEXT: store <vscale x 2 x i64> [[ADD_A_I64]], ptr [[A_I64_GEP]], align 16
; CHECK-NEXT: ret void
;
More information about the llvm-commits
mailing list