[llvm] [AArch64] Keep v16i8 -> v2i64 partial_reduce fixed-length for VL > 128 (PR #204938)
Matthew Blewitt via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 30 03:17:28 PDT 2026
https://github.com/mble updated https://github.com/llvm/llvm-project/pull/204938
>From 0fc04bb72e838fc40507128139d5b7f902c6b156 Mon Sep 17 00:00:00 2001
From: Matt Blewitt <mble at planetscale.com>
Date: Sat, 20 Jun 2026 10:28:18 -0700
Subject: [PATCH 1/6] [AArch64] Keep v16i8 -> v2i64 partial_reduce fixed-length
for VL > 128
A fixed-length llvm.vector.partial.reduce.add with a <2 x i64>
accumulator and i8 inputs was lowered, on any +sve target, through a
scalable partial reduction whose result is then truncated back to the
low 128 bits via convertFromScalableVector. A partial reduction is not
lane-wise: the scalable result distributes the accumulated sums across
all VL/64 lanes, so for any runtime vector length greater than 128 bits
the high lanes are silently dropped. On a 256-bit machine (e.g. Neoverse
V1) exactly half the result is lost.
This is the i64 sibling of the v16i8 -> v2i32 case fixed in #177119
(issue #176954); the fixed-length support was introduced in #142032.
Keep the v2i64 <- v16i8 reduction fixed-length when NEON is available,
using the VL-independent NEON dot path (udot v.4s + uaddw/uaddw2)
instead of the scalable round-trip.
The carve-out is scoped to the exact 128-bit types: the fixed-length
dot path hardcodes a v4i32 dot node, so a wider <4 x i64> <- <32 x i8>
reduction must stay on its existing path. The UADDW{B,T}/SADDW{B,T}
widening-add branch is additionally restricted to scalable result
types, as those instructions are scalable-only.
Streaming-SVE (NEON unavailable) still takes the scalable path and is
left as a separate fix.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 16 +-
.../sve-fixed-length-partial-reduce.ll | 155 ++++++++----------
2 files changed, 86 insertions(+), 85 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 67ef911117eff..34a8a417192ce 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -33445,9 +33445,18 @@ AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
return DAG.getNode(ISD::SUB, DL, ResultVT, BiasedDot, BiasCorrection);
}
+ // Keep a fixed-length i8 -> i64 reduction fixed-length when NEON is
+ // available. The scalable path spreads the sums across all VL-dependent lanes
+ // but only extracts the low fixed-width lanes, dropping the high lanes for VL
+ // > the fixed width (e.g. 256-bit Neoverse V1). i64 sibling of the v16i8 ->
+ // v2i32 case fixed in PR #177119 / issue #176954.
+ bool KeepFixedI8ToI64 = Subtarget->isNeonAvailable() &&
+ ResultVT == MVT::v2i64 && OpVT == MVT::v16i8;
+
bool ConvertToScalable =
ResultVT.isFixedLengthVector() &&
- useSVEForFixedLengthVectorVT(ResultVT, /*OverrideNEON=*/true);
+ useSVEForFixedLengthVectorVT(ResultVT, /*OverrideNEON=*/true) &&
+ !KeepFixedI8ToI64;
if (ConvertToScalable) {
ResultVT = getContainerForFixedLengthVector(DAG, ResultVT);
@@ -33470,7 +33479,10 @@ AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
SDValue Res;
bool IsUnsigned = Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA;
- if (Subtarget->hasSVE2() || Subtarget->isStreamingSVEAvailable()) {
+ // UADDW{B,T}/SADDW{B,T} are scalable-only; a fixed-length result (the i8 ->
+ // i64 case kept fixed above) must use the NEON widening-add path below.
+ if (ResultVT.isScalableVector() &&
+ (Subtarget->hasSVE2() || Subtarget->isStreamingSVEAvailable())) {
unsigned LoOpcode = IsUnsigned ? AArch64ISD::UADDWB : AArch64ISD::SADDWB;
unsigned HiOpcode = IsUnsigned ? AArch64ISD::UADDWT : AArch64ISD::SADDWT;
SDValue Lo = DAG.getNode(LoOpcode, DL, ResultVT, Acc, DotNode);
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
index ae7aa9b35f62a..e80e2e795c47c 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
@@ -444,30 +444,16 @@ define <4 x i32> @four_way_i8_i32_vl128_sudot(ptr %accptr, ptr %uptr, ptr %sptr)
}
define <2 x i64> @four_way_i8_i64_vl128_usdot(ptr %accptr, ptr %uptr, ptr %sptr) {
-; NEON-LABEL: four_way_i8_i64_vl128_usdot:
-; NEON: // %bb.0:
-; NEON-NEXT: movi v0.2d, #0000000000000000
-; NEON-NEXT: ldr q1, [x1]
-; NEON-NEXT: ldr q2, [x2]
-; NEON-NEXT: usdot v0.4s, v1.16b, v2.16b
-; NEON-NEXT: ldr q1, [x0]
-; NEON-NEXT: saddw v1.2d, v1.2d, v0.2s
-; NEON-NEXT: saddw2 v0.2d, v1.2d, v0.4s
-; NEON-NEXT: ret
-;
-; SVE-LABEL: four_way_i8_i64_vl128_usdot:
-; SVE: // %bb.0:
-; SVE-NEXT: movi v0.2d, #0000000000000000
-; SVE-NEXT: ldr q1, [x1]
-; SVE-NEXT: ldr q2, [x2]
-; SVE-NEXT: usdot z0.s, z1.b, z2.b
-; SVE-NEXT: ldr q2, [x0]
-; SVE-NEXT: sunpklo z1.d, z0.s
-; SVE-NEXT: sunpkhi z0.d, z0.s
-; SVE-NEXT: add z1.d, z2.d, z1.d
-; SVE-NEXT: add z0.d, z1.d, z0.d
-; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
-; SVE-NEXT: ret
+; COMMON-LABEL: four_way_i8_i64_vl128_usdot:
+; COMMON: // %bb.0:
+; COMMON-NEXT: movi v0.2d, #0000000000000000
+; COMMON-NEXT: ldr q1, [x1]
+; COMMON-NEXT: ldr q2, [x2]
+; COMMON-NEXT: usdot v0.4s, v1.16b, v2.16b
+; COMMON-NEXT: ldr q1, [x0]
+; COMMON-NEXT: saddw v1.2d, v1.2d, v0.2s
+; COMMON-NEXT: saddw2 v0.2d, v1.2d, v0.4s
+; COMMON-NEXT: ret
;
; SME-LABEL: four_way_i8_i64_vl128_usdot:
; SME: // %bb.0:
@@ -814,30 +800,16 @@ define <4 x i64> @four_way_i16_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vsca
define <2 x i64> @eight_way_i8_i64_vl128(ptr %accptr, ptr %uptr, ptr %sptr) {
;
-; NEON-LABEL: eight_way_i8_i64_vl128:
-; NEON: // %bb.0:
-; NEON-NEXT: movi v0.2d, #0000000000000000
-; NEON-NEXT: ldr q1, [x1]
-; NEON-NEXT: ldr q2, [x2]
-; NEON-NEXT: udot v0.4s, v2.16b, v1.16b
-; NEON-NEXT: ldr q1, [x0]
-; NEON-NEXT: uaddw v1.2d, v1.2d, v0.2s
-; NEON-NEXT: uaddw2 v0.2d, v1.2d, v0.4s
-; NEON-NEXT: ret
-;
-; SVE-LABEL: eight_way_i8_i64_vl128:
-; SVE: // %bb.0:
-; SVE-NEXT: movi v0.2d, #0000000000000000
-; SVE-NEXT: ldr q1, [x1]
-; SVE-NEXT: ldr q2, [x2]
-; SVE-NEXT: udot z0.s, z2.b, z1.b
-; SVE-NEXT: ldr q2, [x0]
-; SVE-NEXT: uunpklo z1.d, z0.s
-; SVE-NEXT: uunpkhi z0.d, z0.s
-; SVE-NEXT: add z1.d, z2.d, z1.d
-; SVE-NEXT: add z0.d, z1.d, z0.d
-; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
-; SVE-NEXT: ret
+; COMMON-LABEL: eight_way_i8_i64_vl128:
+; COMMON: // %bb.0:
+; COMMON-NEXT: movi v0.2d, #0000000000000000
+; COMMON-NEXT: ldr q1, [x1]
+; COMMON-NEXT: ldr q2, [x2]
+; COMMON-NEXT: udot v0.4s, v2.16b, v1.16b
+; COMMON-NEXT: ldr q1, [x0]
+; COMMON-NEXT: uaddw v1.2d, v1.2d, v0.2s
+; COMMON-NEXT: uaddw2 v0.2d, v1.2d, v0.4s
+; COMMON-NEXT: ret
;
; SME-LABEL: eight_way_i8_i64_vl128:
; SME: // %bb.0:
@@ -859,43 +831,60 @@ define <2 x i64> @eight_way_i8_i64_vl128(ptr %accptr, ptr %uptr, ptr %sptr) {
ret <2 x i64> %partial.reduce
}
-define <4 x i64> @four_way_i8_i64_vl128_double_width(ptr %accptr, ptr %uptr, ptr %sptr) {
+; Regression test for the off-diagonal case: a 128-bit (<2 x i64>) result at
+; VL=256, where the fixed result width is smaller than the SVE vector length.
+; Before the fix the SVE run lowered via a scalable partial reduction whose
+; sums were spread across 4 d-lanes and then truncated to the low 2, dropping
+; half the count at runtime. The fix keeps it on the VL-independent NEON path.
+define <2 x i64> @eight_way_i8_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscale_range(2,2) {
;
-; NEON-LABEL: four_way_i8_i64_vl128_double_width:
-; NEON: // %bb.0:
-; NEON-NEXT: movi v1.2d, #0000000000000000
-; NEON-NEXT: movi v0.2d, #0000000000000000
-; NEON-NEXT: ldp q3, q2, [x1]
-; NEON-NEXT: ldp q5, q4, [x2]
-; NEON-NEXT: udot v0.4s, v5.16b, v3.16b
-; NEON-NEXT: udot v1.4s, v4.16b, v2.16b
-; NEON-NEXT: ldp q3, q2, [x0]
-; NEON-NEXT: uaddw v3.2d, v3.2d, v0.2s
-; NEON-NEXT: uaddw v2.2d, v2.2d, v1.2s
-; NEON-NEXT: uaddw2 v0.2d, v3.2d, v0.4s
-; NEON-NEXT: uaddw2 v1.2d, v2.2d, v1.4s
-; NEON-NEXT: ret
+; COMMON-LABEL: eight_way_i8_i64_vl256:
+; COMMON: // %bb.0:
+; COMMON-NEXT: movi v0.2d, #0000000000000000
+; COMMON-NEXT: ldr q1, [x1]
+; COMMON-NEXT: ldr q2, [x2]
+; COMMON-NEXT: udot v0.4s, v2.16b, v1.16b
+; COMMON-NEXT: ldr q1, [x0]
+; COMMON-NEXT: uaddw v1.2d, v1.2d, v0.2s
+; COMMON-NEXT: uaddw2 v0.2d, v1.2d, v0.4s
+; COMMON-NEXT: ret
;
-; SVE-LABEL: four_way_i8_i64_vl128_double_width:
-; SVE: // %bb.0:
-; SVE-NEXT: movi v0.2d, #0000000000000000
-; SVE-NEXT: movi v1.2d, #0000000000000000
-; SVE-NEXT: ldp q3, q2, [x1]
-; SVE-NEXT: ldp q5, q4, [x2]
-; SVE-NEXT: udot z1.s, z5.b, z3.b
-; SVE-NEXT: udot z0.s, z4.b, z2.b
-; SVE-NEXT: ldp q5, q4, [x0]
-; SVE-NEXT: uunpklo z2.d, z1.s
-; SVE-NEXT: uunpklo z3.d, z0.s
-; SVE-NEXT: uunpkhi z1.d, z1.s
-; SVE-NEXT: uunpkhi z6.d, z0.s
-; SVE-NEXT: add z0.d, z5.d, z2.d
-; SVE-NEXT: add z2.d, z4.d, z3.d
-; SVE-NEXT: add z0.d, z0.d, z1.d
-; SVE-NEXT: add z1.d, z2.d, z6.d
-; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
-; SVE-NEXT: // kill: def $q1 killed $q1 killed $z1
-; SVE-NEXT: ret
+; SME-LABEL: eight_way_i8_i64_vl256:
+; SME: // %bb.0:
+; SME-NEXT: mov z0.s, #0 // =0x0
+; SME-NEXT: ldr q1, [x1]
+; SME-NEXT: ldr q2, [x2]
+; SME-NEXT: udot z0.s, z2.b, z1.b
+; SME-NEXT: ldr q1, [x0]
+; SME-NEXT: uaddwb z1.d, z1.d, z0.s
+; SME-NEXT: uaddwt z0.d, z1.d, z0.s
+; SME-NEXT: ret
+ %acc = load <2 x i64>, ptr %accptr
+ %u = load <16 x i8>, ptr %uptr
+ %s = load <16 x i8>, ptr %sptr
+ %u.wide = zext <16 x i8> %u to <16 x i64>
+ %s.wide = zext <16 x i8> %s to <16 x i64>
+ %mult = mul nuw nsw <16 x i64> %s.wide, %u.wide
+ %partial.reduce = tail call <2 x i64> @llvm.vector.partial.reduce.add(<2 x i64> %acc, <16 x i64> %mult)
+ ret <2 x i64> %partial.reduce
+}
+
+define <4 x i64> @four_way_i8_i64_vl128_double_width(ptr %accptr, ptr %uptr, ptr %sptr) {
+;
+; COMMON-LABEL: four_way_i8_i64_vl128_double_width:
+; COMMON: // %bb.0:
+; COMMON-NEXT: movi v1.2d, #0000000000000000
+; COMMON-NEXT: movi v0.2d, #0000000000000000
+; COMMON-NEXT: ldp q3, q2, [x1]
+; COMMON-NEXT: ldp q5, q4, [x2]
+; COMMON-NEXT: udot v0.4s, v5.16b, v3.16b
+; COMMON-NEXT: udot v1.4s, v4.16b, v2.16b
+; COMMON-NEXT: ldp q3, q2, [x0]
+; COMMON-NEXT: uaddw v3.2d, v3.2d, v0.2s
+; COMMON-NEXT: uaddw v2.2d, v2.2d, v1.2s
+; COMMON-NEXT: uaddw2 v0.2d, v3.2d, v0.4s
+; COMMON-NEXT: uaddw2 v1.2d, v2.2d, v1.4s
+; COMMON-NEXT: ret
;
; SME-LABEL: four_way_i8_i64_vl128_double_width:
; SME: // %bb.0:
>From 88b0f0c6a223a6b7c0fa8b62f1b5419ea1053b51 Mon Sep 17 00:00:00 2001
From: Matt Blewitt <mble at planetscale.com>
Date: Mon, 22 Jun 2026 12:52:54 +0100
Subject: [PATCH 2/6] [AArch64] Rework: convert dot from scalable before
splitting
Per review (MacDue): keep the scalable dot and convert it back to
fixed-length v4i32 before splitting/widening, instead of carving the
whole op out to a NEON fixed-length path. Scoped to the v2i64 <- v16i8
case; wider results stay on the scalable path.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 46 +++--
.../sve-fixed-length-partial-reduce.ll | 189 ++++++++++++------
2 files changed, 158 insertions(+), 77 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 34a8a417192ce..f23268d50ed1d 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -33411,6 +33411,7 @@ AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
EVT ResultVT = Op.getValueType();
EVT OrigResultVT = ResultVT;
EVT OpVT = LHS.getValueType();
+ EVT OrigOpVT = OpVT;
// We can handle this case natively by accumulating into a wider
// zero-padded vector.
@@ -33445,19 +33446,11 @@ AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
return DAG.getNode(ISD::SUB, DL, ResultVT, BiasedDot, BiasCorrection);
}
- // Keep a fixed-length i8 -> i64 reduction fixed-length when NEON is
- // available. The scalable path spreads the sums across all VL-dependent lanes
- // but only extracts the low fixed-width lanes, dropping the high lanes for VL
- // > the fixed width (e.g. 256-bit Neoverse V1). i64 sibling of the v16i8 ->
- // v2i32 case fixed in PR #177119 / issue #176954.
- bool KeepFixedI8ToI64 = Subtarget->isNeonAvailable() &&
- ResultVT == MVT::v2i64 && OpVT == MVT::v16i8;
-
bool ConvertToScalable =
ResultVT.isFixedLengthVector() &&
- useSVEForFixedLengthVectorVT(ResultVT, /*OverrideNEON=*/true) &&
- !KeepFixedI8ToI64;
+ useSVEForFixedLengthVectorVT(ResultVT, /*OverrideNEON=*/true);
+ SDValue OrigAcc = Acc;
if (ConvertToScalable) {
ResultVT = getContainerForFixedLengthVector(DAG, ResultVT);
OpVT = getContainerForFixedLengthVector(DAG, LHS.getValueType());
@@ -33479,10 +33472,35 @@ AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
SDValue Res;
bool IsUnsigned = Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA;
- // UADDW{B,T}/SADDW{B,T} are scalable-only; a fixed-length result (the i8 ->
- // i64 case kept fixed above) must use the NEON widening-add path below.
- if (ResultVT.isScalableVector() &&
- (Subtarget->hasSVE2() || Subtarget->isStreamingSVEAvailable())) {
+
+ // A 128-bit v2i64 <- v16i8 reduction must produce its result in fixed-length
+ // v2i64. The scalable fold below (UADDW{B,T}/SADDW{B,T} or the scalable
+ // split) spreads the sums across all VL/64 lanes, and the trailing extract
+ // then keeps only the low two, silently dropping the rest for any VL > 128
+ // (e.g. 256-bit Neoverse V1). The dot itself is VL-independent: its
+ // meaningful sums occupy the low four i32 lanes regardless of VL (the wider
+ // input lanes are zero-padded), so convert it back to fixed-length v4i32 and
+ // do the split, widen and accumulate in fixed-length. Wider fixed-length
+ // results (e.g. v4i64) are vector-length pinned, so the scalable fold keeps
+ // all their lanes and stays correct. i64 sibling of the v16i8 -> v2i32 case
+ // fixed in PR #177119 / issue #176954.
+ if (OrigResultVT == MVT::v2i64 && OrigOpVT == MVT::v16i8) {
+ SDValue FixedDot =
+ ConvertToScalable ? convertFromScalableVector(DAG, MVT::v4i32, DotNode)
+ : DotNode;
+ auto [DotNodeLo, DotNodeHi] = DAG.SplitVector(FixedDot, DL);
+ if (IsUnsigned) {
+ DotNodeLo = DAG.getZExtOrTrunc(DotNodeLo, DL, OrigResultVT);
+ DotNodeHi = DAG.getZExtOrTrunc(DotNodeHi, DL, OrigResultVT);
+ } else {
+ DotNodeLo = DAG.getSExtOrTrunc(DotNodeLo, DL, OrigResultVT);
+ DotNodeHi = DAG.getSExtOrTrunc(DotNodeHi, DL, OrigResultVT);
+ }
+ SDValue Lo = DAG.getNode(ISD::ADD, DL, OrigResultVT, OrigAcc, DotNodeLo);
+ return DAG.getNode(ISD::ADD, DL, OrigResultVT, Lo, DotNodeHi);
+ }
+
+ if (Subtarget->hasSVE2() || Subtarget->isStreamingSVEAvailable()) {
unsigned LoOpcode = IsUnsigned ? AArch64ISD::UADDWB : AArch64ISD::SADDWB;
unsigned HiOpcode = IsUnsigned ? AArch64ISD::UADDWT : AArch64ISD::SADDWT;
SDValue Lo = DAG.getNode(LoOpcode, DL, ResultVT, Acc, DotNode);
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
index e80e2e795c47c..e38457579387d 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
@@ -444,16 +444,27 @@ define <4 x i32> @four_way_i8_i32_vl128_sudot(ptr %accptr, ptr %uptr, ptr %sptr)
}
define <2 x i64> @four_way_i8_i64_vl128_usdot(ptr %accptr, ptr %uptr, ptr %sptr) {
-; COMMON-LABEL: four_way_i8_i64_vl128_usdot:
-; COMMON: // %bb.0:
-; COMMON-NEXT: movi v0.2d, #0000000000000000
-; COMMON-NEXT: ldr q1, [x1]
-; COMMON-NEXT: ldr q2, [x2]
-; COMMON-NEXT: usdot v0.4s, v1.16b, v2.16b
-; COMMON-NEXT: ldr q1, [x0]
-; COMMON-NEXT: saddw v1.2d, v1.2d, v0.2s
-; COMMON-NEXT: saddw2 v0.2d, v1.2d, v0.4s
-; COMMON-NEXT: ret
+; NEON-LABEL: four_way_i8_i64_vl128_usdot:
+; NEON: // %bb.0:
+; NEON-NEXT: movi v0.2d, #0000000000000000
+; NEON-NEXT: ldr q1, [x1]
+; NEON-NEXT: ldr q2, [x2]
+; NEON-NEXT: usdot v0.4s, v1.16b, v2.16b
+; NEON-NEXT: ldr q1, [x0]
+; NEON-NEXT: saddw v1.2d, v1.2d, v0.2s
+; NEON-NEXT: saddw2 v0.2d, v1.2d, v0.4s
+; NEON-NEXT: ret
+;
+; SVE-LABEL: four_way_i8_i64_vl128_usdot:
+; SVE: // %bb.0:
+; SVE-NEXT: movi v0.2d, #0000000000000000
+; SVE-NEXT: ldr q1, [x1]
+; SVE-NEXT: ldr q2, [x2]
+; SVE-NEXT: usdot z0.s, z1.b, z2.b
+; SVE-NEXT: ldr q1, [x0]
+; SVE-NEXT: saddw v1.2d, v1.2d, v0.2s
+; SVE-NEXT: saddw2 v0.2d, v1.2d, v0.4s
+; SVE-NEXT: ret
;
; SME-LABEL: four_way_i8_i64_vl128_usdot:
; SME: // %bb.0:
@@ -461,9 +472,12 @@ define <2 x i64> @four_way_i8_i64_vl128_usdot(ptr %accptr, ptr %uptr, ptr %sptr)
; SME-NEXT: ldr q1, [x1]
; SME-NEXT: ldr q2, [x2]
; SME-NEXT: usdot z0.s, z1.b, z2.b
-; SME-NEXT: ldr q1, [x0]
-; SME-NEXT: saddwb z1.d, z1.d, z0.s
-; SME-NEXT: saddwt z0.d, z1.d, z0.s
+; SME-NEXT: ldr q2, [x0]
+; SME-NEXT: sunpklo z1.d, z0.s
+; SME-NEXT: ext z0.b, z0.b, z0.b, #8
+; SME-NEXT: sunpklo z0.d, z0.s
+; SME-NEXT: add z1.d, z2.d, z1.d
+; SME-NEXT: add z0.d, z1.d, z0.d
; SME-NEXT: ret
%acc = load <2 x i64>, ptr %accptr
%u = load <16 x i8>, ptr %uptr
@@ -800,16 +814,27 @@ define <4 x i64> @four_way_i16_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vsca
define <2 x i64> @eight_way_i8_i64_vl128(ptr %accptr, ptr %uptr, ptr %sptr) {
;
-; COMMON-LABEL: eight_way_i8_i64_vl128:
-; COMMON: // %bb.0:
-; COMMON-NEXT: movi v0.2d, #0000000000000000
-; COMMON-NEXT: ldr q1, [x1]
-; COMMON-NEXT: ldr q2, [x2]
-; COMMON-NEXT: udot v0.4s, v2.16b, v1.16b
-; COMMON-NEXT: ldr q1, [x0]
-; COMMON-NEXT: uaddw v1.2d, v1.2d, v0.2s
-; COMMON-NEXT: uaddw2 v0.2d, v1.2d, v0.4s
-; COMMON-NEXT: ret
+; NEON-LABEL: eight_way_i8_i64_vl128:
+; NEON: // %bb.0:
+; NEON-NEXT: movi v0.2d, #0000000000000000
+; NEON-NEXT: ldr q1, [x1]
+; NEON-NEXT: ldr q2, [x2]
+; NEON-NEXT: udot v0.4s, v2.16b, v1.16b
+; NEON-NEXT: ldr q1, [x0]
+; NEON-NEXT: uaddw v1.2d, v1.2d, v0.2s
+; NEON-NEXT: uaddw2 v0.2d, v1.2d, v0.4s
+; NEON-NEXT: ret
+;
+; SVE-LABEL: eight_way_i8_i64_vl128:
+; SVE: // %bb.0:
+; SVE-NEXT: movi v0.2d, #0000000000000000
+; SVE-NEXT: ldr q1, [x1]
+; SVE-NEXT: ldr q2, [x2]
+; SVE-NEXT: udot z0.s, z2.b, z1.b
+; SVE-NEXT: ldr q1, [x0]
+; SVE-NEXT: uaddw v1.2d, v1.2d, v0.2s
+; SVE-NEXT: uaddw2 v0.2d, v1.2d, v0.4s
+; SVE-NEXT: ret
;
; SME-LABEL: eight_way_i8_i64_vl128:
; SME: // %bb.0:
@@ -817,9 +842,12 @@ define <2 x i64> @eight_way_i8_i64_vl128(ptr %accptr, ptr %uptr, ptr %sptr) {
; SME-NEXT: ldr q1, [x1]
; SME-NEXT: ldr q2, [x2]
; SME-NEXT: udot z0.s, z2.b, z1.b
-; SME-NEXT: ldr q1, [x0]
-; SME-NEXT: uaddwb z1.d, z1.d, z0.s
-; SME-NEXT: uaddwt z0.d, z1.d, z0.s
+; SME-NEXT: ldr q2, [x0]
+; SME-NEXT: uunpklo z1.d, z0.s
+; SME-NEXT: ext z0.b, z0.b, z0.b, #8
+; SME-NEXT: uunpklo z0.d, z0.s
+; SME-NEXT: add z1.d, z2.d, z1.d
+; SME-NEXT: add z0.d, z1.d, z0.d
; SME-NEXT: ret
%acc = load <2 x i64>, ptr %accptr
%u = load <16 x i8>, ptr %uptr
@@ -838,16 +866,27 @@ define <2 x i64> @eight_way_i8_i64_vl128(ptr %accptr, ptr %uptr, ptr %sptr) {
; half the count at runtime. The fix keeps it on the VL-independent NEON path.
define <2 x i64> @eight_way_i8_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscale_range(2,2) {
;
-; COMMON-LABEL: eight_way_i8_i64_vl256:
-; COMMON: // %bb.0:
-; COMMON-NEXT: movi v0.2d, #0000000000000000
-; COMMON-NEXT: ldr q1, [x1]
-; COMMON-NEXT: ldr q2, [x2]
-; COMMON-NEXT: udot v0.4s, v2.16b, v1.16b
-; COMMON-NEXT: ldr q1, [x0]
-; COMMON-NEXT: uaddw v1.2d, v1.2d, v0.2s
-; COMMON-NEXT: uaddw2 v0.2d, v1.2d, v0.4s
-; COMMON-NEXT: ret
+; NEON-LABEL: eight_way_i8_i64_vl256:
+; NEON: // %bb.0:
+; NEON-NEXT: movi v0.2d, #0000000000000000
+; NEON-NEXT: ldr q1, [x1]
+; NEON-NEXT: ldr q2, [x2]
+; NEON-NEXT: udot v0.4s, v2.16b, v1.16b
+; NEON-NEXT: ldr q1, [x0]
+; NEON-NEXT: uaddw v1.2d, v1.2d, v0.2s
+; NEON-NEXT: uaddw2 v0.2d, v1.2d, v0.4s
+; NEON-NEXT: ret
+;
+; SVE-LABEL: eight_way_i8_i64_vl256:
+; SVE: // %bb.0:
+; SVE-NEXT: movi v0.2d, #0000000000000000
+; SVE-NEXT: ldr q1, [x1]
+; SVE-NEXT: ldr q2, [x2]
+; SVE-NEXT: udot z0.s, z2.b, z1.b
+; SVE-NEXT: ldr q1, [x0]
+; SVE-NEXT: uaddw v1.2d, v1.2d, v0.2s
+; SVE-NEXT: uaddw2 v0.2d, v1.2d, v0.4s
+; SVE-NEXT: ret
;
; SME-LABEL: eight_way_i8_i64_vl256:
; SME: // %bb.0:
@@ -855,9 +894,12 @@ define <2 x i64> @eight_way_i8_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vsca
; SME-NEXT: ldr q1, [x1]
; SME-NEXT: ldr q2, [x2]
; SME-NEXT: udot z0.s, z2.b, z1.b
-; SME-NEXT: ldr q1, [x0]
-; SME-NEXT: uaddwb z1.d, z1.d, z0.s
-; SME-NEXT: uaddwt z0.d, z1.d, z0.s
+; SME-NEXT: ldr q2, [x0]
+; SME-NEXT: uunpklo z1.d, z0.s
+; SME-NEXT: ext z0.b, z0.b, z0.b, #8
+; SME-NEXT: uunpklo z0.d, z0.s
+; SME-NEXT: add z1.d, z2.d, z1.d
+; SME-NEXT: add z0.d, z1.d, z0.d
; SME-NEXT: ret
%acc = load <2 x i64>, ptr %accptr
%u = load <16 x i8>, ptr %uptr
@@ -871,34 +913,55 @@ define <2 x i64> @eight_way_i8_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vsca
define <4 x i64> @four_way_i8_i64_vl128_double_width(ptr %accptr, ptr %uptr, ptr %sptr) {
;
-; COMMON-LABEL: four_way_i8_i64_vl128_double_width:
-; COMMON: // %bb.0:
-; COMMON-NEXT: movi v1.2d, #0000000000000000
-; COMMON-NEXT: movi v0.2d, #0000000000000000
-; COMMON-NEXT: ldp q3, q2, [x1]
-; COMMON-NEXT: ldp q5, q4, [x2]
-; COMMON-NEXT: udot v0.4s, v5.16b, v3.16b
-; COMMON-NEXT: udot v1.4s, v4.16b, v2.16b
-; COMMON-NEXT: ldp q3, q2, [x0]
-; COMMON-NEXT: uaddw v3.2d, v3.2d, v0.2s
-; COMMON-NEXT: uaddw v2.2d, v2.2d, v1.2s
-; COMMON-NEXT: uaddw2 v0.2d, v3.2d, v0.4s
-; COMMON-NEXT: uaddw2 v1.2d, v2.2d, v1.4s
-; COMMON-NEXT: ret
+; NEON-LABEL: four_way_i8_i64_vl128_double_width:
+; NEON: // %bb.0:
+; NEON-NEXT: movi v1.2d, #0000000000000000
+; NEON-NEXT: movi v0.2d, #0000000000000000
+; NEON-NEXT: ldp q3, q2, [x1]
+; NEON-NEXT: ldp q5, q4, [x2]
+; NEON-NEXT: udot v0.4s, v5.16b, v3.16b
+; NEON-NEXT: udot v1.4s, v4.16b, v2.16b
+; NEON-NEXT: ldp q3, q2, [x0]
+; NEON-NEXT: uaddw v3.2d, v3.2d, v0.2s
+; NEON-NEXT: uaddw v2.2d, v2.2d, v1.2s
+; NEON-NEXT: uaddw2 v0.2d, v3.2d, v0.4s
+; NEON-NEXT: uaddw2 v1.2d, v2.2d, v1.4s
+; NEON-NEXT: ret
+;
+; SVE-LABEL: four_way_i8_i64_vl128_double_width:
+; SVE: // %bb.0:
+; SVE-NEXT: movi v1.2d, #0000000000000000
+; SVE-NEXT: movi v0.2d, #0000000000000000
+; SVE-NEXT: ldp q3, q2, [x1]
+; SVE-NEXT: ldp q5, q4, [x2]
+; SVE-NEXT: udot z0.s, z5.b, z3.b
+; SVE-NEXT: udot z1.s, z4.b, z2.b
+; SVE-NEXT: ldp q3, q2, [x0]
+; SVE-NEXT: uaddw v3.2d, v3.2d, v0.2s
+; SVE-NEXT: uaddw v2.2d, v2.2d, v1.2s
+; SVE-NEXT: uaddw2 v0.2d, v3.2d, v0.4s
+; SVE-NEXT: uaddw2 v1.2d, v2.2d, v1.4s
+; SVE-NEXT: ret
;
; SME-LABEL: four_way_i8_i64_vl128_double_width:
; SME: // %bb.0:
-; SME-NEXT: mov z1.s, #0 // =0x0
; SME-NEXT: mov z0.s, #0 // =0x0
-; SME-NEXT: ldp q3, q2, [x1]
-; SME-NEXT: ldp q5, q4, [x2]
-; SME-NEXT: udot z0.s, z5.b, z3.b
-; SME-NEXT: udot z1.s, z4.b, z2.b
-; SME-NEXT: ldp q3, q2, [x0]
-; SME-NEXT: uaddwb z3.d, z3.d, z0.s
-; SME-NEXT: uaddwb z2.d, z2.d, z1.s
-; SME-NEXT: uaddwt z0.d, z3.d, z0.s
-; SME-NEXT: uaddwt z1.d, z2.d, z1.s
+; SME-NEXT: mov z1.s, #0 // =0x0
+; SME-NEXT: ldp q2, q5, [x2]
+; SME-NEXT: ldp q3, q4, [x1]
+; SME-NEXT: udot z0.s, z2.b, z3.b
+; SME-NEXT: udot z1.s, z5.b, z4.b
+; SME-NEXT: ldp q5, q4, [x0]
+; SME-NEXT: uunpklo z2.d, z0.s
+; SME-NEXT: ext z0.b, z0.b, z0.b, #8
+; SME-NEXT: uunpklo z3.d, z1.s
+; SME-NEXT: ext z1.b, z1.b, z1.b, #8
+; SME-NEXT: uunpklo z0.d, z0.s
+; SME-NEXT: uunpklo z1.d, z1.s
+; SME-NEXT: add z2.d, z5.d, z2.d
+; SME-NEXT: add z3.d, z4.d, z3.d
+; SME-NEXT: add z0.d, z2.d, z0.d
+; SME-NEXT: add z1.d, z3.d, z1.d
; SME-NEXT: ret
%acc = load <4 x i64>, ptr %accptr
%u = load <32 x i8>, ptr %uptr
>From 65f63a140db536b4280c2a3f33827789dfb73ec4 Mon Sep 17 00:00:00 2001
From: Matt Blewitt <mble at planetscale.com>
Date: Mon, 22 Jun 2026 12:52:55 +0100
Subject: [PATCH 3/6] [AArch64] Trim comments
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 16 +++++-----------
1 file changed, 5 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index f23268d50ed1d..8e423797d2acc 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -33473,17 +33473,11 @@ AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
SDValue Res;
bool IsUnsigned = Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA;
- // A 128-bit v2i64 <- v16i8 reduction must produce its result in fixed-length
- // v2i64. The scalable fold below (UADDW{B,T}/SADDW{B,T} or the scalable
- // split) spreads the sums across all VL/64 lanes, and the trailing extract
- // then keeps only the low two, silently dropping the rest for any VL > 128
- // (e.g. 256-bit Neoverse V1). The dot itself is VL-independent: its
- // meaningful sums occupy the low four i32 lanes regardless of VL (the wider
- // input lanes are zero-padded), so convert it back to fixed-length v4i32 and
- // do the split, widen and accumulate in fixed-length. Wider fixed-length
- // results (e.g. v4i64) are vector-length pinned, so the scalable fold keeps
- // all their lanes and stays correct. i64 sibling of the v16i8 -> v2i32 case
- // fixed in PR #177119 / issue #176954.
+ // The scalable fold below spreads the sums across all VL/64 lanes, so the
+ // trailing extract drops the high lanes of a 128-bit v2i64 result for VL >
+ // 128. The dot is VL-independent (sums in the low four i32 lanes), so fold it
+ // back in fixed-length. Wider results (e.g. v4i64) are VL-pinned and stay
+ // correct on the scalable path. See PR #177119 / issue #176954.
if (OrigResultVT == MVT::v2i64 && OrigOpVT == MVT::v16i8) {
SDValue FixedDot =
ConvertToScalable ? convertFromScalableVector(DAG, MVT::v4i32, DotNode)
>From b372872787dfdd402a887c51c892d06859f14114 Mon Sep 17 00:00:00 2001
From: Matt Blewitt <mble at planetscale.com>
Date: Mon, 22 Jun 2026 12:52:56 +0100
Subject: [PATCH 4/6] [AArch64] Generalize fold to all i64 widths; gate UADDW
to scalable
Drop the per-type special case: the fold derives the fixed dot width
from the result (2 i32 lanes per i64 lane), so v4i64 works too. Restrict
the UADDW{B,T}/SADDW{B,T} path to genuinely scalable results.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 68 ++++++++-----------
.../sve-fixed-length-partial-reduce.ll | 13 ++--
2 files changed, 38 insertions(+), 43 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 8e423797d2acc..fe38ef553645f 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -33411,7 +33411,6 @@ AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
EVT ResultVT = Op.getValueType();
EVT OrigResultVT = ResultVT;
EVT OpVT = LHS.getValueType();
- EVT OrigOpVT = OpVT;
// We can handle this case natively by accumulating into a wider
// zero-padded vector.
@@ -33470,51 +33469,42 @@ AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
SDValue DotNode = DAG.getNode(Op.getOpcode(), DL, DotVT,
DAG.getConstant(0, DL, DotVT), LHS, RHS);
- SDValue Res;
bool IsUnsigned = Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA;
- // The scalable fold below spreads the sums across all VL/64 lanes, so the
- // trailing extract drops the high lanes of a 128-bit v2i64 result for VL >
- // 128. The dot is VL-independent (sums in the low four i32 lanes), so fold it
- // back in fixed-length. Wider results (e.g. v4i64) are VL-pinned and stay
- // correct on the scalable path. See PR #177119 / issue #176954.
- if (OrigResultVT == MVT::v2i64 && OrigOpVT == MVT::v16i8) {
- SDValue FixedDot =
- ConvertToScalable ? convertFromScalableVector(DAG, MVT::v4i32, DotNode)
- : DotNode;
- auto [DotNodeLo, DotNodeHi] = DAG.SplitVector(FixedDot, DL);
- if (IsUnsigned) {
- DotNodeLo = DAG.getZExtOrTrunc(DotNodeLo, DL, OrigResultVT);
- DotNodeHi = DAG.getZExtOrTrunc(DotNodeHi, DL, OrigResultVT);
- } else {
- DotNodeLo = DAG.getSExtOrTrunc(DotNodeLo, DL, OrigResultVT);
- DotNodeHi = DAG.getSExtOrTrunc(DotNodeHi, DL, OrigResultVT);
- }
- SDValue Lo = DAG.getNode(ISD::ADD, DL, OrigResultVT, OrigAcc, DotNodeLo);
- return DAG.getNode(ISD::ADD, DL, OrigResultVT, Lo, DotNodeHi);
- }
-
- if (Subtarget->hasSVE2() || Subtarget->isStreamingSVEAvailable()) {
+ // UADDW{B,T}/SADDW{B,T} fold the dot in the scalable domain, spreading the
+ // sums across all VL/64 lanes. That is only valid for a genuinely scalable
+ // result; a fixed-length result must convert from the scalable container
+ // before splitting (below), else the trailing extract drops the high lanes
+ // for any VL > the fixed width.
+ if (OrigResultVT.isScalableVector() &&
+ (Subtarget->hasSVE2() || Subtarget->isStreamingSVEAvailable())) {
unsigned LoOpcode = IsUnsigned ? AArch64ISD::UADDWB : AArch64ISD::SADDWB;
unsigned HiOpcode = IsUnsigned ? AArch64ISD::UADDWT : AArch64ISD::SADDWT;
SDValue Lo = DAG.getNode(LoOpcode, DL, ResultVT, Acc, DotNode);
- Res = DAG.getNode(HiOpcode, DL, ResultVT, Lo, DotNode);
- } else {
- // Fold (nx)v4i32 into (nx)v2i64
- auto [DotNodeLo, DotNodeHi] = DAG.SplitVector(DotNode, DL);
- if (IsUnsigned) {
- DotNodeLo = DAG.getZExtOrTrunc(DotNodeLo, DL, ResultVT);
- DotNodeHi = DAG.getZExtOrTrunc(DotNodeHi, DL, ResultVT);
- } else {
- DotNodeLo = DAG.getSExtOrTrunc(DotNodeLo, DL, ResultVT);
- DotNodeHi = DAG.getSExtOrTrunc(DotNodeHi, DL, ResultVT);
- }
- auto Lo = DAG.getNode(ISD::ADD, DL, ResultVT, Acc, DotNodeLo);
- Res = DAG.getNode(ISD::ADD, DL, ResultVT, Lo, DotNodeHi);
+ return DAG.getNode(HiOpcode, DL, ResultVT, Lo, DotNode);
}
- return ConvertToScalable ? convertFromScalableVector(DAG, OrigResultVT, Res)
- : Res;
+ // Fold the (nx)v4i32 dot into the (nx)v2i64 result. For a fixed-length
+ // result, convert from the scalable container before splitting: the sums sit
+ // in the low i32 lanes regardless of VL, so splitting after the extract would
+ // drop a 128-bit result's high lanes for VL > 128. See PR #177119 / issue
+ // #176954.
+ SDValue FoldDot = DotNode;
+ if (ConvertToScalable) {
+ EVT FixedDotVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
+ OrigResultVT.getVectorNumElements() * 2);
+ FoldDot = convertFromScalableVector(DAG, FixedDotVT, DotNode);
+ }
+ auto [DotNodeLo, DotNodeHi] = DAG.SplitVector(FoldDot, DL);
+ if (IsUnsigned) {
+ DotNodeLo = DAG.getZExtOrTrunc(DotNodeLo, DL, OrigResultVT);
+ DotNodeHi = DAG.getZExtOrTrunc(DotNodeHi, DL, OrigResultVT);
+ } else {
+ DotNodeLo = DAG.getSExtOrTrunc(DotNodeLo, DL, OrigResultVT);
+ DotNodeHi = DAG.getSExtOrTrunc(DotNodeHi, DL, OrigResultVT);
+ }
+ SDValue Lo = DAG.getNode(ISD::ADD, DL, OrigResultVT, OrigAcc, DotNodeLo);
+ return DAG.getNode(ISD::ADD, DL, OrigResultVT, Lo, DotNodeHi);
}
SDValue
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
index e38457579387d..4a3c213e60cb3 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
@@ -997,7 +997,8 @@ define <4 x i64> @four_way_i8_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscal
; SVE-NEXT: udot z0.s, z2.b, z1.b
; SVE-NEXT: ldr z2, [x0]
; SVE-NEXT: uunpklo z1.d, z0.s
-; SVE-NEXT: uunpkhi z0.d, z0.s
+; SVE-NEXT: ext z0.b, z0.b, z0.b, #16
+; SVE-NEXT: uunpklo z0.d, z0.s
; SVE-NEXT: add z1.d, z2.d, z1.d
; SVE-NEXT: add z0.d, z1.d, z0.d
; SVE-NEXT: movprfx z1, z0
@@ -1012,9 +1013,13 @@ define <4 x i64> @four_way_i8_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscal
; SME-NEXT: ldr z1, [x2]
; SME-NEXT: mov z2.s, #0 // =0x0
; SME-NEXT: udot z2.s, z1.b, z0.b
-; SME-NEXT: ldr z0, [x0]
-; SME-NEXT: uaddwb z0.d, z0.d, z2.s
-; SME-NEXT: uaddwt z0.d, z0.d, z2.s
+; SME-NEXT: uunpklo z0.d, z2.s
+; SME-NEXT: movprfx z1, z2
+; SME-NEXT: ext z1.b, z1.b, z2.b, #16
+; SME-NEXT: ldr z2, [x0]
+; SME-NEXT: uunpklo z1.d, z1.s
+; SME-NEXT: add z0.d, z2.d, z0.d
+; SME-NEXT: add z0.d, z0.d, z1.d
; SME-NEXT: movprfx z1, z0
; SME-NEXT: ext z1.b, z1.b, z0.b, #16
; SME-NEXT: ret
>From 6001976e5a393376b5b6c7b591431884a593c114 Mon Sep 17 00:00:00 2001
From: Matt Blewitt <mble at planetscale.com>
Date: Sat, 20 Jun 2026 10:28:18 -0700
Subject: [PATCH 5/6] [AArch64] Fix v16i8 -> v2i64 partial_reduce for VL > 128
A fixed-length llvm.vector.partial.reduce.add with a <2 x i64>
accumulator and i8 inputs was lowered, on a +sve (non-SVE2,
non-streaming) target, through a scalable partial reduction whose nxv4i32
dot was split in the scalable domain and only then converted back to
fixed-length. Splitting the scalable dot puts its low VL/64 i32 lanes
into the low half, which then map to the high lanes of the fixed-length
result; the trailing extract drops those for any runtime vector length
greater than 128 bits. On a 256-bit machine (e.g. Neoverse V1) exactly
half the result is lost.
The dot product is VL-independent: the fixed-length input is zero-padded
into the scalable container, so its meaningful sums occupy the low i32
lanes regardless of VL. Fix the fold to convert the dot back to a
fixed-length i32 vector before splitting, so the split, widen and
accumulate all happen in fixed-length and no lanes are dropped. The
UADDW{B,T}/SADDW{B,T} path is unchanged: it folds even/odd lanes into the
low result lanes and is already correct for fixed and scalable results.
This is the i64 sibling of the v16i8 -> v2i32 case fixed in #177119
(issue #176954); the fixed-length support was introduced in #142032.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 24 +++----
.../sve-fixed-length-partial-reduce.ll | 63 +++++++------------
2 files changed, 32 insertions(+), 55 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index fe38ef553645f..775ac8431cf28 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -33406,6 +33406,7 @@ AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
SDLoc DL(Op);
SDValue Acc = Op.getOperand(0);
+ SDValue OrigAcc = Acc;
SDValue LHS = Op.getOperand(1);
SDValue RHS = Op.getOperand(2);
EVT ResultVT = Op.getValueType();
@@ -33449,7 +33450,6 @@ AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
ResultVT.isFixedLengthVector() &&
useSVEForFixedLengthVectorVT(ResultVT, /*OverrideNEON=*/true);
- SDValue OrigAcc = Acc;
if (ConvertToScalable) {
ResultVT = getContainerForFixedLengthVector(DAG, ResultVT);
OpVT = getContainerForFixedLengthVector(DAG, LHS.getValueType());
@@ -33471,26 +33471,22 @@ AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
bool IsUnsigned = Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA;
- // UADDW{B,T}/SADDW{B,T} fold the dot in the scalable domain, spreading the
- // sums across all VL/64 lanes. That is only valid for a genuinely scalable
- // result; a fixed-length result must convert from the scalable container
- // before splitting (below), else the trailing extract drops the high lanes
- // for any VL > the fixed width.
- if (OrigResultVT.isScalableVector() &&
- (Subtarget->hasSVE2() || Subtarget->isStreamingSVEAvailable())) {
+ if (Subtarget->hasSVE2() || Subtarget->isStreamingSVEAvailable()) {
unsigned LoOpcode = IsUnsigned ? AArch64ISD::UADDWB : AArch64ISD::SADDWB;
unsigned HiOpcode = IsUnsigned ? AArch64ISD::UADDWT : AArch64ISD::SADDWT;
SDValue Lo = DAG.getNode(LoOpcode, DL, ResultVT, Acc, DotNode);
- return DAG.getNode(HiOpcode, DL, ResultVT, Lo, DotNode);
+ SDValue Res = DAG.getNode(HiOpcode, DL, ResultVT, Lo, DotNode);
+ return ConvertToScalable ? convertFromScalableVector(DAG, OrigResultVT, Res)
+ : Res;
}
- // Fold the (nx)v4i32 dot into the (nx)v2i64 result. For a fixed-length
- // result, convert from the scalable container before splitting: the sums sit
- // in the low i32 lanes regardless of VL, so splitting after the extract would
- // drop a 128-bit result's high lanes for VL > 128. See PR #177119 / issue
- // #176954.
+ // Fold (nx)v4i32 into (nx)v2i64. Convert from scalable vectors before
+ // splitting: the dot sums sit in the low i32 lanes regardless of VL, so
+ // splitting after the extract would drop a fixed-length result's high lanes
+ // for VL > 128. See PR #177119 / issue #176954.
SDValue FoldDot = DotNode;
if (ConvertToScalable) {
+ // The dot holds two i32 lanes per i64 result lane.
EVT FixedDotVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
OrigResultVT.getVectorNumElements() * 2);
FoldDot = convertFromScalableVector(DAG, FixedDotVT, DotNode);
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
index 4a3c213e60cb3..c009aadd52c1e 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
@@ -472,12 +472,9 @@ define <2 x i64> @four_way_i8_i64_vl128_usdot(ptr %accptr, ptr %uptr, ptr %sptr)
; SME-NEXT: ldr q1, [x1]
; SME-NEXT: ldr q2, [x2]
; SME-NEXT: usdot z0.s, z1.b, z2.b
-; SME-NEXT: ldr q2, [x0]
-; SME-NEXT: sunpklo z1.d, z0.s
-; SME-NEXT: ext z0.b, z0.b, z0.b, #8
-; SME-NEXT: sunpklo z0.d, z0.s
-; SME-NEXT: add z1.d, z2.d, z1.d
-; SME-NEXT: add z0.d, z1.d, z0.d
+; SME-NEXT: ldr q1, [x0]
+; SME-NEXT: saddwb z1.d, z1.d, z0.s
+; SME-NEXT: saddwt z0.d, z1.d, z0.s
; SME-NEXT: ret
%acc = load <2 x i64>, ptr %accptr
%u = load <16 x i8>, ptr %uptr
@@ -842,12 +839,9 @@ define <2 x i64> @eight_way_i8_i64_vl128(ptr %accptr, ptr %uptr, ptr %sptr) {
; SME-NEXT: ldr q1, [x1]
; SME-NEXT: ldr q2, [x2]
; SME-NEXT: udot z0.s, z2.b, z1.b
-; SME-NEXT: ldr q2, [x0]
-; SME-NEXT: uunpklo z1.d, z0.s
-; SME-NEXT: ext z0.b, z0.b, z0.b, #8
-; SME-NEXT: uunpklo z0.d, z0.s
-; SME-NEXT: add z1.d, z2.d, z1.d
-; SME-NEXT: add z0.d, z1.d, z0.d
+; SME-NEXT: ldr q1, [x0]
+; SME-NEXT: uaddwb z1.d, z1.d, z0.s
+; SME-NEXT: uaddwt z0.d, z1.d, z0.s
; SME-NEXT: ret
%acc = load <2 x i64>, ptr %accptr
%u = load <16 x i8>, ptr %uptr
@@ -894,12 +888,9 @@ define <2 x i64> @eight_way_i8_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vsca
; SME-NEXT: ldr q1, [x1]
; SME-NEXT: ldr q2, [x2]
; SME-NEXT: udot z0.s, z2.b, z1.b
-; SME-NEXT: ldr q2, [x0]
-; SME-NEXT: uunpklo z1.d, z0.s
-; SME-NEXT: ext z0.b, z0.b, z0.b, #8
-; SME-NEXT: uunpklo z0.d, z0.s
-; SME-NEXT: add z1.d, z2.d, z1.d
-; SME-NEXT: add z0.d, z1.d, z0.d
+; SME-NEXT: ldr q1, [x0]
+; SME-NEXT: uaddwb z1.d, z1.d, z0.s
+; SME-NEXT: uaddwt z0.d, z1.d, z0.s
; SME-NEXT: ret
%acc = load <2 x i64>, ptr %accptr
%u = load <16 x i8>, ptr %uptr
@@ -945,23 +936,17 @@ define <4 x i64> @four_way_i8_i64_vl128_double_width(ptr %accptr, ptr %uptr, ptr
;
; SME-LABEL: four_way_i8_i64_vl128_double_width:
; SME: // %bb.0:
-; SME-NEXT: mov z0.s, #0 // =0x0
; SME-NEXT: mov z1.s, #0 // =0x0
-; SME-NEXT: ldp q2, q5, [x2]
-; SME-NEXT: ldp q3, q4, [x1]
-; SME-NEXT: udot z0.s, z2.b, z3.b
-; SME-NEXT: udot z1.s, z5.b, z4.b
-; SME-NEXT: ldp q5, q4, [x0]
-; SME-NEXT: uunpklo z2.d, z0.s
-; SME-NEXT: ext z0.b, z0.b, z0.b, #8
-; SME-NEXT: uunpklo z3.d, z1.s
-; SME-NEXT: ext z1.b, z1.b, z1.b, #8
-; SME-NEXT: uunpklo z0.d, z0.s
-; SME-NEXT: uunpklo z1.d, z1.s
-; SME-NEXT: add z2.d, z5.d, z2.d
-; SME-NEXT: add z3.d, z4.d, z3.d
-; SME-NEXT: add z0.d, z2.d, z0.d
-; SME-NEXT: add z1.d, z3.d, z1.d
+; SME-NEXT: mov z0.s, #0 // =0x0
+; SME-NEXT: ldp q3, q2, [x1]
+; SME-NEXT: ldp q5, q4, [x2]
+; SME-NEXT: udot z0.s, z5.b, z3.b
+; SME-NEXT: udot z1.s, z4.b, z2.b
+; SME-NEXT: ldp q3, q2, [x0]
+; SME-NEXT: uaddwb z3.d, z3.d, z0.s
+; SME-NEXT: uaddwb z2.d, z2.d, z1.s
+; SME-NEXT: uaddwt z0.d, z3.d, z0.s
+; SME-NEXT: uaddwt z1.d, z2.d, z1.s
; SME-NEXT: ret
%acc = load <4 x i64>, ptr %accptr
%u = load <32 x i8>, ptr %uptr
@@ -1013,13 +998,9 @@ define <4 x i64> @four_way_i8_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscal
; SME-NEXT: ldr z1, [x2]
; SME-NEXT: mov z2.s, #0 // =0x0
; SME-NEXT: udot z2.s, z1.b, z0.b
-; SME-NEXT: uunpklo z0.d, z2.s
-; SME-NEXT: movprfx z1, z2
-; SME-NEXT: ext z1.b, z1.b, z2.b, #16
-; SME-NEXT: ldr z2, [x0]
-; SME-NEXT: uunpklo z1.d, z1.s
-; SME-NEXT: add z0.d, z2.d, z0.d
-; SME-NEXT: add z0.d, z0.d, z1.d
+; SME-NEXT: ldr z0, [x0]
+; SME-NEXT: uaddwb z0.d, z0.d, z2.s
+; SME-NEXT: uaddwt z0.d, z0.d, z2.s
; SME-NEXT: movprfx z1, z0
; SME-NEXT: ext z1.b, z1.b, z0.b, #16
; SME-NEXT: ret
>From 5a9e4661f8b8e5d7538800c4241a09feb428aee6 Mon Sep 17 00:00:00 2001
From: Matt Blewitt <mble at planetscale.com>
Date: Tue, 30 Jun 2026 11:17:12 +0100
Subject: [PATCH 6/6] [AArch64] Add +sve2 coverage for i8 -> i64 partial_reduce
Add an explicit +sve2 RUN line: the fixed-length i8 -> i64 cases take the
UADDW{B,T}/SADDW{B,T} path there, which folds adjacent i32 lane pairs into
the low result lanes and is correct for fixed-length results (only the
SplitVector fold dropped lanes). Previously this was exercised only via
the streaming SME run.
Document four_way_i8_i64_vl256 as the regression guard for the dynamic
fixed-dot width: on plain SVE this VL-pinned <4 x i64> reaches the fold
with an eight-lane nxv4i32 dot, so the fixed dot width must scale with
the result; a static v4i32 asserts on the v2i32 -> v4i64 extend.
---
.../sve-fixed-length-partial-reduce.ll | 566 +++++++++++++++---
1 file changed, 467 insertions(+), 99 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
index c009aadd52c1e..5d2787610f4ad 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
@@ -1,6 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mattr=+dotprod,+i8mm < %s | FileCheck %s --check-prefixes=COMMON,NEON
; RUN: llc -mattr=+sve,+dotprod,+i8mm < %s | FileCheck %s --check-prefixes=COMMON,SVE
+; RUN: llc -mattr=+sve2,+dotprod,+i8mm < %s | FileCheck %s --check-prefixes=COMMON,SVE2
; RUN: llc -mattr=+sme,+i8mm -force-streaming < %s | FileCheck %s --check-prefix=SME
target triple = "aarch64"
@@ -11,14 +12,33 @@ target triple = "aarch64"
define <8 x i16> @two_way_i8_i16_vl128(ptr %accptr, ptr %uptr, ptr %sptr) {
;
-; COMMON-LABEL: two_way_i8_i16_vl128:
-; COMMON: // %bb.0:
-; COMMON-NEXT: ldr q0, [x0]
-; COMMON-NEXT: ldr q1, [x1]
-; COMMON-NEXT: ldr q2, [x2]
-; COMMON-NEXT: umlal v0.8h, v2.8b, v1.8b
-; COMMON-NEXT: umlal2 v0.8h, v2.16b, v1.16b
-; COMMON-NEXT: ret
+; NEON-LABEL: two_way_i8_i16_vl128:
+; NEON: // %bb.0:
+; NEON-NEXT: ldr q0, [x0]
+; NEON-NEXT: ldr q1, [x1]
+; NEON-NEXT: ldr q2, [x2]
+; NEON-NEXT: umlal v0.8h, v2.8b, v1.8b
+; NEON-NEXT: umlal2 v0.8h, v2.16b, v1.16b
+; NEON-NEXT: ret
+;
+; SVE-LABEL: two_way_i8_i16_vl128:
+; SVE: // %bb.0:
+; SVE-NEXT: ldr q0, [x0]
+; SVE-NEXT: ldr q1, [x1]
+; SVE-NEXT: ldr q2, [x2]
+; SVE-NEXT: umlal v0.8h, v2.8b, v1.8b
+; SVE-NEXT: umlal2 v0.8h, v2.16b, v1.16b
+; SVE-NEXT: ret
+;
+; SVE2-LABEL: two_way_i8_i16_vl128:
+; SVE2: // %bb.0:
+; SVE2-NEXT: ldr q0, [x0]
+; SVE2-NEXT: ldr q1, [x1]
+; SVE2-NEXT: ldr q2, [x2]
+; SVE2-NEXT: umlalb z0.h, z2.b, z1.b
+; SVE2-NEXT: umlalt z0.h, z2.b, z1.b
+; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT: ret
;
; SME-LABEL: two_way_i8_i16_vl128:
; SME: // %bb.0:
@@ -40,16 +60,40 @@ define <8 x i16> @two_way_i8_i16_vl128(ptr %accptr, ptr %uptr, ptr %sptr) {
define <16 x i16> @two_way_i8_i16_vl128_double_width(ptr %accptr, ptr %uptr, ptr %sptr) {
;
-; COMMON-LABEL: two_way_i8_i16_vl128_double_width:
-; COMMON: // %bb.0:
-; COMMON-NEXT: ldp q0, q1, [x0]
-; COMMON-NEXT: ldp q2, q3, [x1]
-; COMMON-NEXT: ldp q4, q5, [x2]
-; COMMON-NEXT: umlal v0.8h, v4.8b, v2.8b
-; COMMON-NEXT: umlal v1.8h, v5.8b, v3.8b
-; COMMON-NEXT: umlal2 v0.8h, v4.16b, v2.16b
-; COMMON-NEXT: umlal2 v1.8h, v5.16b, v3.16b
-; COMMON-NEXT: ret
+; NEON-LABEL: two_way_i8_i16_vl128_double_width:
+; NEON: // %bb.0:
+; NEON-NEXT: ldp q0, q1, [x0]
+; NEON-NEXT: ldp q2, q3, [x1]
+; NEON-NEXT: ldp q4, q5, [x2]
+; NEON-NEXT: umlal v0.8h, v4.8b, v2.8b
+; NEON-NEXT: umlal v1.8h, v5.8b, v3.8b
+; NEON-NEXT: umlal2 v0.8h, v4.16b, v2.16b
+; NEON-NEXT: umlal2 v1.8h, v5.16b, v3.16b
+; NEON-NEXT: ret
+;
+; SVE-LABEL: two_way_i8_i16_vl128_double_width:
+; SVE: // %bb.0:
+; SVE-NEXT: ldp q0, q1, [x0]
+; SVE-NEXT: ldp q2, q3, [x1]
+; SVE-NEXT: ldp q4, q5, [x2]
+; SVE-NEXT: umlal v0.8h, v4.8b, v2.8b
+; SVE-NEXT: umlal v1.8h, v5.8b, v3.8b
+; SVE-NEXT: umlal2 v0.8h, v4.16b, v2.16b
+; SVE-NEXT: umlal2 v1.8h, v5.16b, v3.16b
+; SVE-NEXT: ret
+;
+; SVE2-LABEL: two_way_i8_i16_vl128_double_width:
+; SVE2: // %bb.0:
+; SVE2-NEXT: ldp q0, q1, [x0]
+; SVE2-NEXT: ldp q3, q2, [x1]
+; SVE2-NEXT: ldp q5, q4, [x2]
+; SVE2-NEXT: umlalb z0.h, z5.b, z3.b
+; SVE2-NEXT: umlalb z1.h, z4.b, z2.b
+; SVE2-NEXT: umlalt z0.h, z5.b, z3.b
+; SVE2-NEXT: umlalt z1.h, z4.b, z2.b
+; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT: // kill: def $q1 killed $q1 killed $z1
+; SVE2-NEXT: ret
;
; SME-LABEL: two_way_i8_i16_vl128_double_width:
; SME: // %bb.0:
@@ -103,6 +147,19 @@ define <16 x i16> @two_way_i8_i16_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscal
; SVE-NEXT: // kill: def $q1 killed $q1 killed $z1
; SVE-NEXT: ret
;
+; SVE2-LABEL: two_way_i8_i16_vl256:
+; SVE2: // %bb.0:
+; SVE2-NEXT: ldr z0, [x0]
+; SVE2-NEXT: ldr z1, [x1]
+; SVE2-NEXT: ldr z2, [x2]
+; SVE2-NEXT: umlalb z0.h, z2.b, z1.b
+; SVE2-NEXT: umlalt z0.h, z2.b, z1.b
+; SVE2-NEXT: movprfx z1, z0
+; SVE2-NEXT: ext z1.b, z1.b, z0.b, #16
+; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT: // kill: def $q1 killed $q1 killed $z1
+; SVE2-NEXT: ret
+;
; SME-LABEL: two_way_i8_i16_vl256:
; SME: // %bb.0:
; SME-NEXT: ldr z0, [x0]
@@ -129,14 +186,33 @@ define <16 x i16> @two_way_i8_i16_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscal
define <4 x i32> @two_way_i16_i32_vl128(ptr %accptr, ptr %uptr, ptr %sptr) {
;
-; COMMON-LABEL: two_way_i16_i32_vl128:
-; COMMON: // %bb.0:
-; COMMON-NEXT: ldr q0, [x0]
-; COMMON-NEXT: ldr q1, [x1]
-; COMMON-NEXT: ldr q2, [x2]
-; COMMON-NEXT: umlal v0.4s, v2.4h, v1.4h
-; COMMON-NEXT: umlal2 v0.4s, v2.8h, v1.8h
-; COMMON-NEXT: ret
+; NEON-LABEL: two_way_i16_i32_vl128:
+; NEON: // %bb.0:
+; NEON-NEXT: ldr q0, [x0]
+; NEON-NEXT: ldr q1, [x1]
+; NEON-NEXT: ldr q2, [x2]
+; NEON-NEXT: umlal v0.4s, v2.4h, v1.4h
+; NEON-NEXT: umlal2 v0.4s, v2.8h, v1.8h
+; NEON-NEXT: ret
+;
+; SVE-LABEL: two_way_i16_i32_vl128:
+; SVE: // %bb.0:
+; SVE-NEXT: ldr q0, [x0]
+; SVE-NEXT: ldr q1, [x1]
+; SVE-NEXT: ldr q2, [x2]
+; SVE-NEXT: umlal v0.4s, v2.4h, v1.4h
+; SVE-NEXT: umlal2 v0.4s, v2.8h, v1.8h
+; SVE-NEXT: ret
+;
+; SVE2-LABEL: two_way_i16_i32_vl128:
+; SVE2: // %bb.0:
+; SVE2-NEXT: ldr q0, [x0]
+; SVE2-NEXT: ldr q1, [x1]
+; SVE2-NEXT: ldr q2, [x2]
+; SVE2-NEXT: umlalb z0.s, z2.h, z1.h
+; SVE2-NEXT: umlalt z0.s, z2.h, z1.h
+; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT: ret
;
; SME-LABEL: two_way_i16_i32_vl128:
; SME: // %bb.0:
@@ -158,16 +234,40 @@ define <4 x i32> @two_way_i16_i32_vl128(ptr %accptr, ptr %uptr, ptr %sptr) {
define <8 x i32> @two_way_i16_i32_vl128_double_width(ptr %accptr, ptr %uptr, ptr %sptr) {
;
-; COMMON-LABEL: two_way_i16_i32_vl128_double_width:
-; COMMON: // %bb.0:
-; COMMON-NEXT: ldp q0, q1, [x0]
-; COMMON-NEXT: ldp q2, q3, [x1]
-; COMMON-NEXT: ldp q4, q5, [x2]
-; COMMON-NEXT: umlal v0.4s, v4.4h, v2.4h
-; COMMON-NEXT: umlal v1.4s, v5.4h, v3.4h
-; COMMON-NEXT: umlal2 v0.4s, v4.8h, v2.8h
-; COMMON-NEXT: umlal2 v1.4s, v5.8h, v3.8h
-; COMMON-NEXT: ret
+; NEON-LABEL: two_way_i16_i32_vl128_double_width:
+; NEON: // %bb.0:
+; NEON-NEXT: ldp q0, q1, [x0]
+; NEON-NEXT: ldp q2, q3, [x1]
+; NEON-NEXT: ldp q4, q5, [x2]
+; NEON-NEXT: umlal v0.4s, v4.4h, v2.4h
+; NEON-NEXT: umlal v1.4s, v5.4h, v3.4h
+; NEON-NEXT: umlal2 v0.4s, v4.8h, v2.8h
+; NEON-NEXT: umlal2 v1.4s, v5.8h, v3.8h
+; NEON-NEXT: ret
+;
+; SVE-LABEL: two_way_i16_i32_vl128_double_width:
+; SVE: // %bb.0:
+; SVE-NEXT: ldp q0, q1, [x0]
+; SVE-NEXT: ldp q2, q3, [x1]
+; SVE-NEXT: ldp q4, q5, [x2]
+; SVE-NEXT: umlal v0.4s, v4.4h, v2.4h
+; SVE-NEXT: umlal v1.4s, v5.4h, v3.4h
+; SVE-NEXT: umlal2 v0.4s, v4.8h, v2.8h
+; SVE-NEXT: umlal2 v1.4s, v5.8h, v3.8h
+; SVE-NEXT: ret
+;
+; SVE2-LABEL: two_way_i16_i32_vl128_double_width:
+; SVE2: // %bb.0:
+; SVE2-NEXT: ldp q0, q1, [x0]
+; SVE2-NEXT: ldp q3, q2, [x1]
+; SVE2-NEXT: ldp q5, q4, [x2]
+; SVE2-NEXT: umlalb z0.s, z5.h, z3.h
+; SVE2-NEXT: umlalb z1.s, z4.h, z2.h
+; SVE2-NEXT: umlalt z0.s, z5.h, z3.h
+; SVE2-NEXT: umlalt z1.s, z4.h, z2.h
+; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT: // kill: def $q1 killed $q1 killed $z1
+; SVE2-NEXT: ret
;
; SME-LABEL: two_way_i16_i32_vl128_double_width:
; SME: // %bb.0:
@@ -221,6 +321,19 @@ define <8 x i32> @two_way_i16_i32_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscal
; SVE-NEXT: // kill: def $q1 killed $q1 killed $z1
; SVE-NEXT: ret
;
+; SVE2-LABEL: two_way_i16_i32_vl256:
+; SVE2: // %bb.0:
+; SVE2-NEXT: ldr z0, [x0]
+; SVE2-NEXT: ldr z1, [x1]
+; SVE2-NEXT: ldr z2, [x2]
+; SVE2-NEXT: umlalb z0.s, z2.h, z1.h
+; SVE2-NEXT: umlalt z0.s, z2.h, z1.h
+; SVE2-NEXT: movprfx z1, z0
+; SVE2-NEXT: ext z1.b, z1.b, z0.b, #16
+; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT: // kill: def $q1 killed $q1 killed $z1
+; SVE2-NEXT: ret
+;
; SME-LABEL: two_way_i16_i32_vl256:
; SME: // %bb.0:
; SME-NEXT: ldr z0, [x0]
@@ -247,14 +360,33 @@ define <8 x i32> @two_way_i16_i32_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscal
define <2 x i64> @two_way_i32_i64_vl128(ptr %accptr, ptr %uptr, ptr %sptr) {
;
-; COMMON-LABEL: two_way_i32_i64_vl128:
-; COMMON: // %bb.0:
-; COMMON-NEXT: ldr q0, [x0]
-; COMMON-NEXT: ldr q1, [x1]
-; COMMON-NEXT: ldr q2, [x2]
-; COMMON-NEXT: umlal v0.2d, v2.2s, v1.2s
-; COMMON-NEXT: umlal2 v0.2d, v2.4s, v1.4s
-; COMMON-NEXT: ret
+; NEON-LABEL: two_way_i32_i64_vl128:
+; NEON: // %bb.0:
+; NEON-NEXT: ldr q0, [x0]
+; NEON-NEXT: ldr q1, [x1]
+; NEON-NEXT: ldr q2, [x2]
+; NEON-NEXT: umlal v0.2d, v2.2s, v1.2s
+; NEON-NEXT: umlal2 v0.2d, v2.4s, v1.4s
+; NEON-NEXT: ret
+;
+; SVE-LABEL: two_way_i32_i64_vl128:
+; SVE: // %bb.0:
+; SVE-NEXT: ldr q0, [x0]
+; SVE-NEXT: ldr q1, [x1]
+; SVE-NEXT: ldr q2, [x2]
+; SVE-NEXT: umlal v0.2d, v2.2s, v1.2s
+; SVE-NEXT: umlal2 v0.2d, v2.4s, v1.4s
+; SVE-NEXT: ret
+;
+; SVE2-LABEL: two_way_i32_i64_vl128:
+; SVE2: // %bb.0:
+; SVE2-NEXT: ldr q0, [x0]
+; SVE2-NEXT: ldr q1, [x1]
+; SVE2-NEXT: ldr q2, [x2]
+; SVE2-NEXT: umlalb z0.d, z2.s, z1.s
+; SVE2-NEXT: umlalt z0.d, z2.s, z1.s
+; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT: ret
;
; SME-LABEL: two_way_i32_i64_vl128:
; SME: // %bb.0:
@@ -276,16 +408,40 @@ define <2 x i64> @two_way_i32_i64_vl128(ptr %accptr, ptr %uptr, ptr %sptr) {
define <4 x i64> @two_way_i32_i64_vl128_double_width(ptr %accptr, ptr %uptr, ptr %sptr) {
;
-; COMMON-LABEL: two_way_i32_i64_vl128_double_width:
-; COMMON: // %bb.0:
-; COMMON-NEXT: ldp q0, q1, [x0]
-; COMMON-NEXT: ldp q2, q3, [x1]
-; COMMON-NEXT: ldp q4, q5, [x2]
-; COMMON-NEXT: umlal v0.2d, v4.2s, v2.2s
-; COMMON-NEXT: umlal v1.2d, v5.2s, v3.2s
-; COMMON-NEXT: umlal2 v0.2d, v4.4s, v2.4s
-; COMMON-NEXT: umlal2 v1.2d, v5.4s, v3.4s
-; COMMON-NEXT: ret
+; NEON-LABEL: two_way_i32_i64_vl128_double_width:
+; NEON: // %bb.0:
+; NEON-NEXT: ldp q0, q1, [x0]
+; NEON-NEXT: ldp q2, q3, [x1]
+; NEON-NEXT: ldp q4, q5, [x2]
+; NEON-NEXT: umlal v0.2d, v4.2s, v2.2s
+; NEON-NEXT: umlal v1.2d, v5.2s, v3.2s
+; NEON-NEXT: umlal2 v0.2d, v4.4s, v2.4s
+; NEON-NEXT: umlal2 v1.2d, v5.4s, v3.4s
+; NEON-NEXT: ret
+;
+; SVE-LABEL: two_way_i32_i64_vl128_double_width:
+; SVE: // %bb.0:
+; SVE-NEXT: ldp q0, q1, [x0]
+; SVE-NEXT: ldp q2, q3, [x1]
+; SVE-NEXT: ldp q4, q5, [x2]
+; SVE-NEXT: umlal v0.2d, v4.2s, v2.2s
+; SVE-NEXT: umlal v1.2d, v5.2s, v3.2s
+; SVE-NEXT: umlal2 v0.2d, v4.4s, v2.4s
+; SVE-NEXT: umlal2 v1.2d, v5.4s, v3.4s
+; SVE-NEXT: ret
+;
+; SVE2-LABEL: two_way_i32_i64_vl128_double_width:
+; SVE2: // %bb.0:
+; SVE2-NEXT: ldp q0, q1, [x0]
+; SVE2-NEXT: ldp q3, q2, [x1]
+; SVE2-NEXT: ldp q5, q4, [x2]
+; SVE2-NEXT: umlalb z0.d, z5.s, z3.s
+; SVE2-NEXT: umlalb z1.d, z4.s, z2.s
+; SVE2-NEXT: umlalt z0.d, z5.s, z3.s
+; SVE2-NEXT: umlalt z1.d, z4.s, z2.s
+; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT: // kill: def $q1 killed $q1 killed $z1
+; SVE2-NEXT: ret
;
; SME-LABEL: two_way_i32_i64_vl128_double_width:
; SME: // %bb.0:
@@ -339,6 +495,19 @@ define <4 x i64> @two_way_i32_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscal
; SVE-NEXT: // kill: def $q1 killed $q1 killed $z1
; SVE-NEXT: ret
;
+; SVE2-LABEL: two_way_i32_i64_vl256:
+; SVE2: // %bb.0:
+; SVE2-NEXT: ldr z0, [x0]
+; SVE2-NEXT: ldr z1, [x1]
+; SVE2-NEXT: ldr z2, [x2]
+; SVE2-NEXT: umlalb z0.d, z2.s, z1.s
+; SVE2-NEXT: umlalt z0.d, z2.s, z1.s
+; SVE2-NEXT: movprfx z1, z0
+; SVE2-NEXT: ext z1.b, z1.b, z0.b, #16
+; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT: // kill: def $q1 killed $q1 killed $z1
+; SVE2-NEXT: ret
+;
; SME-LABEL: two_way_i32_i64_vl256:
; SME: // %bb.0:
; SME-NEXT: ldr z0, [x0]
@@ -366,13 +535,30 @@ define <4 x i64> @two_way_i32_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscal
define <4 x i32> @four_way_i8_i32_vl128(ptr %accptr, ptr %uptr, ptr %sptr) {
;
-; COMMON-LABEL: four_way_i8_i32_vl128:
-; COMMON: // %bb.0:
-; COMMON-NEXT: ldr q0, [x0]
-; COMMON-NEXT: ldr q1, [x1]
-; COMMON-NEXT: ldr q2, [x2]
-; COMMON-NEXT: udot v0.4s, v2.16b, v1.16b
-; COMMON-NEXT: ret
+; NEON-LABEL: four_way_i8_i32_vl128:
+; NEON: // %bb.0:
+; NEON-NEXT: ldr q0, [x0]
+; NEON-NEXT: ldr q1, [x1]
+; NEON-NEXT: ldr q2, [x2]
+; NEON-NEXT: udot v0.4s, v2.16b, v1.16b
+; NEON-NEXT: ret
+;
+; SVE-LABEL: four_way_i8_i32_vl128:
+; SVE: // %bb.0:
+; SVE-NEXT: ldr q0, [x0]
+; SVE-NEXT: ldr q1, [x1]
+; SVE-NEXT: ldr q2, [x2]
+; SVE-NEXT: udot v0.4s, v2.16b, v1.16b
+; SVE-NEXT: ret
+;
+; SVE2-LABEL: four_way_i8_i32_vl128:
+; SVE2: // %bb.0:
+; SVE2-NEXT: ldr q0, [x0]
+; SVE2-NEXT: ldr q1, [x1]
+; SVE2-NEXT: ldr q2, [x2]
+; SVE2-NEXT: udot z0.s, z2.b, z1.b
+; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT: ret
;
; SME-LABEL: four_way_i8_i32_vl128:
; SME: // %bb.0:
@@ -466,6 +652,18 @@ define <2 x i64> @four_way_i8_i64_vl128_usdot(ptr %accptr, ptr %uptr, ptr %sptr)
; SVE-NEXT: saddw2 v0.2d, v1.2d, v0.4s
; SVE-NEXT: ret
;
+; SVE2-LABEL: four_way_i8_i64_vl128_usdot:
+; SVE2: // %bb.0:
+; SVE2-NEXT: movi v0.2d, #0000000000000000
+; SVE2-NEXT: ldr q1, [x1]
+; SVE2-NEXT: ldr q2, [x2]
+; SVE2-NEXT: usdot z0.s, z1.b, z2.b
+; SVE2-NEXT: ldr q1, [x0]
+; SVE2-NEXT: saddwb z1.d, z1.d, z0.s
+; SVE2-NEXT: saddwt z0.d, z1.d, z0.s
+; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT: ret
+;
; SME-LABEL: four_way_i8_i64_vl128_usdot:
; SME: // %bb.0:
; SME-NEXT: mov z0.s, #0 // =0x0
@@ -534,14 +732,34 @@ define <2 x i64> @four_way_i16_i64_vl128_usdot(ptr %accptr, ptr %uptr, ptr %sptr
define <8 x i32> @four_way_i8_i32_vl128_double_width(ptr %accptr, ptr %uptr, ptr %sptr) {
;
-; COMMON-LABEL: four_way_i8_i32_vl128_double_width:
-; COMMON: // %bb.0:
-; COMMON-NEXT: ldp q0, q1, [x0]
-; COMMON-NEXT: ldp q3, q2, [x1]
-; COMMON-NEXT: ldp q5, q4, [x2]
-; COMMON-NEXT: udot v0.4s, v5.16b, v3.16b
-; COMMON-NEXT: udot v1.4s, v4.16b, v2.16b
-; COMMON-NEXT: ret
+; NEON-LABEL: four_way_i8_i32_vl128_double_width:
+; NEON: // %bb.0:
+; NEON-NEXT: ldp q0, q1, [x0]
+; NEON-NEXT: ldp q3, q2, [x1]
+; NEON-NEXT: ldp q5, q4, [x2]
+; NEON-NEXT: udot v0.4s, v5.16b, v3.16b
+; NEON-NEXT: udot v1.4s, v4.16b, v2.16b
+; NEON-NEXT: ret
+;
+; SVE-LABEL: four_way_i8_i32_vl128_double_width:
+; SVE: // %bb.0:
+; SVE-NEXT: ldp q0, q1, [x0]
+; SVE-NEXT: ldp q3, q2, [x1]
+; SVE-NEXT: ldp q5, q4, [x2]
+; SVE-NEXT: udot v0.4s, v5.16b, v3.16b
+; SVE-NEXT: udot v1.4s, v4.16b, v2.16b
+; SVE-NEXT: ret
+;
+; SVE2-LABEL: four_way_i8_i32_vl128_double_width:
+; SVE2: // %bb.0:
+; SVE2-NEXT: ldp q0, q1, [x0]
+; SVE2-NEXT: ldp q3, q2, [x1]
+; SVE2-NEXT: ldp q5, q4, [x2]
+; SVE2-NEXT: udot z0.s, z5.b, z3.b
+; SVE2-NEXT: udot z1.s, z4.b, z2.b
+; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT: // kill: def $q1 killed $q1 killed $z1
+; SVE2-NEXT: ret
;
; SME-LABEL: four_way_i8_i32_vl128_double_width:
; SME: // %bb.0:
@@ -614,6 +832,18 @@ define <8 x i32> @four_way_i8_i32_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscal
; SVE-NEXT: // kill: def $q1 killed $q1 killed $z1
; SVE-NEXT: ret
;
+; SVE2-LABEL: four_way_i8_i32_vl256:
+; SVE2: // %bb.0:
+; SVE2-NEXT: ldr z0, [x0]
+; SVE2-NEXT: ldr z1, [x1]
+; SVE2-NEXT: ldr z2, [x2]
+; SVE2-NEXT: udot z0.s, z2.b, z1.b
+; SVE2-NEXT: movprfx z1, z0
+; SVE2-NEXT: ext z1.b, z1.b, z0.b, #16
+; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT: // kill: def $q1 killed $q1 killed $z1
+; SVE2-NEXT: ret
+;
; SME-LABEL: four_way_i8_i32_vl256:
; SME: // %bb.0:
; SME-NEXT: ldr z0, [x0]
@@ -657,6 +887,18 @@ define <8 x i32> @four_way_i8_i32_vl256_usdot(ptr %accptr, ptr %uptr, ptr %sptr)
; SVE-NEXT: // kill: def $q1 killed $q1 killed $z1
; SVE-NEXT: ret
;
+; SVE2-LABEL: four_way_i8_i32_vl256_usdot:
+; SVE2: // %bb.0:
+; SVE2-NEXT: ldr z0, [x0]
+; SVE2-NEXT: ldr z1, [x1]
+; SVE2-NEXT: ldr z2, [x2]
+; SVE2-NEXT: usdot z0.s, z1.b, z2.b
+; SVE2-NEXT: movprfx z1, z0
+; SVE2-NEXT: ext z1.b, z1.b, z0.b, #16
+; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT: // kill: def $q1 killed $q1 killed $z1
+; SVE2-NEXT: ret
+;
; SME-LABEL: four_way_i8_i32_vl256_usdot:
; SME: // %bb.0:
; SME-NEXT: ldr z0, [x0]
@@ -682,18 +924,40 @@ define <8 x i32> @four_way_i8_i32_vl256_usdot(ptr %accptr, ptr %uptr, ptr %sptr)
define <2 x i64> @four_way_i16_i64_vl128(ptr %accptr, ptr %uptr, ptr %sptr) {
;
-; COMMON-LABEL: four_way_i16_i64_vl128:
-; COMMON: // %bb.0:
-; COMMON-NEXT: ldr q0, [x1]
-; COMMON-NEXT: ldr q1, [x2]
-; COMMON-NEXT: ldr q3, [x0]
-; COMMON-NEXT: umull v2.4s, v1.4h, v0.4h
-; COMMON-NEXT: umull2 v0.4s, v1.8h, v0.8h
-; COMMON-NEXT: uaddw v3.2d, v3.2d, v2.2s
-; COMMON-NEXT: uaddw2 v1.2d, v3.2d, v2.4s
-; COMMON-NEXT: uaddw v1.2d, v1.2d, v0.2s
-; COMMON-NEXT: uaddw2 v0.2d, v1.2d, v0.4s
-; COMMON-NEXT: ret
+; NEON-LABEL: four_way_i16_i64_vl128:
+; NEON: // %bb.0:
+; NEON-NEXT: ldr q0, [x1]
+; NEON-NEXT: ldr q1, [x2]
+; NEON-NEXT: ldr q3, [x0]
+; NEON-NEXT: umull v2.4s, v1.4h, v0.4h
+; NEON-NEXT: umull2 v0.4s, v1.8h, v0.8h
+; NEON-NEXT: uaddw v3.2d, v3.2d, v2.2s
+; NEON-NEXT: uaddw2 v1.2d, v3.2d, v2.4s
+; NEON-NEXT: uaddw v1.2d, v1.2d, v0.2s
+; NEON-NEXT: uaddw2 v0.2d, v1.2d, v0.4s
+; NEON-NEXT: ret
+;
+; SVE-LABEL: four_way_i16_i64_vl128:
+; SVE: // %bb.0:
+; SVE-NEXT: ldr q0, [x1]
+; SVE-NEXT: ldr q1, [x2]
+; SVE-NEXT: ldr q3, [x0]
+; SVE-NEXT: umull v2.4s, v1.4h, v0.4h
+; SVE-NEXT: umull2 v0.4s, v1.8h, v0.8h
+; SVE-NEXT: uaddw v3.2d, v3.2d, v2.2s
+; SVE-NEXT: uaddw2 v1.2d, v3.2d, v2.4s
+; SVE-NEXT: uaddw v1.2d, v1.2d, v0.2s
+; SVE-NEXT: uaddw2 v0.2d, v1.2d, v0.4s
+; SVE-NEXT: ret
+;
+; SVE2-LABEL: four_way_i16_i64_vl128:
+; SVE2: // %bb.0:
+; SVE2-NEXT: ldr q0, [x0]
+; SVE2-NEXT: ldr q1, [x1]
+; SVE2-NEXT: ldr q2, [x2]
+; SVE2-NEXT: udot z0.d, z2.h, z1.h
+; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT: ret
;
; SME-LABEL: four_way_i16_i64_vl128:
; SME: // %bb.0:
@@ -714,24 +978,54 @@ define <2 x i64> @four_way_i16_i64_vl128(ptr %accptr, ptr %uptr, ptr %sptr) {
define <4 x i64> @four_way_i16_i64_vl128_double_width(ptr %accptr, ptr %uptr, ptr %sptr) {
;
-; COMMON-LABEL: four_way_i16_i64_vl128_double_width:
-; COMMON: // %bb.0:
-; COMMON-NEXT: ldp q0, q1, [x1]
-; COMMON-NEXT: ldp q2, q3, [x2]
-; COMMON-NEXT: ldp q7, q6, [x0]
-; COMMON-NEXT: umull v4.4s, v3.4h, v1.4h
-; COMMON-NEXT: umull v5.4s, v2.4h, v0.4h
-; COMMON-NEXT: umull2 v1.4s, v3.8h, v1.8h
-; COMMON-NEXT: umull2 v0.4s, v2.8h, v0.8h
-; COMMON-NEXT: uaddw v7.2d, v7.2d, v5.2s
-; COMMON-NEXT: uaddw v6.2d, v6.2d, v4.2s
-; COMMON-NEXT: uaddw2 v2.2d, v7.2d, v5.4s
-; COMMON-NEXT: uaddw2 v3.2d, v6.2d, v4.4s
-; COMMON-NEXT: uaddw v2.2d, v2.2d, v0.2s
-; COMMON-NEXT: uaddw v3.2d, v3.2d, v1.2s
-; COMMON-NEXT: uaddw2 v0.2d, v2.2d, v0.4s
-; COMMON-NEXT: uaddw2 v1.2d, v3.2d, v1.4s
-; COMMON-NEXT: ret
+; NEON-LABEL: four_way_i16_i64_vl128_double_width:
+; NEON: // %bb.0:
+; NEON-NEXT: ldp q0, q1, [x1]
+; NEON-NEXT: ldp q2, q3, [x2]
+; NEON-NEXT: ldp q7, q6, [x0]
+; NEON-NEXT: umull v4.4s, v3.4h, v1.4h
+; NEON-NEXT: umull v5.4s, v2.4h, v0.4h
+; NEON-NEXT: umull2 v1.4s, v3.8h, v1.8h
+; NEON-NEXT: umull2 v0.4s, v2.8h, v0.8h
+; NEON-NEXT: uaddw v7.2d, v7.2d, v5.2s
+; NEON-NEXT: uaddw v6.2d, v6.2d, v4.2s
+; NEON-NEXT: uaddw2 v2.2d, v7.2d, v5.4s
+; NEON-NEXT: uaddw2 v3.2d, v6.2d, v4.4s
+; NEON-NEXT: uaddw v2.2d, v2.2d, v0.2s
+; NEON-NEXT: uaddw v3.2d, v3.2d, v1.2s
+; NEON-NEXT: uaddw2 v0.2d, v2.2d, v0.4s
+; NEON-NEXT: uaddw2 v1.2d, v3.2d, v1.4s
+; NEON-NEXT: ret
+;
+; SVE-LABEL: four_way_i16_i64_vl128_double_width:
+; SVE: // %bb.0:
+; SVE-NEXT: ldp q0, q1, [x1]
+; SVE-NEXT: ldp q2, q3, [x2]
+; SVE-NEXT: ldp q7, q6, [x0]
+; SVE-NEXT: umull v4.4s, v3.4h, v1.4h
+; SVE-NEXT: umull v5.4s, v2.4h, v0.4h
+; SVE-NEXT: umull2 v1.4s, v3.8h, v1.8h
+; SVE-NEXT: umull2 v0.4s, v2.8h, v0.8h
+; SVE-NEXT: uaddw v7.2d, v7.2d, v5.2s
+; SVE-NEXT: uaddw v6.2d, v6.2d, v4.2s
+; SVE-NEXT: uaddw2 v2.2d, v7.2d, v5.4s
+; SVE-NEXT: uaddw2 v3.2d, v6.2d, v4.4s
+; SVE-NEXT: uaddw v2.2d, v2.2d, v0.2s
+; SVE-NEXT: uaddw v3.2d, v3.2d, v1.2s
+; SVE-NEXT: uaddw2 v0.2d, v2.2d, v0.4s
+; SVE-NEXT: uaddw2 v1.2d, v3.2d, v1.4s
+; SVE-NEXT: ret
+;
+; SVE2-LABEL: four_way_i16_i64_vl128_double_width:
+; SVE2: // %bb.0:
+; SVE2-NEXT: ldp q0, q1, [x0]
+; SVE2-NEXT: ldp q3, q2, [x1]
+; SVE2-NEXT: ldp q5, q4, [x2]
+; SVE2-NEXT: udot z0.d, z5.h, z3.h
+; SVE2-NEXT: udot z1.d, z4.h, z2.h
+; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT: // kill: def $q1 killed $q1 killed $z1
+; SVE2-NEXT: ret
;
; SME-LABEL: four_way_i16_i64_vl128_double_width:
; SME: // %bb.0:
@@ -785,6 +1079,18 @@ define <4 x i64> @four_way_i16_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vsca
; SVE-NEXT: // kill: def $q1 killed $q1 killed $z1
; SVE-NEXT: ret
;
+; SVE2-LABEL: four_way_i16_i64_vl256:
+; SVE2: // %bb.0:
+; SVE2-NEXT: ldr z0, [x0]
+; SVE2-NEXT: ldr z1, [x1]
+; SVE2-NEXT: ldr z2, [x2]
+; SVE2-NEXT: udot z0.d, z2.h, z1.h
+; SVE2-NEXT: movprfx z1, z0
+; SVE2-NEXT: ext z1.b, z1.b, z0.b, #16
+; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT: // kill: def $q1 killed $q1 killed $z1
+; SVE2-NEXT: ret
+;
; SME-LABEL: four_way_i16_i64_vl256:
; SME: // %bb.0:
; SME-NEXT: ldr z0, [x0]
@@ -833,6 +1139,18 @@ define <2 x i64> @eight_way_i8_i64_vl128(ptr %accptr, ptr %uptr, ptr %sptr) {
; SVE-NEXT: uaddw2 v0.2d, v1.2d, v0.4s
; SVE-NEXT: ret
;
+; SVE2-LABEL: eight_way_i8_i64_vl128:
+; SVE2: // %bb.0:
+; SVE2-NEXT: movi v0.2d, #0000000000000000
+; SVE2-NEXT: ldr q1, [x1]
+; SVE2-NEXT: ldr q2, [x2]
+; SVE2-NEXT: udot z0.s, z2.b, z1.b
+; SVE2-NEXT: ldr q1, [x0]
+; SVE2-NEXT: uaddwb z1.d, z1.d, z0.s
+; SVE2-NEXT: uaddwt z0.d, z1.d, z0.s
+; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT: ret
+;
; SME-LABEL: eight_way_i8_i64_vl128:
; SME: // %bb.0:
; SME-NEXT: mov z0.s, #0 // =0x0
@@ -882,6 +1200,18 @@ define <2 x i64> @eight_way_i8_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vsca
; SVE-NEXT: uaddw2 v0.2d, v1.2d, v0.4s
; SVE-NEXT: ret
;
+; SVE2-LABEL: eight_way_i8_i64_vl256:
+; SVE2: // %bb.0:
+; SVE2-NEXT: movi v0.2d, #0000000000000000
+; SVE2-NEXT: ldr q1, [x1]
+; SVE2-NEXT: ldr q2, [x2]
+; SVE2-NEXT: udot z0.s, z2.b, z1.b
+; SVE2-NEXT: ldr q1, [x0]
+; SVE2-NEXT: uaddwb z1.d, z1.d, z0.s
+; SVE2-NEXT: uaddwt z0.d, z1.d, z0.s
+; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT: ret
+;
; SME-LABEL: eight_way_i8_i64_vl256:
; SME: // %bb.0:
; SME-NEXT: mov z0.s, #0 // =0x0
@@ -934,6 +1264,23 @@ define <4 x i64> @four_way_i8_i64_vl128_double_width(ptr %accptr, ptr %uptr, ptr
; SVE-NEXT: uaddw2 v1.2d, v2.2d, v1.4s
; SVE-NEXT: ret
;
+; SVE2-LABEL: four_way_i8_i64_vl128_double_width:
+; SVE2: // %bb.0:
+; SVE2-NEXT: movi v1.2d, #0000000000000000
+; SVE2-NEXT: movi v0.2d, #0000000000000000
+; SVE2-NEXT: ldp q3, q2, [x1]
+; SVE2-NEXT: ldp q5, q4, [x2]
+; SVE2-NEXT: udot z0.s, z5.b, z3.b
+; SVE2-NEXT: udot z1.s, z4.b, z2.b
+; SVE2-NEXT: ldp q3, q2, [x0]
+; SVE2-NEXT: uaddwb z3.d, z3.d, z0.s
+; SVE2-NEXT: uaddwb z2.d, z2.d, z1.s
+; SVE2-NEXT: uaddwt z0.d, z3.d, z0.s
+; SVE2-NEXT: uaddwt z1.d, z2.d, z1.s
+; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT: // kill: def $q1 killed $q1 killed $z1
+; SVE2-NEXT: ret
+;
; SME-LABEL: four_way_i8_i64_vl128_double_width:
; SME: // %bb.0:
; SME-NEXT: mov z1.s, #0 // =0x0
@@ -958,6 +1305,12 @@ define <4 x i64> @four_way_i8_i64_vl128_double_width(ptr %accptr, ptr %uptr, ptr
ret <4 x i64> %partial.reduce
}
+; Regression test for the dynamic fixed-dot width. On plain SVE (no SVE2) this
+; VL-pinned <4 x i64> result is not split by type legalization and reaches the
+; v4i32-fold, where the scalable dot is nxv4i32 holding eight i32 lanes at
+; VL=256. The fold derives the fixed dot width from the result (v8i32 here); a
+; static v4i32 would keep only four lanes and assert on the v2i32 -> v4i64
+; extend. The SVE check below is what exercises that path.
define <4 x i64> @four_way_i8_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscale_range(2,2) {
; NEON-LABEL: four_way_i8_i64_vl256:
; NEON: // %bb.0:
@@ -992,6 +1345,21 @@ define <4 x i64> @four_way_i8_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscal
; SVE-NEXT: // kill: def $q1 killed $q1 killed $z1
; SVE-NEXT: ret
;
+; SVE2-LABEL: four_way_i8_i64_vl256:
+; SVE2: // %bb.0:
+; SVE2-NEXT: movi v0.2d, #0000000000000000
+; SVE2-NEXT: ldr z1, [x1]
+; SVE2-NEXT: ldr z2, [x2]
+; SVE2-NEXT: udot z0.s, z2.b, z1.b
+; SVE2-NEXT: ldr z1, [x0]
+; SVE2-NEXT: uaddwb z1.d, z1.d, z0.s
+; SVE2-NEXT: uaddwt z0.d, z1.d, z0.s
+; SVE2-NEXT: movprfx z1, z0
+; SVE2-NEXT: ext z1.b, z1.b, z0.b, #16
+; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT: // kill: def $q1 killed $q1 killed $z1
+; SVE2-NEXT: ret
+;
; SME-LABEL: four_way_i8_i64_vl256:
; SME: // %bb.0:
; SME-NEXT: ldr z0, [x1]
More information about the llvm-commits
mailing list