[llvm] [AArch64][ISel] Prefer SVE over NEON for fixed-width ASL LSR (PR #199003)
Harry Ramsey via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 23 07:31:16 PDT 2026
https://github.com/Harry-Ramsey updated https://github.com/llvm/llvm-project/pull/199003
>From 361b2e0386b966cf3efb73a05769e07a3704681f Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Thu, 23 Jul 2026 10:45:59 +0000
Subject: [PATCH] [AArch64][ISel] Prefer SVE over NEON for fixed-width ASL LSR
Prefer SVE ASR/LSR for fixed-length vector SRA/SRL when SVE is available
whilst retaining existing NEON lowering for immediate right shifts and
negative shift-count splats.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 33 ++++-
.../AArch64/sve-fixed-length-int-shifts.ll | 128 +++++++++++-------
2 files changed, 111 insertions(+), 50 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 1c649261752df..9a0be58398237 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -17970,6 +17970,29 @@ static bool canLowerSRLToRoundingShiftForVT(SDValue Shift, EVT ResVT,
return true;
}
+static bool isNegatedSplatShiftAmount(SDValue Op, SelectionDAG &DAG) {
+ SDValue Splat = DAG.getSplatValue(Op);
+ if (!Splat)
+ return false;
+
+ while (true) {
+ switch (Splat.getOpcode()) {
+ case ISD::ANY_EXTEND:
+ case ISD::SIGN_EXTEND:
+ case ISD::ZERO_EXTEND:
+ case ISD::TRUNCATE:
+ case ISD::AssertSext:
+ case ISD::AssertZext:
+ case ISD::SIGN_EXTEND_INREG:
+ Splat = Splat.getOperand(0);
+ continue;
+ default:
+ return Splat.getOpcode() == ISD::SUB &&
+ isNullConstant(Splat.getOperand(0));
+ }
+ }
+}
+
SDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(SDValue Op,
SelectionDAG &DAG) const {
EVT VT = Op.getValueType();
@@ -18006,15 +18029,21 @@ SDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(SDValue Op,
DAG.getTargetConstant(ShiftValue, DL, MVT::i32));
}
+ bool IsImmediateRShift =
+ isVShiftRImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize;
+ bool OverrideNEON =
+ !Subtarget->isNeonAvailable() ||
+ (!IsImmediateRShift && !isNegatedSplatShiftAmount(Op.getOperand(1), DAG));
+
if (VT.isScalableVector() ||
- useSVEForFixedLengthVectorVT(VT, !Subtarget->isNeonAvailable())) {
+ useSVEForFixedLengthVectorVT(VT, OverrideNEON)) {
unsigned Opc = Op.getOpcode() == ISD::SRA ? AArch64ISD::SRA_PRED
: AArch64ISD::SRL_PRED;
return LowerToPredicatedOp(Op, DAG, Opc);
}
// Right shift immediate
- if (isVShiftRImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize) {
+ if (IsImmediateRShift) {
unsigned Opc =
(Op.getOpcode() == ISD::SRA) ? AArch64ISD::VASHR : AArch64ISD::VLSHR;
return DAG.getNode(Opc, DL, VT, Op.getOperand(0),
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-shifts.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-shifts.ll
index a8afa90df96e4..e785e22c775ec 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-shifts.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-shifts.ll
@@ -9,23 +9,27 @@ target triple = "aarch64-unknown-linux-gnu"
; ASHR
;
-; Don't use SVE for 64-bit vectors.
define <8 x i8> @ashr_v8i8(<8 x i8> %op1, <8 x i8> %op2) vscale_range(2,0) #0 {
; CHECK-LABEL: ashr_v8i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: neg v1.8b, v1.8b
-; CHECK-NEXT: sshl v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: ptrue p0.b, vl8
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT: asr z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT: ret
%res = ashr <8 x i8> %op1, %op2
ret <8 x i8> %res
}
-; Don't use SVE for 128-bit vectors.
define <16 x i8> @ashr_v16i8(<16 x i8> %op1, <16 x i8> %op2) vscale_range(2,0) #0 {
; CHECK-LABEL: ashr_v16i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: neg v1.16b, v1.16b
-; CHECK-NEXT: sshl v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: asr z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT: ret
%res = ashr <16 x i8> %op1, %op2
ret <16 x i8> %res
@@ -109,23 +113,27 @@ define void @ashr_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 {
ret void
}
-; Don't use SVE for 64-bit vectors.
define <4 x i16> @ashr_v4i16(<4 x i16> %op1, <4 x i16> %op2) vscale_range(2,0) #0 {
; CHECK-LABEL: ashr_v4i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: neg v1.4h, v1.4h
-; CHECK-NEXT: sshl v0.4h, v0.4h, v1.4h
+; CHECK-NEXT: ptrue p0.h, vl4
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT: asr z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT: ret
%res = ashr <4 x i16> %op1, %op2
ret <4 x i16> %res
}
-; Don't use SVE for 128-bit vectors.
define <8 x i16> @ashr_v8i16(<8 x i16> %op1, <8 x i16> %op2) vscale_range(2,0) #0 {
; CHECK-LABEL: ashr_v8i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: neg v1.8h, v1.8h
-; CHECK-NEXT: sshl v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: ptrue p0.h, vl8
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: asr z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT: ret
%res = ashr <8 x i16> %op1, %op2
ret <8 x i16> %res
@@ -209,23 +217,27 @@ define void @ashr_v128i16(ptr %a, ptr %b) vscale_range(16,0) #0 {
ret void
}
-; Don't use SVE for 64-bit vectors.
define <2 x i32> @ashr_v2i32(<2 x i32> %op1, <2 x i32> %op2) vscale_range(2,0) #0 {
; CHECK-LABEL: ashr_v2i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: neg v1.2s, v1.2s
-; CHECK-NEXT: sshl v0.2s, v0.2s, v1.2s
+; CHECK-NEXT: ptrue p0.s, vl2
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT: asr z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT: ret
%res = ashr <2 x i32> %op1, %op2
ret <2 x i32> %res
}
-; Don't use SVE for 128-bit vectors.
define <4 x i32> @ashr_v4i32(<4 x i32> %op1, <4 x i32> %op2) vscale_range(2,0) #0 {
; CHECK-LABEL: ashr_v4i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: neg v1.4s, v1.4s
-; CHECK-NEXT: sshl v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: asr z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT: ret
%res = ashr <4 x i32> %op1, %op2
ret <4 x i32> %res
@@ -309,23 +321,27 @@ define void @ashr_v64i32(ptr %a, ptr %b) vscale_range(16,0) #0 {
ret void
}
-; Don't use SVE for 64-bit vectors.
define <1 x i64> @ashr_v1i64(<1 x i64> %op1, <1 x i64> %op2) vscale_range(2,0) #0 {
; CHECK-LABEL: ashr_v1i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: neg d1, d1
-; CHECK-NEXT: sshl d0, d0, d1
+; CHECK-NEXT: ptrue p0.d, vl1
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT: asr z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT: ret
%res = ashr <1 x i64> %op1, %op2
ret <1 x i64> %res
}
-; Don't use SVE for 128-bit vectors.
define <2 x i64> @ashr_v2i64(<2 x i64> %op1, <2 x i64> %op2) vscale_range(2,0) #0 {
; CHECK-LABEL: ashr_v2i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: neg v1.2d, v1.2d
-; CHECK-NEXT: sshl v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: asr z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT: ret
%res = ashr <2 x i64> %op1, %op2
ret <2 x i64> %res
@@ -413,23 +429,27 @@ define void @ashr_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 {
; LSHR
;
-; Don't use SVE for 64-bit vectors.
define <8 x i8> @lshr_v8i8(<8 x i8> %op1, <8 x i8> %op2) vscale_range(2,0) #0 {
; CHECK-LABEL: lshr_v8i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: neg v1.8b, v1.8b
-; CHECK-NEXT: ushl v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: ptrue p0.b, vl8
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT: lsr z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT: ret
%res = lshr <8 x i8> %op1, %op2
ret <8 x i8> %res
}
-; Don't use SVE for 128-bit vectors.
define <16 x i8> @lshr_v16i8(<16 x i8> %op1, <16 x i8> %op2) vscale_range(2,0) #0 {
; CHECK-LABEL: lshr_v16i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: neg v1.16b, v1.16b
-; CHECK-NEXT: ushl v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: lsr z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT: ret
%res = lshr <16 x i8> %op1, %op2
ret <16 x i8> %res
@@ -513,23 +533,27 @@ define void @lshr_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 {
ret void
}
-; Don't use SVE for 64-bit vectors.
define <4 x i16> @lshr_v4i16(<4 x i16> %op1, <4 x i16> %op2) vscale_range(2,0) #0 {
; CHECK-LABEL: lshr_v4i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: neg v1.4h, v1.4h
-; CHECK-NEXT: ushl v0.4h, v0.4h, v1.4h
+; CHECK-NEXT: ptrue p0.h, vl4
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT: lsr z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT: ret
%res = lshr <4 x i16> %op1, %op2
ret <4 x i16> %res
}
-; Don't use SVE for 128-bit vectors.
define <8 x i16> @lshr_v8i16(<8 x i16> %op1, <8 x i16> %op2) vscale_range(2,0) #0 {
; CHECK-LABEL: lshr_v8i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: neg v1.8h, v1.8h
-; CHECK-NEXT: ushl v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: ptrue p0.h, vl8
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: lsr z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT: ret
%res = lshr <8 x i16> %op1, %op2
ret <8 x i16> %res
@@ -613,23 +637,27 @@ define void @lshr_v128i16(ptr %a, ptr %b) vscale_range(16,0) #0 {
ret void
}
-; Don't use SVE for 64-bit vectors.
define <2 x i32> @lshr_v2i32(<2 x i32> %op1, <2 x i32> %op2) vscale_range(2,0) #0 {
; CHECK-LABEL: lshr_v2i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: neg v1.2s, v1.2s
-; CHECK-NEXT: ushl v0.2s, v0.2s, v1.2s
+; CHECK-NEXT: ptrue p0.s, vl2
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT: lsr z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT: ret
%res = lshr <2 x i32> %op1, %op2
ret <2 x i32> %res
}
-; Don't use SVE for 128-bit vectors.
define <4 x i32> @lshr_v4i32(<4 x i32> %op1, <4 x i32> %op2) vscale_range(2,0) #0 {
; CHECK-LABEL: lshr_v4i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: neg v1.4s, v1.4s
-; CHECK-NEXT: ushl v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: lsr z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT: ret
%res = lshr <4 x i32> %op1, %op2
ret <4 x i32> %res
@@ -713,23 +741,27 @@ define void @lshr_v64i32(ptr %a, ptr %b) vscale_range(16,0) #0 {
ret void
}
-; Don't use SVE for 64-bit vectors.
define <1 x i64> @lshr_v1i64(<1 x i64> %op1, <1 x i64> %op2) vscale_range(2,0) #0 {
; CHECK-LABEL: lshr_v1i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: neg d1, d1
-; CHECK-NEXT: ushl d0, d0, d1
+; CHECK-NEXT: ptrue p0.d, vl1
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT: lsr z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT: ret
%res = lshr <1 x i64> %op1, %op2
ret <1 x i64> %res
}
-; Don't use SVE for 128-bit vectors.
define <2 x i64> @lshr_v2i64(<2 x i64> %op1, <2 x i64> %op2) vscale_range(2,0) #0 {
; CHECK-LABEL: lshr_v2i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: neg v1.2d, v1.2d
-; CHECK-NEXT: ushl v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: lsr z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT: ret
%res = lshr <2 x i64> %op1, %op2
ret <2 x i64> %res
More information about the llvm-commits
mailing list