[llvm] [AArch64] Fold vector shifts guarded against oversized amounts into USHL (PR #207628)
Adam Scott via llvm-commits
llvm-commits at lists.llvm.org
Sun Jul 19 13:46:37 PDT 2026
https://github.com/as4230 updated https://github.com/llvm/llvm-project/pull/207628
>From 3f382b9996db37011399e9784e5e4d8a844edb8a Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Fri, 10 Jul 2026 02:06:47 +0000
Subject: [PATCH 1/3] [AArch64] Add tests for masked vector shift selects. NFC
---
.../CodeGen/AArch64/vselect-masked-shift.ll | 250 ++++++++++++++++++
1 file changed, 250 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/vselect-masked-shift.ll
diff --git a/llvm/test/CodeGen/AArch64/vselect-masked-shift.ll b/llvm/test/CodeGen/AArch64/vselect-masked-shift.ll
new file mode 100644
index 0000000000000..ce111c7950e4d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/vselect-masked-shift.ll
@@ -0,0 +1,250 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mtriple=aarch64-none-elf < %s | FileCheck %s
+
+define <4 x i32> @masked_shl_v4i32(<4 x i32> %x, <4 x i32> %amt) {
+; CHECK-LABEL: masked_shl_v4i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi v2.4s, #63
+; CHECK-NEXT: movi v3.4s, #32
+; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
+; CHECK-NEXT: ushl v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: cmhi v1.4s, v3.4s, v1.4s
+; CHECK-NEXT: and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT: ret
+entry:
+ %m = and <4 x i32> %amt, splat (i32 63)
+ %shl = shl <4 x i32> %x, %m
+ %ok = icmp ult <4 x i32> %m, splat (i32 32)
+ %res = select <4 x i1> %ok, <4 x i32> %shl, <4 x i32> zeroinitializer
+ ret <4 x i32> %res
+}
+
+define <4 x i32> @masked_shl_v4i32_swapped(<4 x i32> %x, <4 x i32> %amt) {
+; CHECK-LABEL: masked_shl_v4i32_swapped:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi v2.4s, #63
+; CHECK-NEXT: movi v3.4s, #32
+; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
+; CHECK-NEXT: ushl v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: cmhi v1.4s, v3.4s, v1.4s
+; CHECK-NEXT: and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT: ret
+entry:
+ %m = and <4 x i32> %amt, splat (i32 63)
+ %shl = shl <4 x i32> %x, %m
+ %oob = icmp uge <4 x i32> %m, splat (i32 32)
+ %res = select <4 x i1> %oob, <4 x i32> zeroinitializer, <4 x i32> %shl
+ ret <4 x i32> %res
+}
+
+define <4 x i32> @masked_srl_v4i32(<4 x i32> %x, <4 x i32> %amt) {
+; CHECK-LABEL: masked_srl_v4i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi v2.4s, #63
+; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
+; CHECK-NEXT: movi v2.4s, #32
+; CHECK-NEXT: neg v3.4s, v1.4s
+; CHECK-NEXT: cmhi v1.4s, v2.4s, v1.4s
+; CHECK-NEXT: ushl v0.4s, v0.4s, v3.4s
+; CHECK-NEXT: and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT: ret
+entry:
+ %m = and <4 x i32> %amt, splat (i32 63)
+ %srl = lshr <4 x i32> %x, %m
+ %ok = icmp ult <4 x i32> %m, splat (i32 32)
+ %res = select <4 x i1> %ok, <4 x i32> %srl, <4 x i32> zeroinitializer
+ ret <4 x i32> %res
+}
+
+define <16 x i8> @masked_shl_v16i8(<16 x i8> %x, <16 x i8> %amt) {
+; CHECK-LABEL: masked_shl_v16i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi v2.16b, #15
+; CHECK-NEXT: movi v3.16b, #8
+; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
+; CHECK-NEXT: ushl v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: cmhi v1.16b, v3.16b, v1.16b
+; CHECK-NEXT: and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT: ret
+entry:
+ %m = and <16 x i8> %amt, splat (i8 15)
+ %shl = shl <16 x i8> %x, %m
+ %ok = icmp ult <16 x i8> %m, splat (i8 8)
+ %res = select <16 x i1> %ok, <16 x i8> %shl, <16 x i8> zeroinitializer
+ ret <16 x i8> %res
+}
+
+define <8 x i16> @masked_shl_v8i16(<8 x i16> %x, <8 x i16> %amt) {
+; CHECK-LABEL: masked_shl_v8i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi v2.8h, #31
+; CHECK-NEXT: movi v3.8h, #16
+; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
+; CHECK-NEXT: ushl v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: cmhi v1.8h, v3.8h, v1.8h
+; CHECK-NEXT: and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT: ret
+entry:
+ %m = and <8 x i16> %amt, splat (i16 31)
+ %shl = shl <8 x i16> %x, %m
+ %ok = icmp ult <8 x i16> %m, splat (i16 16)
+ %res = select <8 x i1> %ok, <8 x i16> %shl, <8 x i16> zeroinitializer
+ ret <8 x i16> %res
+}
+
+define <2 x i64> @masked_shl_v2i64(<2 x i64> %x, <2 x i64> %amt) {
+; CHECK-LABEL: masked_shl_v2i64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: mov w8, #127 // =0x7f
+; CHECK-NEXT: dup v2.2d, x8
+; CHECK-NEXT: mov w8, #64 // =0x40
+; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
+; CHECK-NEXT: dup v2.2d, x8
+; CHECK-NEXT: ushl v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: cmhi v1.2d, v2.2d, v1.2d
+; CHECK-NEXT: and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT: ret
+entry:
+ %m = and <2 x i64> %amt, splat (i64 127)
+ %shl = shl <2 x i64> %x, %m
+ %ok = icmp ult <2 x i64> %m, splat (i64 64)
+ %res = select <2 x i1> %ok, <2 x i64> %shl, <2 x i64> zeroinitializer
+ ret <2 x i64> %res
+}
+
+define <2 x i32> @masked_shl_v2i32(<2 x i32> %x, <2 x i32> %amt) {
+; CHECK-LABEL: masked_shl_v2i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi v2.2s, #63
+; CHECK-NEXT: movi v3.2s, #32
+; CHECK-NEXT: and v1.8b, v1.8b, v2.8b
+; CHECK-NEXT: ushl v0.2s, v0.2s, v1.2s
+; CHECK-NEXT: cmhi v1.2s, v3.2s, v1.2s
+; CHECK-NEXT: and v0.8b, v1.8b, v0.8b
+; CHECK-NEXT: ret
+entry:
+ %m = and <2 x i32> %amt, splat (i32 63)
+ %shl = shl <2 x i32> %x, %m
+ %ok = icmp ult <2 x i32> %m, splat (i32 32)
+ %res = select <2 x i1> %ok, <2 x i32> %shl, <2 x i32> zeroinitializer
+ ret <2 x i32> %res
+}
+
+define <4 x i32> @unbounded_shl_v4i32(<4 x i32> %x, <4 x i32> %amt) {
+; CHECK-LABEL: unbounded_shl_v4i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi v2.4s, #32
+; CHECK-NEXT: ushl v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: cmhi v1.4s, v2.4s, v1.4s
+; CHECK-NEXT: and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT: ret
+entry:
+ %shl = shl <4 x i32> %x, %amt
+ %ok = icmp ult <4 x i32> %amt, splat (i32 32)
+ %res = select <4 x i1> %ok, <4 x i32> %shl, <4 x i32> zeroinitializer
+ ret <4 x i32> %res
+}
+
+define <4 x i32> @unbounded_srl_v4i32(<4 x i32> %x, <4 x i32> %amt) {
+; CHECK-LABEL: unbounded_srl_v4i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi v2.4s, #32
+; CHECK-NEXT: neg v3.4s, v1.4s
+; CHECK-NEXT: ushl v0.4s, v0.4s, v3.4s
+; CHECK-NEXT: cmhi v1.4s, v2.4s, v1.4s
+; CHECK-NEXT: and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT: ret
+entry:
+ %srl = lshr <4 x i32> %x, %amt
+ %ok = icmp ult <4 x i32> %amt, splat (i32 32)
+ %res = select <4 x i1> %ok, <4 x i32> %srl, <4 x i32> zeroinitializer
+ ret <4 x i32> %res
+}
+
+define <4 x i32> @masked_shl_v4i32_mask255(<4 x i32> %x, <4 x i32> %amt) {
+; CHECK-LABEL: masked_shl_v4i32_mask255:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi v2.2d, #0x0000ff000000ff
+; CHECK-NEXT: movi v3.4s, #32
+; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
+; CHECK-NEXT: ushl v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: cmhi v1.4s, v3.4s, v1.4s
+; CHECK-NEXT: and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT: ret
+entry:
+ %m = and <4 x i32> %amt, splat (i32 255)
+ %shl = shl <4 x i32> %x, %m
+ %ok = icmp ult <4 x i32> %m, splat (i32 32)
+ %res = select <4 x i1> %ok, <4 x i32> %shl, <4 x i32> zeroinitializer
+ ret <4 x i32> %res
+}
+
+define <2 x i64> @unbounded_shl_v2i64_sve(<2 x i64> %x, <2 x i64> %amt) #0 {
+; CHECK-LABEL: unbounded_shl_v2i64_sve:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: mov z2.d, #64 // =0x40
+; CHECK-NEXT: ushl v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: cmhi v1.2d, v2.2d, v1.2d
+; CHECK-NEXT: and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT: ret
+entry:
+ %shl = shl <2 x i64> %x, %amt
+ %ok = icmp ult <2 x i64> %amt, splat (i64 64)
+ %res = select <2 x i1> %ok, <2 x i64> %shl, <2 x i64> zeroinitializer
+ ret <2 x i64> %res
+}
+
+define <2 x i64> @neg_unbounded_shl_v2i64(<2 x i64> %x, <2 x i64> %amt) {
+; CHECK-LABEL: neg_unbounded_shl_v2i64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: mov w8, #64 // =0x40
+; CHECK-NEXT: ushl v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: dup v2.2d, x8
+; CHECK-NEXT: cmhi v1.2d, v2.2d, v1.2d
+; CHECK-NEXT: and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT: ret
+entry:
+ %shl = shl <2 x i64> %x, %amt
+ %ok = icmp ult <2 x i64> %amt, splat (i64 64)
+ %res = select <2 x i1> %ok, <2 x i64> %shl, <2 x i64> zeroinitializer
+ ret <2 x i64> %res
+}
+
+define <4 x i32> @neg_masked_shl_v4i32_wrong_bound(<4 x i32> %x, <4 x i32> %amt) {
+; CHECK-LABEL: neg_masked_shl_v4i32_wrong_bound:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi v2.4s, #63
+; CHECK-NEXT: movi v3.4s, #16
+; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
+; CHECK-NEXT: ushl v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: cmhi v1.4s, v3.4s, v1.4s
+; CHECK-NEXT: and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT: ret
+entry:
+ %m = and <4 x i32> %amt, splat (i32 63)
+ %shl = shl <4 x i32> %x, %m
+ %ok = icmp ult <4 x i32> %m, splat (i32 16)
+ %res = select <4 x i1> %ok, <4 x i32> %shl, <4 x i32> zeroinitializer
+ ret <4 x i32> %res
+}
+
+define <4 x i32> @neg_masked_ashr_v4i32(<4 x i32> %x, <4 x i32> %amt) {
+; CHECK-LABEL: neg_masked_ashr_v4i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi v2.4s, #63
+; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
+; CHECK-NEXT: movi v2.4s, #32
+; CHECK-NEXT: neg v3.4s, v1.4s
+; CHECK-NEXT: cmhi v1.4s, v2.4s, v1.4s
+; CHECK-NEXT: sshl v0.4s, v0.4s, v3.4s
+; CHECK-NEXT: and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT: ret
+entry:
+ %m = and <4 x i32> %amt, splat (i32 63)
+ %ashr = ashr <4 x i32> %x, %m
+ %ok = icmp ult <4 x i32> %m, splat (i32 32)
+ %res = select <4 x i1> %ok, <4 x i32> %ashr, <4 x i32> zeroinitializer
+ ret <4 x i32> %res
+}
+
+attributes #0 = { "target-features"="+sve" }
>From 8666b022ba8faf71f1e9038763200a17a003c53f Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Fri, 10 Jul 2026 02:08:02 +0000
Subject: [PATCH 2/3] [AArch64] Fold vector shifts guarded against oversized
amounts into USHL
---
.../Target/AArch64/AArch64ISelLowering.cpp | 79 +++++++++++++++++++
.../CodeGen/AArch64/vselect-masked-shift.ll | 66 ++++++++--------
2 files changed, 110 insertions(+), 35 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 2f62476743396..9cdd0500db3cd 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -29138,6 +29138,82 @@ static SDValue performVselectPowCombine(SDNode *N,
return DAG.getNode(ISD::VSELECT, DL, VT, Cond, TrueVal, NewPow);
}
+// A vselect can zero the lanes with an out-of-range shift amount with either:
+// vselect(setcc_ult(amt, EltSize), val, zeros)
+// vselect(setcc_uge(amt, EltSize), zeros, val)
+// Returns val for whichever polarity is present and sets RequiredCC to the
+// condition code the setcc has to be using.
+static SDValue matchZeroSelectArm(SDValue TVal, SDValue FVal,
+ ISD::CondCode &RequiredCC) {
+ if (ISD::isConstantSplatVectorAllZeros(FVal.getNode())) {
+ RequiredCC = ISD::SETULT;
+ return TVal;
+ }
+ if (ISD::isConstantSplatVectorAllZeros(TVal.getNode())) {
+ RequiredCC = ISD::SETUGE;
+ return FVal;
+ }
+ return SDValue();
+}
+
+// ushl already produces zero for shift amounts of EltSize or more, so a
+// select zeroing those lanes is redundant. However, ushl reads each lane's
+// shift amount as a signed value from its low byte and would misread amounts
+// above 127, so the amounts must either be provably at most 127 or get
+// clamped to EltSize with umin.
+static SDValue foldMaskedShiftToUSHL(SelectionDAG &DAG,
+ const AArch64Subtarget *Subtarget,
+ SDNode *N, SDValue X, SDValue Amt,
+ SDValue Cond, ISD::CondCode RequiredCC,
+ bool IsSRL) {
+ using namespace llvm::SDPatternMatch;
+ EVT VT = N->getValueType(0);
+ if (!Subtarget->isNeonAvailable() || !VT.isFixedLengthVector() ||
+ !VT.isInteger() || !DAG.getTargetLoweringInfo().isTypeLegal(VT))
+ return SDValue();
+
+ unsigned EltSize = VT.getScalarSizeInBits();
+ if (!sd_match(Cond, m_SetCC(m_Specific(Amt), m_SpecificInt(EltSize),
+ m_SpecificCondCode(RequiredCC))))
+ return SDValue();
+
+ SDLoc DL(N);
+
+ // Amounts that might exceed 127 need the umin clamp.
+ if (!DAG.computeKnownBits(Amt).getMaxValue().ule(127)) {
+ // Only SVE has a umin for 64-bit lanes.
+ if (EltSize == 64 && !Subtarget->isSVEAvailable())
+ return SDValue();
+ Amt = DAG.getNode(ISD::UMIN, DL, VT, Amt, DAG.getConstant(EltSize, DL, VT));
+ }
+
+ // There is no shift right register instruction but ushl shifts right when
+ // the amount is negative.
+ if (IsSRL)
+ Amt = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Amt);
+ return DAG.getNode(
+ ISD::INTRINSIC_WO_CHAIN, DL, VT,
+ DAG.getTargetConstant(Intrinsic::aarch64_neon_ushl, DL, MVT::i32), X,
+ Amt);
+}
+
+// vselect(setcc_ult(amt, EltSize), shl(x, amt), zeros) -> ushl(x, amt)
+// vselect(setcc_ult(amt, EltSize), srl(x, amt), zeros) -> ushl(x, -amt)
+static SDValue
+performVSelectMaskedShiftCombine(SDNode *N, SelectionDAG &DAG,
+ const AArch64Subtarget *Subtarget) {
+ ISD::CondCode RequiredCC;
+ SDValue Shift =
+ matchZeroSelectArm(N->getOperand(1), N->getOperand(2), RequiredCC);
+ if (!Shift ||
+ (Shift.getOpcode() != ISD::SHL && Shift.getOpcode() != ISD::SRL))
+ return SDValue();
+
+ return foldMaskedShiftToUSHL(DAG, Subtarget, N, Shift.getOperand(0),
+ Shift.getOperand(1), N->getOperand(0),
+ RequiredCC, Shift.getOpcode() == ISD::SRL);
+}
+
// vselect (v1i1 setcc) ->
// vselect (v1iXX setcc) (XX is the size of the compared operand type)
// FIXME: Currently the type legalizer can't handle VSELECT having v1i1 as
@@ -29151,6 +29227,9 @@ static SDValue performVSelectCombine(SDNode *N,
if (auto SwapResult = trySwapVSelectOperands(N, DAG))
return SwapResult;
+ if (SDValue Shift = performVSelectMaskedShiftCombine(N, DAG, Subtarget))
+ return Shift;
+
SDValue N0 = N->getOperand(0);
SDValue IfTrue = N->getOperand(1);
SDValue IfFalse = N->getOperand(2);
diff --git a/llvm/test/CodeGen/AArch64/vselect-masked-shift.ll b/llvm/test/CodeGen/AArch64/vselect-masked-shift.ll
index ce111c7950e4d..42c29447c0e31 100644
--- a/llvm/test/CodeGen/AArch64/vselect-masked-shift.ll
+++ b/llvm/test/CodeGen/AArch64/vselect-masked-shift.ll
@@ -5,11 +5,8 @@ define <4 x i32> @masked_shl_v4i32(<4 x i32> %x, <4 x i32> %amt) {
; CHECK-LABEL: masked_shl_v4i32:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: movi v2.4s, #63
-; CHECK-NEXT: movi v3.4s, #32
; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
; CHECK-NEXT: ushl v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: cmhi v1.4s, v3.4s, v1.4s
-; CHECK-NEXT: and v0.16b, v1.16b, v0.16b
; CHECK-NEXT: ret
entry:
%m = and <4 x i32> %amt, splat (i32 63)
@@ -23,11 +20,8 @@ define <4 x i32> @masked_shl_v4i32_swapped(<4 x i32> %x, <4 x i32> %amt) {
; CHECK-LABEL: masked_shl_v4i32_swapped:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: movi v2.4s, #63
-; CHECK-NEXT: movi v3.4s, #32
; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
; CHECK-NEXT: ushl v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: cmhi v1.4s, v3.4s, v1.4s
-; CHECK-NEXT: and v0.16b, v1.16b, v0.16b
; CHECK-NEXT: ret
entry:
%m = and <4 x i32> %amt, splat (i32 63)
@@ -42,11 +36,8 @@ define <4 x i32> @masked_srl_v4i32(<4 x i32> %x, <4 x i32> %amt) {
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: movi v2.4s, #63
; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
-; CHECK-NEXT: movi v2.4s, #32
-; CHECK-NEXT: neg v3.4s, v1.4s
-; CHECK-NEXT: cmhi v1.4s, v2.4s, v1.4s
-; CHECK-NEXT: ushl v0.4s, v0.4s, v3.4s
-; CHECK-NEXT: and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT: neg v1.4s, v1.4s
+; CHECK-NEXT: ushl v0.4s, v0.4s, v1.4s
; CHECK-NEXT: ret
entry:
%m = and <4 x i32> %amt, splat (i32 63)
@@ -60,11 +51,8 @@ define <16 x i8> @masked_shl_v16i8(<16 x i8> %x, <16 x i8> %amt) {
; CHECK-LABEL: masked_shl_v16i8:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: movi v2.16b, #15
-; CHECK-NEXT: movi v3.16b, #8
; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
; CHECK-NEXT: ushl v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: cmhi v1.16b, v3.16b, v1.16b
-; CHECK-NEXT: and v0.16b, v1.16b, v0.16b
; CHECK-NEXT: ret
entry:
%m = and <16 x i8> %amt, splat (i8 15)
@@ -78,11 +66,8 @@ define <8 x i16> @masked_shl_v8i16(<8 x i16> %x, <8 x i16> %amt) {
; CHECK-LABEL: masked_shl_v8i16:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: movi v2.8h, #31
-; CHECK-NEXT: movi v3.8h, #16
; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
; CHECK-NEXT: ushl v0.8h, v0.8h, v1.8h
-; CHECK-NEXT: cmhi v1.8h, v3.8h, v1.8h
-; CHECK-NEXT: and v0.16b, v1.16b, v0.16b
; CHECK-NEXT: ret
entry:
%m = and <8 x i16> %amt, splat (i16 31)
@@ -97,12 +82,8 @@ define <2 x i64> @masked_shl_v2i64(<2 x i64> %x, <2 x i64> %amt) {
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: mov w8, #127 // =0x7f
; CHECK-NEXT: dup v2.2d, x8
-; CHECK-NEXT: mov w8, #64 // =0x40
; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
-; CHECK-NEXT: dup v2.2d, x8
; CHECK-NEXT: ushl v0.2d, v0.2d, v1.2d
-; CHECK-NEXT: cmhi v1.2d, v2.2d, v1.2d
-; CHECK-NEXT: and v0.16b, v1.16b, v0.16b
; CHECK-NEXT: ret
entry:
%m = and <2 x i64> %amt, splat (i64 127)
@@ -116,11 +97,8 @@ define <2 x i32> @masked_shl_v2i32(<2 x i32> %x, <2 x i32> %amt) {
; CHECK-LABEL: masked_shl_v2i32:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: movi v2.2s, #63
-; CHECK-NEXT: movi v3.2s, #32
; CHECK-NEXT: and v1.8b, v1.8b, v2.8b
; CHECK-NEXT: ushl v0.2s, v0.2s, v1.2s
-; CHECK-NEXT: cmhi v1.2s, v3.2s, v1.2s
-; CHECK-NEXT: and v0.8b, v1.8b, v0.8b
; CHECK-NEXT: ret
entry:
%m = and <2 x i32> %amt, splat (i32 63)
@@ -134,9 +112,8 @@ define <4 x i32> @unbounded_shl_v4i32(<4 x i32> %x, <4 x i32> %amt) {
; CHECK-LABEL: unbounded_shl_v4i32:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: movi v2.4s, #32
+; CHECK-NEXT: umin v1.4s, v1.4s, v2.4s
; CHECK-NEXT: ushl v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: cmhi v1.4s, v2.4s, v1.4s
-; CHECK-NEXT: and v0.16b, v1.16b, v0.16b
; CHECK-NEXT: ret
entry:
%shl = shl <4 x i32> %x, %amt
@@ -149,10 +126,9 @@ define <4 x i32> @unbounded_srl_v4i32(<4 x i32> %x, <4 x i32> %amt) {
; CHECK-LABEL: unbounded_srl_v4i32:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: movi v2.4s, #32
-; CHECK-NEXT: neg v3.4s, v1.4s
-; CHECK-NEXT: ushl v0.4s, v0.4s, v3.4s
-; CHECK-NEXT: cmhi v1.4s, v2.4s, v1.4s
-; CHECK-NEXT: and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT: umin v1.4s, v1.4s, v2.4s
+; CHECK-NEXT: neg v1.4s, v1.4s
+; CHECK-NEXT: ushl v0.4s, v0.4s, v1.4s
; CHECK-NEXT: ret
entry:
%srl = lshr <4 x i32> %x, %amt
@@ -167,9 +143,8 @@ define <4 x i32> @masked_shl_v4i32_mask255(<4 x i32> %x, <4 x i32> %amt) {
; CHECK-NEXT: movi v2.2d, #0x0000ff000000ff
; CHECK-NEXT: movi v3.4s, #32
; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
+; CHECK-NEXT: umin v1.4s, v1.4s, v3.4s
; CHECK-NEXT: ushl v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: cmhi v1.4s, v3.4s, v1.4s
-; CHECK-NEXT: and v0.16b, v1.16b, v0.16b
; CHECK-NEXT: ret
entry:
%m = and <4 x i32> %amt, splat (i32 255)
@@ -182,10 +157,9 @@ entry:
define <2 x i64> @unbounded_shl_v2i64_sve(<2 x i64> %x, <2 x i64> %amt) #0 {
; CHECK-LABEL: unbounded_shl_v2i64_sve:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: mov z2.d, #64 // =0x40
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: umin z1.d, z1.d, #64
; CHECK-NEXT: ushl v0.2d, v0.2d, v1.2d
-; CHECK-NEXT: cmhi v1.2d, v2.2d, v1.2d
-; CHECK-NEXT: and v0.16b, v1.16b, v0.16b
; CHECK-NEXT: ret
entry:
%shl = shl <2 x i64> %x, %amt
@@ -194,6 +168,28 @@ entry:
ret <2 x i64> %res
}
+define <1 x i128> @neg_masked_shl_v1i128(<1 x i128> %x, <1 x i128> %amt) {
+; CHECK-LABEL: neg_masked_shl_v1i128:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: and x8, x2, #0x3f
+; CHECK-NEXT: lsr x9, x0, #1
+; CHECK-NEXT: lsl x11, x1, x2
+; CHECK-NEXT: eor x10, x8, #0x3f
+; CHECK-NEXT: cmp x8, #128
+; CHECK-NEXT: lsr x9, x9, x10
+; CHECK-NEXT: lsl x10, x0, x2
+; CHECK-NEXT: orr x9, x11, x9
+; CHECK-NEXT: csel x0, x10, xzr, lo
+; CHECK-NEXT: csel x1, x9, xzr, lo
+; CHECK-NEXT: ret
+entry:
+ %m = and <1 x i128> %amt, splat (i128 63)
+ %shl = shl <1 x i128> %x, %m
+ %ok = icmp ult <1 x i128> %m, splat (i128 128)
+ %res = select <1 x i1> %ok, <1 x i128> %shl, <1 x i128> zeroinitializer
+ ret <1 x i128> %res
+}
+
define <2 x i64> @neg_unbounded_shl_v2i64(<2 x i64> %x, <2 x i64> %amt) {
; CHECK-LABEL: neg_unbounded_shl_v2i64:
; CHECK: // %bb.0: // %entry
>From 52f8d83f9d654624081acc36c656708ceed90209 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Thu, 16 Jul 2026 05:09:05 +0000
Subject: [PATCH 3/3] [AArch64] Only fold to USHL for 64 and 128 bit vectors
---
.../Target/AArch64/AArch64ISelLowering.cpp | 4 ++-
.../CodeGen/AArch64/vselect-masked-shift.ll | 31 +++++++++++++++++++
2 files changed, 34 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 9cdd0500db3cd..9d272c07da277 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -29168,8 +29168,10 @@ static SDValue foldMaskedShiftToUSHL(SelectionDAG &DAG,
bool IsSRL) {
using namespace llvm::SDPatternMatch;
EVT VT = N->getValueType(0);
+ // ushl only exists for 64 and 128 bit vectors.
if (!Subtarget->isNeonAvailable() || !VT.isFixedLengthVector() ||
- !VT.isInteger() || !DAG.getTargetLoweringInfo().isTypeLegal(VT))
+ !VT.isInteger() || !(VT.is64BitVector() || VT.is128BitVector()) ||
+ !DAG.getTargetLoweringInfo().isTypeLegal(VT))
return SDValue();
unsigned EltSize = VT.getScalarSizeInBits();
diff --git a/llvm/test/CodeGen/AArch64/vselect-masked-shift.ll b/llvm/test/CodeGen/AArch64/vselect-masked-shift.ll
index 42c29447c0e31..2d5db9ad8b2f1 100644
--- a/llvm/test/CodeGen/AArch64/vselect-masked-shift.ll
+++ b/llvm/test/CodeGen/AArch64/vselect-masked-shift.ll
@@ -244,3 +244,34 @@ entry:
}
attributes #0 = { "target-features"="+sve" }
+
+define <8 x i32> @neg_masked_shl_v8i32_sve_vls(<8 x i32> %x, <8 x i32> %amt) #1 {
+; CHECK-LABEL: neg_masked_shl_v8i32_sve_vls:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: ptrue p1.s
+; CHECK-NEXT: splice z2.s, p0, z2.s, z3.s
+; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s
+; CHECK-NEXT: and z2.s, z2.s, #0x3f
+; CHECK-NEXT: cmplo p0.s, p1/z, z2.s, #32
+; CHECK-NEXT: lsl z0.s, p1/m, z0.s, z2.s
+; CHECK-NEXT: mov z1.s, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: and z0.d, z1.d, z0.d
+; CHECK-NEXT: movprfx z1, z0
+; CHECK-NEXT: ext z1.b, z1.b, z0.b, #16
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z1
+; CHECK-NEXT: ret
+entry:
+ %m = and <8 x i32> %amt, splat (i32 63)
+ %shl = shl <8 x i32> %x, %m
+ %ok = icmp ult <8 x i32> %m, splat (i32 32)
+ %res = select <8 x i1> %ok, <8 x i32> %shl, <8 x i32> zeroinitializer
+ ret <8 x i32> %res
+}
+
+attributes #1 = { "target-features"="+sve" vscale_range(2,2) }
More information about the llvm-commits
mailing list