[llvm] [AArch64] Combine SVE shift and truncate into deinterleave (PR #213252)
Jacob Crawley via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 31 09:00:56 PDT 2026
https://github.com/jacob-crawley updated https://github.com/llvm/llvm-project/pull/213252
>From 0554d0b45d73ff2e549f0fb41ab9b0c7a4334866 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Fri, 31 Jul 2026 11:38:12 +0000
Subject: [PATCH 1/2] [AArch64] Combine SVE shift and truncate into
deinterleave
Fold a logical or arithmetic right shift by half the source element width
followed by a truncation into a bitcast and VECTOR_DEINTERLEAVE. This
selects to UZP and can also fold into an SVE interleaved load.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 51 +++++
.../AArch64/sve-shift-trunc-combine.ll | 204 ++++++++++++++++++
2 files changed, 255 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/sve-shift-trunc-combine.ll
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 9f5d254ad469e..cc5afa5a8daed 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -23643,8 +23643,59 @@ static SDValue trySQDMULHCombine(SDNode *N, SelectionDAG &DAG) {
return DAG.getNode(ISD::SIGN_EXTEND, DL, DestVT, SQDMULH);
}
+// Fold a shift by half the source element width followed by a truncation into
+// extraction of the upper half of each source element.
+static SDValue tryShiftTruncateCombine(SDNode *N, SelectionDAG &DAG,
+ TargetLowering::DAGCombinerInfo &DCI) {
+ if (!DCI.isBeforeLegalize())
+ return SDValue();
+
+ EVT DstVT = N->getValueType(0);
+ SDValue Shift = N->getOperand(0);
+
+ if (!DstVT.isScalableVector() || !DstVT.getVectorElementType().isInteger() ||
+ !DAG.getTargetLoweringInfo().isTypeLegal(DstVT))
+ return SDValue();
+
+ if ((Shift.getOpcode() != ISD::SRL && Shift.getOpcode() != ISD::SRA) ||
+ !Shift.hasOneUse())
+ return SDValue();
+
+ EVT SrcVT = Shift.getValueType();
+ if (!SrcVT.isScalableVector() || !SrcVT.getVectorElementType().isInteger() ||
+ SrcVT.getVectorElementCount() != DstVT.getVectorElementCount() ||
+ SrcVT.getScalarSizeInBits() != 2 * DstVT.getScalarSizeInBits())
+ return SDValue();
+
+ ConstantSDNode *ShiftAmount = isConstOrConstSplat(Shift.getOperand(1));
+ if (!ShiftAmount ||
+ ShiftAmount->getAsZExtVal() != DstVT.getScalarSizeInBits())
+ return SDValue();
+
+ SDLoc DL(N);
+
+ EVT BitcastVT = DstVT.getDoubleNumVectorElementsVT(*DAG.getContext());
+ SDValue Bitcast = DAG.getBitcast(BitcastVT, Shift.getOperand(0));
+
+ SDValue Lo = DAG.getExtractSubvector(DL, DstVT, Bitcast, 0);
+ SDValue Hi = DAG.getExtractSubvector(DL, DstVT, Bitcast,
+ DstVT.getVectorMinNumElements());
+
+ SDValue Deinterleave = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
+ DAG.getVTList(DstVT, DstVT), {Lo, Hi});
+
+ // After bitcasting, the byte containing the shifted result is in the odd
+ // deinterleave result on little-endian targets, and the in the even result
+ // on big-endian targets.
+ unsigned Res = DAG.getDataLayout().isLittleEndian() ? 1 : 0;
+ return Deinterleave.getValue(Res);
+}
+
static SDValue performTruncateCombine(SDNode *N, SelectionDAG &DAG,
TargetLowering::DAGCombinerInfo &DCI) {
+ if (SDValue V = tryShiftTruncateCombine(N, DAG, DCI))
+ return V;
+
SDLoc DL(N);
EVT VT = N->getValueType(0);
SDValue N0 = N->getOperand(0);
diff --git a/llvm/test/CodeGen/AArch64/sve-shift-trunc-combine.ll b/llvm/test/CodeGen/AArch64/sve-shift-trunc-combine.ll
new file mode 100644
index 0000000000000..43b65a7f22a91
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-shift-trunc-combine.ll
@@ -0,0 +1,204 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64_be -mattr=+sve -verify-machineinstrs < %s | FileCheck %s --check-prefix=BE
+target triple = "aarch64"
+
+define void @lshr_trunc_i16_load(ptr %src, ptr %dst) {
+; CHECK-LABEL: lshr_trunc_i16_load:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr z0, [x0, #1, mul vl]
+; CHECK-NEXT: ldr z1, [x0]
+; CHECK-NEXT: uzp2 z0.b, z1.b, z0.b
+; CHECK-NEXT: str z0, [x1]
+; CHECK-NEXT: ret
+;
+; BE-LABEL: lshr_trunc_i16_load:
+; BE: // %bb.0:
+; BE-NEXT: ptrue p0.h
+; BE-NEXT: ld1h { z0.h }, p0/z, [x0, #1, mul vl]
+; BE-NEXT: ld1h { z1.h }, p0/z, [x0]
+; BE-NEXT: revb z0.h, p0/m, z0.h
+; BE-NEXT: revb z1.h, p0/m, z1.h
+; BE-NEXT: ptrue p0.b
+; BE-NEXT: uzp1 z0.b, z1.b, z0.b
+; BE-NEXT: st1b { z0.b }, p0, [x1]
+; BE-NEXT: ret
+ %x = load <vscale x 16 x i16>, ptr %src, align 2
+ %shift = lshr <vscale x 16 x i16> %x, splat (i16 8)
+ %result = trunc <vscale x 16 x i16> %shift to <vscale x 16 x i8>
+ store <vscale x 16 x i8> %result, ptr %dst, align 1
+ ret void
+}
+
+define void @ashr_trunc_i16_load(ptr %src, ptr %dst) {
+; CHECK-LABEL: ashr_trunc_i16_load:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr z0, [x0, #1, mul vl]
+; CHECK-NEXT: ldr z1, [x0]
+; CHECK-NEXT: uzp2 z0.b, z1.b, z0.b
+; CHECK-NEXT: str z0, [x1]
+; CHECK-NEXT: ret
+;
+; BE-LABEL: ashr_trunc_i16_load:
+; BE: // %bb.0:
+; BE-NEXT: ptrue p0.h
+; BE-NEXT: ld1h { z0.h }, p0/z, [x0, #1, mul vl]
+; BE-NEXT: ld1h { z1.h }, p0/z, [x0]
+; BE-NEXT: revb z0.h, p0/m, z0.h
+; BE-NEXT: revb z1.h, p0/m, z1.h
+; BE-NEXT: ptrue p0.b
+; BE-NEXT: uzp1 z0.b, z1.b, z0.b
+; BE-NEXT: st1b { z0.b }, p0, [x1]
+; BE-NEXT: ret
+ %x = load <vscale x 16 x i16>, ptr %src, align 2
+ %shift = ashr <vscale x 16 x i16> %x, splat (i16 8)
+ %result = trunc <vscale x 16 x i16> %shift to <vscale x 16 x i8>
+ store <vscale x 16 x i8> %result, ptr %dst, align 1
+ ret void
+}
+
+define void @lshr_trunc_i32_load(ptr %src, ptr %dst) {
+; CHECK-LABEL: lshr_trunc_i32_load:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr z0, [x0, #1, mul vl]
+; CHECK-NEXT: ldr z1, [x0]
+; CHECK-NEXT: uzp2 z0.h, z1.h, z0.h
+; CHECK-NEXT: str z0, [x1]
+; CHECK-NEXT: ret
+;
+; BE-LABEL: lshr_trunc_i32_load:
+; BE: // %bb.0:
+; BE-NEXT: ptrue p0.s
+; BE-NEXT: ld1w { z0.s }, p0/z, [x0, #1, mul vl]
+; BE-NEXT: ld1w { z1.s }, p0/z, [x0]
+; BE-NEXT: revb z0.s, p0/m, z0.s
+; BE-NEXT: revb z1.s, p0/m, z1.s
+; BE-NEXT: ptrue p0.h
+; BE-NEXT: revb z0.h, p0/m, z0.h
+; BE-NEXT: revb z1.h, p0/m, z1.h
+; BE-NEXT: uzp1 z0.h, z1.h, z0.h
+; BE-NEXT: st1h { z0.h }, p0, [x1]
+; BE-NEXT: ret
+ %x = load <vscale x 8 x i32>, ptr %src, align 2
+ %shift = lshr <vscale x 8 x i32> %x, splat (i32 16)
+ %result = trunc <vscale x 8 x i32> %shift to <vscale x 8 x i16>
+ store <vscale x 8 x i16> %result, ptr %dst, align 1
+ ret void
+}
+
+define void @lshr_trunc_i64_load(ptr %src, ptr %dst) {
+; CHECK-LABEL: lshr_trunc_i64_load:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr z0, [x0, #1, mul vl]
+; CHECK-NEXT: ldr z1, [x0]
+; CHECK-NEXT: uzp2 z0.s, z1.s, z0.s
+; CHECK-NEXT: str z0, [x1]
+; CHECK-NEXT: ret
+;
+; BE-LABEL: lshr_trunc_i64_load:
+; BE: // %bb.0:
+; BE-NEXT: ptrue p0.s
+; BE-NEXT: ld1w { z0.s }, p0/z, [x0, #1, mul vl]
+; BE-NEXT: ld1w { z1.s }, p0/z, [x0]
+; BE-NEXT: uzp1 z0.s, z1.s, z0.s
+; BE-NEXT: st1w { z0.s }, p0, [x1]
+; BE-NEXT: ret
+ %x = load <vscale x 4 x i64>, ptr %src, align 2
+ %shift = lshr <vscale x 4 x i64> %x, splat (i64 32)
+ %result = trunc <vscale x 4 x i64> %shift to <vscale x 4 x i32>
+ store <vscale x 4 x i32> %result, ptr %dst, align 1
+ ret void
+}
+
+; Keep the load and bitcast in separate blocks so the high-byte extraction
+; remains visible to instruction selection, which can select it as an ld2b.
+define void @lshr_trunc_byte_load(ptr %src, ptr %dst) {
+; CHECK-LABEL: lshr_trunc_byte_load:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: ld2b { z0.b, z1.b }, p0/z, [x0]
+; CHECK-NEXT: str z1, [x1]
+; CHECK-NEXT: ret
+;
+; BE-LABEL: lshr_trunc_byte_load:
+; BE: // %bb.0: // %entry
+; BE-NEXT: ptrue p0.b
+; BE-NEXT: ld2b { z0.b, z1.b }, p0/z, [x0]
+; BE-NEXT: st1b { z0.b }, p0, [x1]
+; BE-NEXT: ret
+entry:
+ %bytes = load <vscale x 32 x i8>, ptr %src, align 2
+ br label %body
+
+body:
+ %x = bitcast <vscale x 32 x i8> %bytes to <vscale x 16 x i16>
+ %shift = lshr <vscale x 16 x i16> %x, splat (i16 8)
+ %result = trunc <vscale x 16 x i16> %shift to <vscale x 16 x i8>
+ store <vscale x 16 x i8> %result, ptr %dst, align 1
+ ret void
+}
+
+; Negative test: the shift amount must be equal to the destination width.
+define void @lshr_trunc_wrong_amount(ptr %src, ptr %dst) {
+; CHECK-LABEL: lshr_trunc_wrong_amount:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr z0, [x0]
+; CHECK-NEXT: ldr z1, [x0, #1, mul vl]
+; CHECK-NEXT: lsr z1.h, z1.h, #7
+; CHECK-NEXT: lsr z0.h, z0.h, #7
+; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b
+; CHECK-NEXT: str z0, [x1]
+; CHECK-NEXT: ret
+;
+; BE-LABEL: lshr_trunc_wrong_amount:
+; BE: // %bb.0:
+; BE-NEXT: ptrue p0.h
+; BE-NEXT: ld1h { z0.h }, p0/z, [x0]
+; BE-NEXT: ld1h { z1.h }, p0/z, [x0, #1, mul vl]
+; BE-NEXT: ptrue p0.b
+; BE-NEXT: lsr z1.h, z1.h, #7
+; BE-NEXT: lsr z0.h, z0.h, #7
+; BE-NEXT: uzp1 z0.b, z0.b, z1.b
+; BE-NEXT: st1b { z0.b }, p0, [x1]
+; BE-NEXT: ret
+ %x = load <vscale x 16 x i16>, ptr %src, align 2
+ %shift = lshr <vscale x 16 x i16> %x, splat (i16 7)
+ %result = trunc <vscale x 16 x i16> %shift to <vscale x 16 x i8>
+ store <vscale x 16 x i8> %result, ptr %dst, align 1
+ ret void
+}
+
+; Negative test: the shift should be preserved if it has another use.
+define void @lshr_trunc_multi_use(ptr %src, ptr %dst.narrow, ptr %dst.wide) {
+; CHECK-LABEL: lshr_trunc_multi_use:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr z0, [x0]
+; CHECK-NEXT: ldr z1, [x0, #1, mul vl]
+; CHECK-NEXT: lsr z1.h, z1.h, #8
+; CHECK-NEXT: lsr z0.h, z0.h, #8
+; CHECK-NEXT: uzp1 z2.b, z0.b, z1.b
+; CHECK-NEXT: str z2, [x1]
+; CHECK-NEXT: str z1, [x2, #1, mul vl]
+; CHECK-NEXT: str z0, [x2]
+; CHECK-NEXT: ret
+;
+; BE-LABEL: lshr_trunc_multi_use:
+; BE: // %bb.0:
+; BE-NEXT: ptrue p0.h
+; BE-NEXT: ptrue p1.b
+; BE-NEXT: ld1h { z0.h }, p0/z, [x0]
+; BE-NEXT: ld1h { z1.h }, p0/z, [x0, #1, mul vl]
+; BE-NEXT: lsr z1.h, z1.h, #8
+; BE-NEXT: lsr z0.h, z0.h, #8
+; BE-NEXT: uzp1 z2.b, z0.b, z1.b
+; BE-NEXT: st1b { z2.b }, p1, [x1]
+; BE-NEXT: st1h { z1.h }, p0, [x2, #1, mul vl]
+; BE-NEXT: st1h { z0.h }, p0, [x2]
+; BE-NEXT: ret
+ %x = load <vscale x 16 x i16>, ptr %src, align 2
+ %shift = lshr <vscale x 16 x i16> %x, splat (i16 8)
+ %result = trunc <vscale x 16 x i16> %shift to <vscale x 16 x i8>
+ store <vscale x 16 x i8> %result, ptr %dst.narrow, align 1
+ store <vscale x 16 x i16> %shift, ptr %dst.wide, align 2
+ ret void
+}
>From 537cda793ca584f179f83092a90262e7768f3a46 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Fri, 31 Jul 2026 16:00:06 +0000
Subject: [PATCH 2/2] Add guards to fix SVE regressions
---
.../Target/AArch64/AArch64ISelLowering.cpp | 27 ++++++++++++++++++-
1 file changed, 26 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index cc5afa5a8daed..d5e490bd6f8b3 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -23664,7 +23664,8 @@ static SDValue tryShiftTruncateCombine(SDNode *N, SelectionDAG &DAG,
EVT SrcVT = Shift.getValueType();
if (!SrcVT.isScalableVector() || !SrcVT.getVectorElementType().isInteger() ||
SrcVT.getVectorElementCount() != DstVT.getVectorElementCount() ||
- SrcVT.getScalarSizeInBits() != 2 * DstVT.getScalarSizeInBits())
+ SrcVT.getScalarSizeInBits() != 2 * DstVT.getScalarSizeInBits() ||
+ SrcVT.getScalarSizeInBits() > 64)
return SDValue();
ConstantSDNode *ShiftAmount = isConstOrConstSplat(Shift.getOperand(1));
@@ -23672,6 +23673,30 @@ static SDValue tryShiftTruncateCombine(SDNode *N, SelectionDAG &DAG,
ShiftAmount->getAsZExtVal() != DstVT.getScalarSizeInBits())
return SDValue();
+ // Preserve widening multiply patterns so they can select to SMULH or UMULH.
+ SDValue ShiftedValue = Shift.getOperand(0);
+ if (ShiftedValue.getOpcode() == ISD::MUL) {
+ SDValue LHS = ShiftedValue.getOperand(0);
+ SDValue RHS = ShiftedValue.getOperand(1);
+ unsigned ExtOpc = LHS.getOpcode();
+
+ if ((ExtOpc == ISD::SIGN_EXTEND || ExtOpc == ISD::ZERO_EXTEND) &&
+ RHS.getOpcode() == ExtOpc &&
+ LHS.getOperand(0).getValueType() == DstVT &&
+ RHS.getOperand(0).getValueType() == DstVT)
+ return SDValue();
+ }
+
+ // Preserve SVE2 rounding-shift patterns so they can select to RSHRNB.
+ const auto &Subtarget = DAG.getSubtarget<AArch64Subtarget>();
+ if (Subtarget.hasSVE2()) {
+ unsigned RoundingShiftAmount;
+ SDValue RoundingOperand;
+ if (canLowerSRLToRoundingShiftForVT(Shift, DstVT, DAG, RoundingShiftAmount,
+ RoundingOperand))
+ return SDValue();
+ }
+
SDLoc DL(N);
EVT BitcastVT = DstVT.getDoubleNumVectorElementsVT(*DAG.getContext());
More information about the llvm-commits
mailing list