[llvm] [AArch64][DAG] Widen small types for Min/Max opertations (PR #212966)
Jack Styles via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 30 02:17:09 PDT 2026
https://github.com/Stylie777 created https://github.com/llvm/llvm-project/pull/212966
For small types using small VF Values, when used with Min/Max Intrinsics, these are currently legalized using Promotion. However the CodeGen for promotion is currently slow and inefficient;
Example of Promotion from v2i8 to v2i32
```
ldr h0, [x1]
ldr h1, [x0], #2
sshll v0.8h, v0.8b, #0
sshll v1.8h, v1.8b, #0
sshll v0.4s, v0.4h, #0
sshll v1.4s, v1.4h, #0
smin v0.2s, v0.2s, v1.2s
mov s1, v0.s[1]
str b0, [x1]
```
This can be simplified to widen the type instead, only requiring ldr and str instructions to get the required;
```
ldr s0, [x1]
ldr s1, [x0], #4
smin v0.8b, v0.8b, v1.8b
str s0, [x1], #4
```
This provides a clear improvement in performance on loops where small types and small VF values are being utilised.
Assisted-by: Codex (Tests)
>From d5f22048b61d3c2da230824de205f178c4873edc Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Mon, 27 Jul 2026 11:12:37 +0100
Subject: [PATCH] [AArch64][DAG] Widen small types for Min/Max opertations
For small types using small VF Values, when used with Min/Max
Intrinsics, these are currently legalized using Promotion.
However the CodeGen for promotion is currently slow and
inefficient;
Example of Promotion from v2i8 to v2i32
```
ldr h0, [x1]
ldr h1, [x0], #2
sshll v0.8h, v0.8b, #0
sshll v1.8h, v1.8b, #0
sshll v0.4s, v0.4h, #0
sshll v1.4s, v1.4h, #0
smin v0.2s, v0.2s, v1.2s
mov s1, v0.s[1]
str b0, [x1]
```
This can be simplified to widen the type instead, only requiring
ldr and str instructions to get the required;
```
ldr s0, [x1]
ldr s1, [x0], #4
smin v0.8b, v0.8b, v1.8b
str s0, [x1], #4
```
This provides a clear improvement in performance on loops where
small types and small VF values are being utilised.
Assisted-by: Codex (Tests)
---
.../Target/AArch64/AArch64ISelLowering.cpp | 88 ++++++++-
.../AArch64/aarch64-minmax-small-type.ll | 176 ++++++++++++++++++
2 files changed, 262 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/aarch64-minmax-small-type.ll
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 8e3cf9d31a923..a81a1d1799e9b 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27525,6 +27525,46 @@ static SDValue
performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
SelectionDAG &DAG);
+static SDValue tryNarrowWidenedSmallVectorStore(
+ StoreSDNode *ST, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) {
+ SDValue Value = ST->getValue();
+ if (!DCI.isBeforeLegalize() || Value->getOpcode() != ISD::EXTRACT_SUBVECTOR ||
+ !isNullConstant(Value.getOperand(1)))
+ return SDValue();
+ SDValue WideValue = Value.getOperand(0);
+ unsigned int WideValueOpcode = WideValue.getOpcode();
+ // Only MinMax intrinsics have support for widening of the specific types
+ // currently
+ if (WideValueOpcode != ISD::SMIN && WideValueOpcode != ISD::SMAX &&
+ WideValueOpcode != ISD::UMIN && WideValueOpcode != ISD::UMAX)
+ return SDValue();
+
+ unsigned SubRegType;
+ MVT SubRegWidth;
+ EVT ValueVT = Value.getValueType();
+ if (!ValueVT.isSimple())
+ return SDValue();
+ switch (ValueVT.getSimpleVT().SimpleTy) {
+ case MVT::v2i8:
+ SubRegType = AArch64::hsub;
+ SubRegWidth = MVT::f16;
+ break;
+ case MVT::v4i8:
+ case MVT::v2i16:
+ SubRegType = AArch64::ssub;
+ SubRegWidth = MVT::f32;
+ break;
+ default:
+ return SDValue();
+ }
+ SDLoc DL(ST);
+ SDValue SubReg =
+ DAG.getTargetExtractSubreg(SubRegType, DL, SubRegWidth, WideValue);
+
+ return DAG.getStore(ST->getChain(), DL, SubReg, ST->getBasePtr(),
+ ST->getMemOperand());
+}
+
static SDValue performSTORECombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
SelectionDAG &DAG,
@@ -27553,6 +27593,8 @@ static SDValue performSTORECombine(SDNode *N,
ST->getBaseAlign(), ST->getMemOperand()->getFlags(),
ST->getAAInfo());
+ if (SDValue Res = tryNarrowWidenedSmallVectorStore(ST, DCI, DAG))
+ return Res;
if (SDValue Res = combineStoreValueFPToInt(ST, DCI, DAG, Subtarget))
return Res;
@@ -31068,12 +31110,54 @@ static unsigned getReductionForOpcode(unsigned Op) {
}
}
+SDValue
+tryWidenSmallVectorMinMaxLoadStore(SDNode *N, SelectionDAG &DAG,
+ const AArch64TargetLowering &TLI,
+ TargetLowering::DAGCombinerInfo &DCI) {
+ const AArch64Subtarget &Subtarget = DAG.getSubtarget<AArch64Subtarget>();
+ if (!DCI.isBeforeLegalize() || !Subtarget.isNeonAvailable() ||
+ !N->hasOneUse())
+ return SDValue();
+
+ EVT VT = N->getValueType(0);
+ EVT WideVT;
+ switch (VT.getSimpleVT().SimpleTy) {
+ case MVT::v2i8:
+ case MVT::v4i8:
+ WideVT = MVT::v8i8;
+ break;
+ case MVT::v2i16:
+ WideVT = MVT::v4i16;
+ break;
+ default:
+ return SDValue();
+ };
+ SDLoc DL(N);
+ auto WidenArg = [&](SDValue V) -> SDValue {
+ return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideVT, DAG.getPOISON(WideVT),
+ V, DAG.getVectorIdxConstant(0, DL));
+ };
+ SDValue WideArg0 = WidenArg(N->getOperand(0));
+ SDValue WideArg1 = WidenArg(N->getOperand(1));
+ SDValue WideMinMax =
+ DAG.getNode(N->getOpcode(), DL, WideVT, WideArg0, WideArg1);
+ return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, WideMinMax,
+ DAG.getVectorIdxConstant(0, DL));
+}
+
static SDValue performMINMAXCombine(SDNode *N, SelectionDAG &DAG,
- const AArch64TargetLowering &TLI) {
+ const AArch64TargetLowering &TLI,
+ TargetLowering::DAGCombinerInfo &DCI) {
using namespace llvm::SDPatternMatch;
if (SDValue V = trySQDMULHCombine(N, DAG))
return V;
+ // For small types used within a Min/Max Intrinsic, with a small VF such as
+ // 2, it is better to legalize the type via widening rather than promotion.
+ if (SDValue V = tryWidenSmallVectorMinMaxLoadStore(N, DAG, TLI, DCI)) {
+ return V;
+ }
+
unsigned ReductionOpcode = getReductionForOpcode(N->getOpcode());
if (!TLI.isOperationLegalOrCustom(ReductionOpcode, MVT::v2i64))
return SDValue();
@@ -31155,7 +31239,7 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
case ISD::UMIN:
case ISD::SMAX:
case ISD::SMIN:
- return performMINMAXCombine(N, DAG, *this);
+ return performMINMAXCombine(N, DAG, *this, DCI);
case ISD::TRUNCATE:
return performTruncateCombine(N, DAG, DCI);
case ISD::CTTZ:
diff --git a/llvm/test/CodeGen/AArch64/aarch64-minmax-small-type.ll b/llvm/test/CodeGen/AArch64/aarch64-minmax-small-type.ll
new file mode 100644
index 0000000000000..a9aac0403034e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/aarch64-minmax-small-type.ll
@@ -0,0 +1,176 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -verify-machineinstrs < %s | FileCheck %s
+
+; Check that small fixed-length integer min/max operations are widened in
+; registers to legal NEON types while retaining the original narrow memory
+; accesses.
+
+define void @smin_v2i8(ptr %dst, ptr %a, ptr %b) {
+; CHECK-LABEL: smin_v2i8:
+; CHECK: ldr h[[A:[0-9]+]], [x1]
+; CHECK-NEXT: ldr h[[B:[0-9]+]], [x2]
+; CHECK-NEXT: smin v[[A]].8b, v[[A]].8b, v[[B]].8b
+; CHECK-NEXT: str h[[A]], [x0]
+ %va = load <2 x i8>, ptr %a, align 1
+ %vb = load <2 x i8>, ptr %b, align 1
+ %min = call <2 x i8> @llvm.smin.v2i8(<2 x i8> %va, <2 x i8> %vb)
+ store <2 x i8> %min, ptr %dst, align 1
+ ret void
+}
+
+define void @smax_v2i8(ptr %dst, ptr %a, ptr %b) {
+; CHECK-LABEL: smax_v2i8:
+; CHECK: ldr h[[A:[0-9]+]], [x1]
+; CHECK-NEXT: ldr h[[B:[0-9]+]], [x2]
+; CHECK-NEXT: smax v[[A]].8b, v[[A]].8b, v[[B]].8b
+; CHECK-NEXT: str h[[A]], [x0]
+ %va = load <2 x i8>, ptr %a, align 1
+ %vb = load <2 x i8>, ptr %b, align 1
+ %max = call <2 x i8> @llvm.smax.v2i8(<2 x i8> %va, <2 x i8> %vb)
+ store <2 x i8> %max, ptr %dst, align 1
+ ret void
+}
+
+define void @umin_v2i8(ptr %dst, ptr %a, ptr %b) {
+; CHECK-LABEL: umin_v2i8:
+; CHECK: ldr h[[A:[0-9]+]], [x1]
+; CHECK-NEXT: ldr h[[B:[0-9]+]], [x2]
+; CHECK-NEXT: umin v[[A]].8b, v[[A]].8b, v[[B]].8b
+; CHECK-NEXT: str h[[A]], [x0]
+ %va = load <2 x i8>, ptr %a, align 1
+ %vb = load <2 x i8>, ptr %b, align 1
+ %min = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %va, <2 x i8> %vb)
+ store <2 x i8> %min, ptr %dst, align 1
+ ret void
+}
+
+define void @umax_v2i8(ptr %dst, ptr %a, ptr %b) {
+; CHECK-LABEL: umax_v2i8:
+; CHECK: ldr h[[A:[0-9]+]], [x1]
+; CHECK-NEXT: ldr h[[B:[0-9]+]], [x2]
+; CHECK-NEXT: umax v[[A]].8b, v[[A]].8b, v[[B]].8b
+; CHECK-NEXT: str h[[A]], [x0]
+ %va = load <2 x i8>, ptr %a, align 1
+ %vb = load <2 x i8>, ptr %b, align 1
+ %max = call <2 x i8> @llvm.umax.v2i8(<2 x i8> %va, <2 x i8> %vb)
+ store <2 x i8> %max, ptr %dst, align 1
+ ret void
+}
+
+define void @smin_v4i8(ptr %dst, ptr %a, ptr %b) {
+; CHECK-LABEL: smin_v4i8:
+; CHECK: ldr s[[A:[0-9]+]], [x1]
+; CHECK-NEXT: ldr s[[B:[0-9]+]], [x2]
+; CHECK-NEXT: smin v[[A]].8b, v[[A]].8b, v[[B]].8b
+; CHECK-NEXT: str s[[A]], [x0]
+ %va = load <4 x i8>, ptr %a, align 1
+ %vb = load <4 x i8>, ptr %b, align 1
+ %min = call <4 x i8> @llvm.smin.v4i8(<4 x i8> %va, <4 x i8> %vb)
+ store <4 x i8> %min, ptr %dst, align 1
+ ret void
+}
+
+define void @smax_v4i8(ptr %dst, ptr %a, ptr %b) {
+; CHECK-LABEL: smax_v4i8:
+; CHECK: ldr s[[A:[0-9]+]], [x1]
+; CHECK-NEXT: ldr s[[B:[0-9]+]], [x2]
+; CHECK-NEXT: smax v[[A]].8b, v[[A]].8b, v[[B]].8b
+; CHECK-NEXT: str s[[A]], [x0]
+ %va = load <4 x i8>, ptr %a, align 1
+ %vb = load <4 x i8>, ptr %b, align 1
+ %max = call <4 x i8> @llvm.smax.v4i8(<4 x i8> %va, <4 x i8> %vb)
+ store <4 x i8> %max, ptr %dst, align 1
+ ret void
+}
+
+define void @umin_v4i8(ptr %dst, ptr %a, ptr %b) {
+; CHECK-LABEL: umin_v4i8:
+; CHECK: ldr s[[A:[0-9]+]], [x1]
+; CHECK-NEXT: ldr s[[B:[0-9]+]], [x2]
+; CHECK-NEXT: umin v[[A]].8b, v[[A]].8b, v[[B]].8b
+; CHECK-NEXT: str s[[A]], [x0]
+ %va = load <4 x i8>, ptr %a, align 1
+ %vb = load <4 x i8>, ptr %b, align 1
+ %min = call <4 x i8> @llvm.umin.v4i8(<4 x i8> %va, <4 x i8> %vb)
+ store <4 x i8> %min, ptr %dst, align 1
+ ret void
+}
+
+define void @umax_v4i8(ptr %dst, ptr %a, ptr %b) {
+; CHECK-LABEL: umax_v4i8:
+; CHECK: ldr s[[A:[0-9]+]], [x1]
+; CHECK-NEXT: ldr s[[B:[0-9]+]], [x2]
+; CHECK-NEXT: umax v[[A]].8b, v[[A]].8b, v[[B]].8b
+; CHECK-NEXT: str s[[A]], [x0]
+ %va = load <4 x i8>, ptr %a, align 1
+ %vb = load <4 x i8>, ptr %b, align 1
+ %max = call <4 x i8> @llvm.umax.v4i8(<4 x i8> %va, <4 x i8> %vb)
+ store <4 x i8> %max, ptr %dst, align 1
+ ret void
+}
+
+define void @smin_v2i16(ptr %dst, ptr %a, ptr %b) {
+; CHECK-LABEL: smin_v2i16:
+; CHECK: ldr s[[A:[0-9]+]], [x1]
+; CHECK-NEXT: ldr s[[B:[0-9]+]], [x2]
+; CHECK-NEXT: smin v[[A]].4h, v[[A]].4h, v[[B]].4h
+; CHECK-NEXT: str s[[A]], [x0]
+ %va = load <2 x i16>, ptr %a, align 2
+ %vb = load <2 x i16>, ptr %b, align 2
+ %min = call <2 x i16> @llvm.smin.v2i16(<2 x i16> %va, <2 x i16> %vb)
+ store <2 x i16> %min, ptr %dst, align 2
+ ret void
+}
+
+define void @smax_v2i16(ptr %dst, ptr %a, ptr %b) {
+; CHECK-LABEL: smax_v2i16:
+; CHECK: ldr s[[A:[0-9]+]], [x1]
+; CHECK-NEXT: ldr s[[B:[0-9]+]], [x2]
+; CHECK-NEXT: smax v[[A]].4h, v[[A]].4h, v[[B]].4h
+; CHECK-NEXT: str s[[A]], [x0]
+ %va = load <2 x i16>, ptr %a, align 2
+ %vb = load <2 x i16>, ptr %b, align 2
+ %max = call <2 x i16> @llvm.smax.v2i16(<2 x i16> %va, <2 x i16> %vb)
+ store <2 x i16> %max, ptr %dst, align 2
+ ret void
+}
+
+define void @umin_v2i16(ptr %dst, ptr %a, ptr %b) {
+; CHECK-LABEL: umin_v2i16:
+; CHECK: ldr s[[A:[0-9]+]], [x1]
+; CHECK-NEXT: ldr s[[B:[0-9]+]], [x2]
+; CHECK-NEXT: umin v[[A]].4h, v[[A]].4h, v[[B]].4h
+; CHECK-NEXT: str s[[A]], [x0]
+ %va = load <2 x i16>, ptr %a, align 2
+ %vb = load <2 x i16>, ptr %b, align 2
+ %min = call <2 x i16> @llvm.umin.v2i16(<2 x i16> %va, <2 x i16> %vb)
+ store <2 x i16> %min, ptr %dst, align 2
+ ret void
+}
+
+define void @umax_v2i16(ptr %dst, ptr %a, ptr %b) {
+; CHECK-LABEL: umax_v2i16:
+; CHECK: ldr s[[A:[0-9]+]], [x1]
+; CHECK-NEXT: ldr s[[B:[0-9]+]], [x2]
+; CHECK-NEXT: umax v[[A]].4h, v[[A]].4h, v[[B]].4h
+; CHECK-NEXT: str s[[A]], [x0]
+ %va = load <2 x i16>, ptr %a, align 2
+ %vb = load <2 x i16>, ptr %b, align 2
+ %max = call <2 x i16> @llvm.umax.v2i16(<2 x i16> %va, <2 x i16> %vb)
+ store <2 x i16> %max, ptr %dst, align 2
+ ret void
+}
+
+declare <2 x i8> @llvm.smin.v2i8(<2 x i8>, <2 x i8>)
+declare <2 x i8> @llvm.smax.v2i8(<2 x i8>, <2 x i8>)
+declare <2 x i8> @llvm.umin.v2i8(<2 x i8>, <2 x i8>)
+declare <2 x i8> @llvm.umax.v2i8(<2 x i8>, <2 x i8>)
+
+declare <4 x i8> @llvm.smin.v4i8(<4 x i8>, <4 x i8>)
+declare <4 x i8> @llvm.smax.v4i8(<4 x i8>, <4 x i8>)
+declare <4 x i8> @llvm.umin.v4i8(<4 x i8>, <4 x i8>)
+declare <4 x i8> @llvm.umax.v4i8(<4 x i8>, <4 x i8>)
+
+declare <2 x i16> @llvm.smin.v2i16(<2 x i16>, <2 x i16>)
+declare <2 x i16> @llvm.smax.v2i16(<2 x i16>, <2 x i16>)
+declare <2 x i16> @llvm.umin.v2i16(<2 x i16>, <2 x i16>)
+declare <2 x i16> @llvm.umax.v2i16(<2 x i16>, <2 x i16>)
More information about the llvm-commits
mailing list