[llvm] This fold corrects mismatched widened averaging idioms by folding: (PR #163366)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 14 15:04:04 PDT 2025
https://github.com/laurenmchin updated https://github.com/llvm/llvm-project/pull/163366
>From 1388d4e4832351f69c4e7662750e1d43ad432686 Mon Sep 17 00:00:00 2001
From: Lauren Chin <lchin at berkeley.edu>
Date: Tue, 14 Oct 2025 06:20:55 -0400
Subject: [PATCH] This fold corrects mismatched widened averaging idioms by
folding: `trunc(avgceilu(sext(x), sext(y))) -> avgceils(x, y)`
`trunc(avgceils(zext(x), zext(y))) -> avgceilu(x, y)`
When inputs are sign-extended, unsigned and signed averaging operations produce identical results after truncation, allowing us to use the semantically correct narrow operation.
alive2: https://alive2.llvm.org/ce/z/ZRbfHT
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 26 +++++-
llvm/test/CodeGen/AArch64/sve-hadd.ll | 81 +++++++++++++++++++
2 files changed, 105 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 787a81a99389f..0b0fc21c7247b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -16482,10 +16482,32 @@ SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
DAG, DL);
}
break;
- case ISD::AVGFLOORS:
- case ISD::AVGFLOORU:
case ISD::AVGCEILS:
case ISD::AVGCEILU:
+ // trunc (avgceilu (sext (x), sext (y))) -> avgceils(x, y)
+ // trunc (avgceils (zext (x), zext (y))) -> avgceilu(x, y)
+ if (N0.hasOneUse()) {
+ SDValue Op0 = N0.getOperand(0);
+ SDValue Op1 = N0.getOperand(1);
+ if (N0.getOpcode() == ISD::AVGCEILU) {
+ if (TLI.isOperationLegalOrCustom(ISD::AVGCEILS, VT) &&
+ (Op0.getOpcode() == ISD::SIGN_EXTEND ||
+ Op0.getOpcode() == ISD::SIGN_EXTEND_INREG) &&
+ (Op1.getOpcode() == ISD::SIGN_EXTEND ||
+ Op1.getOpcode() == ISD::SIGN_EXTEND_INREG))
+ return DAG.getNode(ISD::AVGCEILS, DL, VT,
+ N0.getOperand(0).getOperand(0),
+ N0.getOperand(1).getOperand(0));
+ } else {
+ if (TLI.isOperationLegalOrCustom(ISD::AVGCEILU, VT) &&
+ N0.getOperand(0).getOpcode() == ISD::ZERO_EXTEND &&
+ N0.getOperand(1).getOpcode() == ISD::ZERO_EXTEND)
+ return DAG.getNode(ISD::AVGCEILU, DL, VT,
+ N0.getOperand(0).getOperand(0),
+ N0.getOperand(1).getOperand(0));
+ }
+ }
+ [[fallthrough]];
case ISD::ABDS:
case ISD::ABDU:
// (trunc (avg a, b)) -> (avg (trunc a), (trunc b))
diff --git a/llvm/test/CodeGen/AArch64/sve-hadd.ll b/llvm/test/CodeGen/AArch64/sve-hadd.ll
index 978ee4534e5e1..32e4a9accee32 100644
--- a/llvm/test/CodeGen/AArch64/sve-hadd.ll
+++ b/llvm/test/CodeGen/AArch64/sve-hadd.ll
@@ -2,6 +2,87 @@
; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mattr=+sve | FileCheck %s -check-prefixes=CHECK,SVE
; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mattr=+sve2 | FileCheck %s -check-prefixes=CHECK,SVE2
+define <8 x i8> @srhadd_v8i8_trunc(<8 x i8> %s0, <8 x i8> %s1) {
+; CHECK-LABEL: srhadd_v8i8_trunc:
+; CHECK: // %bb.0:
+; CHECK-NEXT: srhadd v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: ret
+ %s0s = sext <8 x i8> %s0 to <8 x i16>
+ %s1s = sext <8 x i8> %s1 to <8 x i16>
+ %s = call <8 x i16> @llvm.aarch64.neon.urhadd.v8i16(<8 x i16> %s0s, <8 x i16> %s1s)
+ %s2 = trunc <8 x i16> %s to <8 x i8>
+ ret <8 x i8> %s2
+}
+
+define <4 x i16> @srhadd_v4i16_trunc(<4 x i16> %s0, <4 x i16> %s1) {
+; CHECK-LABEL: srhadd_v4i16_trunc:
+; CHECK: // %bb.0:
+; CHECK-NEXT: srhadd v0.4h, v0.4h, v1.4h
+; CHECK-NEXT: ret
+ %s0s = sext <4 x i16> %s0 to <4 x i32>
+ %s1s = sext <4 x i16> %s1 to <4 x i32>
+ %s = call <4 x i32> @llvm.aarch64.neon.urhadd.v4i32(<4 x i32> %s0s, <4 x i32> %s1s)
+ %s2 = trunc <4 x i32> %s to <4 x i16>
+ ret <4 x i16> %s2
+}
+
+define <2 x i32> @srhadd_v2i32_trunc(<2 x i32> %s0, <2 x i32> %s1) {
+; CHECK-LABEL: srhadd_v2i32_trunc:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sshll v0.2d, v0.2s, #0
+; CHECK-NEXT: sshll v1.2d, v1.2s, #0
+; CHECK-NEXT: eor v2.16b, v0.16b, v1.16b
+; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: ushr v1.2d, v2.2d, #1
+; CHECK-NEXT: sub v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: xtn v0.2s, v0.2d
+; CHECK-NEXT: ret
+ %s0s = sext <2 x i32> %s0 to <2 x i64>
+ %s1s = sext <2 x i32> %s1 to <2 x i64>
+ %s = call <2 x i64> @llvm.aarch64.neon.urhadd.v2i64(<2 x i64> %s0s, <2 x i64> %s1s)
+ %s2 = trunc <2 x i64> %s to <2 x i32>
+ ret <2 x i32> %s2
+}
+
+define <8 x i8> @urhadd_v8i8_trunc(<8 x i8> %s0, <8 x i8> %s1) {
+; CHECK-LABEL: urhadd_v8i8_trunc:
+; CHECK: // %bb.0:
+; CHECK-NEXT: urhadd v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: ret
+ %s0s = zext <8 x i8> %s0 to <8 x i16>
+ %s1s = zext <8 x i8> %s1 to <8 x i16>
+ %s = call <8 x i16> @llvm.aarch64.neon.srhadd.v8i16(<8 x i16> %s0s, <8 x i16> %s1s)
+ %s2 = trunc <8 x i16> %s to <8 x i8>
+ ret <8 x i8> %s2
+}
+
+define <4 x i16> @urhadd_v4i16_trunc(<4 x i16> %s0, <4 x i16> %s1) {
+; CHECK-LABEL: urhadd_v4i16_trunc:
+; CHECK: // %bb.0:
+; CHECK-NEXT: urhadd v0.4h, v0.4h, v1.4h
+; CHECK-NEXT: ret
+ %s0s = zext <4 x i16> %s0 to <4 x i32>
+ %s1s = zext <4 x i16> %s1 to <4 x i32>
+ %s = call <4 x i32> @llvm.aarch64.neon.srhadd.v4i32(<4 x i32> %s0s, <4 x i32> %s1s)
+ %s2 = trunc <4 x i32> %s to <4 x i16>
+ ret <4 x i16> %s2
+}
+
+define <2 x i32> @urhadd_v2i32_trunc(<2 x i32> %s0, <2 x i32> %s1) {
+; CHECK-LABEL: urhadd_v2i32_trunc:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z2.d, #1 // =0x1
+; CHECK-NEXT: uaddl v0.2d, v0.2s, v1.2s
+; CHECK-NEXT: add v0.2d, v0.2d, v2.2d
+; CHECK-NEXT: shrn v0.2s, v0.2d, #1
+; CHECK-NEXT: ret
+ %s0s = zext <2 x i32> %s0 to <2 x i64>
+ %s1s = zext <2 x i32> %s1 to <2 x i64>
+ %s = call <2 x i64> @llvm.aarch64.neon.srhadd.v2i64(<2 x i64> %s0s, <2 x i64> %s1s)
+ %s2 = trunc <2 x i64> %s to <2 x i32>
+ ret <2 x i32> %s2
+}
+
define <vscale x 2 x i64> @hadds_v2i64(<vscale x 2 x i64> %s0, <vscale x 2 x i64> %s1) {
; SVE-LABEL: hadds_v2i64:
; SVE: // %bb.0: // %entry
More information about the llvm-commits
mailing list