[llvm] [AArch64][ISel] Recover ADDHN from OR comparison masks (PR #210943)
Harry Ramsey via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 22 07:36:05 PDT 2026
https://github.com/Harry-Ramsey updated https://github.com/llvm/llvm-project/pull/210943
>From ab8690061120b19b0b630f7f0cb128a0308bc5e0 Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Tue, 21 Jul 2026 10:07:30 +0000
Subject: [PATCH 1/2] [AArch64][ISel] Recover ADDHN from OR comparison masks
InstCombine converts the high-half addition of vector comparison masks
into an OR. Recognise the resulting trunc(or(setcc, setcc)) DAG and
reconstruct the canonical ADDHN pattern for the supported NEON types.
---
llvm/test/CodeGen/AArch64/neon-addhn.ll | 53 +++++++++++++++++++++++++
1 file changed, 53 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/neon-addhn.ll
diff --git a/llvm/test/CodeGen/AArch64/neon-addhn.ll b/llvm/test/CodeGen/AArch64/neon-addhn.ll
new file mode 100644
index 0000000000000..08f4d2e593716
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/neon-addhn.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=aarch64 | FileCheck %s
+
+define i1 @addhn_setcc_v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> %comparand) {
+; CHECK-LABEL: addhn_setcc_v8i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmeq v0.8h, v0.8h, v2.8h
+; CHECK-NEXT: cmeq v1.8h, v1.8h, v2.8h
+; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: xtn v0.8b, v0.8h
+; CHECK-NEXT: umov w8, v0.b[0]
+; CHECK-NEXT: and w0, w8, #0x1
+; CHECK-NEXT: ret
+ %cmp.a = icmp eq <8 x i16> %a, %comparand
+ %cmp.b = icmp eq <8 x i16> %b, %comparand
+ %either = or <8 x i1> %cmp.a, %cmp.b
+ %lane = extractelement <8 x i1> %either, i64 0
+ ret i1 %lane
+}
+
+define i1 @addhn_setcc_v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %comparand) {
+; CHECK-LABEL: addhn_setcc_v4i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmeq v0.4s, v0.4s, v2.4s
+; CHECK-NEXT: cmeq v1.4s, v1.4s, v2.4s
+; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: xtn v0.4h, v0.4s
+; CHECK-NEXT: umov w8, v0.h[0]
+; CHECK-NEXT: and w0, w8, #0x1
+; CHECK-NEXT: ret
+ %cmp.a = icmp eq <4 x i32> %a, %comparand
+ %cmp.b = icmp eq <4 x i32> %b, %comparand
+ %either = or <4 x i1> %cmp.a, %cmp.b
+ %lane = extractelement <4 x i1> %either, i64 0
+ ret i1 %lane
+}
+
+define i1 @addhn_setcc_v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> %comparand) {
+; CHECK-LABEL: addhn_setcc_v2i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmeq v0.2d, v0.2d, v2.2d
+; CHECK-NEXT: cmeq v1.2d, v1.2d, v2.2d
+; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: xtn v0.2s, v0.2d
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: and w0, w8, #0x1
+; CHECK-NEXT: ret
+ %cmp.a = icmp eq <2 x i64> %a, %comparand
+ %cmp.b = icmp eq <2 x i64> %b, %comparand
+ %either = or <2 x i1> %cmp.a, %cmp.b
+ %lane = extractelement <2 x i1> %either, i64 0
+ ret i1 %lane
+}
>From 821a814c453de721cf41b6ce222c9594faa1b601 Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Tue, 21 Jul 2026 10:07:53 +0000
Subject: [PATCH 2/2] fixup! [AArch64][ISel] Recover ADDHN from OR comparison
masks
---
llvm/lib/Target/AArch64/AArch64Combine.td | 15 ++++-
.../Target/AArch64/AArch64ISelLowering.cpp | 18 +++++
.../GISel/AArch64PostLegalizerCombiner.cpp | 65 +++++++++++++++++++
.../AArch64/intrinsic-vector-match-sve2.ll | 3 +-
llvm/test/CodeGen/AArch64/is_fpclass.ll | 3 +-
llvm/test/CodeGen/AArch64/neon-addhn.ll | 9 +--
.../CodeGen/AArch64/select-with-and-or.ll | 12 ++--
7 files changed, 105 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64Combine.td b/llvm/lib/Target/AArch64/AArch64Combine.td
index 82b9cec960d89..6778344dc8f2f 100644
--- a/llvm/lib/Target/AArch64/AArch64Combine.td
+++ b/llvm/lib/Target/AArch64/AArch64Combine.td
@@ -333,6 +333,16 @@ def mutate_anyext_to_zext : GICombineRule<
(apply [{ applyMutateAnyExtToZExt(*${d}, MRI, B, Observer); }])
>;
+def addhn_from_cmp_masks_matchinfo :
+ GIDefMatchData<"std::pair<Register, Register>">;
+def addhn_from_cmp_masks : GICombineRule<
+ (defs root:$root, addhn_from_cmp_masks_matchinfo:$matchinfo),
+ (match (G_TRUNC $dst, $src):$root,
+ [{ return matchAddhnFromCmpMasks(*${root}, MRI, ${matchinfo}); }]),
+ (apply [{ applyAddhnFromCmpMasks(*${root}, MRI, B, Observer,
+ ${matchinfo}); }])
+>;
+
def split_store_zero_128 : GICombineRule<
(defs root:$d),
(match (G_STORE $src, $addr):$d,
@@ -387,9 +397,9 @@ def AArch64PostLegalizerLowering
// Post-legalization combines which are primarily optimizations.
def AArch64PostLegalizerCombiner
: GICombiner<"AArch64PostLegalizerCombinerImpl",
- [copy_prop, cast_of_cast_combines, constant_fold_fp_ops,
+ [copy_prop, cast_of_cast_combines, constant_fold_fp_ops,
buildvector_of_truncate, integer_of_truncate,
- mutate_anyext_to_zext, combines_for_extload,
+ mutate_anyext_to_zext, combines_for_extload,
combine_indexed_load_store, sext_trunc_sextload,
hoist_logic_op_with_same_opcode_hands,
redundant_and, xor_of_and_with_same_reg,
@@ -407,6 +417,7 @@ def AArch64PostLegalizerCombiner
commute_constant_to_rhs, extract_vec_elt_combines,
push_freeze_to_prevent_poison_from_propagating,
combine_mul_cmlt, combine_use_vector_truncate,
+ addhn_from_cmp_masks,
extmultomull, subaddmulreassoc, truncsat_combines,
lshr_of_trunc_of_lshr,
funnel_shift_from_or_shift_constants_are_legal]> {
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 2e6b0f4006de6..bed8e8cc46f60 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -23690,6 +23690,24 @@ static SDValue performTruncateCombine(SDNode *N, SelectionDAG &DAG,
SDLoc DL(N);
EVT VT = N->getValueType(0);
SDValue N0 = N->getOperand(0);
+
+ // Form the add/shift/truncate DAG matched by the ADDHN patterns.
+ if (DCI.isAfterLegalizeDAG() && N0.getOpcode() == ISD::OR && N0.hasOneUse()) {
+ EVT SrcVT = N0.getValueType();
+ bool IsAddHNType = (VT == MVT::v8i8 && SrcVT == MVT::v8i16) ||
+ (VT == MVT::v4i16 && SrcVT == MVT::v4i32) ||
+ (VT == MVT::v2i32 && SrcVT == MVT::v2i64);
+ if (IsAddHNType && N0.getOperand(0).getOpcode() == ISD::SETCC &&
+ N0.getOperand(1).getOpcode() == ISD::SETCC) {
+ SDValue Add =
+ DAG.getNode(ISD::ADD, DL, SrcVT, N0.getOperand(0), N0.getOperand(1));
+ SDValue Shift = DAG.getNode(
+ AArch64ISD::VLSHR, DL, SrcVT, Add,
+ DAG.getTargetConstant(VT.getScalarSizeInBits(), DL, MVT::i32));
+ return DAG.getNode(ISD::TRUNCATE, DL, VT, Shift);
+ }
+ }
+
if (VT.isFixedLengthVector() && VT.is64BitVector() && N0.hasOneUse() &&
N0.getOpcode() == AArch64ISD::DUP) {
SDValue Op = N0.getOperand(0);
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
index c8f461c746cbf..6a834d78b3de5 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
@@ -278,6 +278,71 @@ void applyFoldMergeToZext(MachineInstr &MI, MachineRegisterInfo &MRI,
Observer.changedInstr(MI);
}
+bool matchAddhnFromCmpMasks(MachineInstr &MI, MachineRegisterInfo &MRI,
+ std::pair<Register, Register> &MatchInfo) {
+ assert(MI.getOpcode() == TargetOpcode::G_TRUNC && "Expected a G_TRUNC");
+
+ Register Dst = MI.getOperand(0).getReg();
+ Register Src = MI.getOperand(1).getReg();
+ LLT DstTy = MRI.getType(Dst);
+ LLT SrcTy = MRI.getType(Src);
+ bool IsAddHNType =
+ (DstTy == LLT::fixed_vector(8, 8) && SrcTy == LLT::fixed_vector(8, 16)) ||
+ (DstTy == LLT::fixed_vector(4, 16) &&
+ SrcTy == LLT::fixed_vector(4, 32)) ||
+ (DstTy == LLT::fixed_vector(2, 32) && SrcTy == LLT::fixed_vector(2, 64));
+ if (!IsAddHNType)
+ return false;
+
+ // ADDHN is only profitable if the narrow result is not immediately extended
+ // again. Later combines can remove the G_TRUNC and leave the expanded
+ // G_ADD/G_VLSHR sequence behind.
+ for (MachineInstr &Use : MRI.use_nodbg_instructions(Dst)) {
+ unsigned Opc = Use.getOpcode();
+ if (Opc == TargetOpcode::G_ANYEXT || Opc == TargetOpcode::G_SEXT ||
+ Opc == TargetOpcode::G_ZEXT)
+ return false;
+ }
+
+ MachineInstr *Or = getOpcodeDef(TargetOpcode::G_OR, Src, MRI);
+ if (!Or || !MRI.hasOneNonDBGUse(Src))
+ return false;
+
+ auto IsComparisonMask = [&](Register Reg) {
+ MachineInstr *Def = getDefIgnoringCopies(Reg, MRI);
+ if (Def && Def->getOpcode() == TargetOpcode::G_FREEZE)
+ Def = getDefIgnoringCopies(Def->getOperand(1).getReg(), MRI);
+ return Def && (Def->getOpcode() == TargetOpcode::G_ICMP ||
+ Def->getOpcode() == TargetOpcode::G_FCMP);
+ };
+
+ Register LHS = Or->getOperand(1).getReg();
+ Register RHS = Or->getOperand(2).getReg();
+ if (!IsComparisonMask(LHS) || !IsComparisonMask(RHS))
+ return false;
+
+ MatchInfo = {LHS, RHS};
+ return true;
+}
+
+void applyAddhnFromCmpMasks(MachineInstr &MI, MachineRegisterInfo &MRI,
+ MachineIRBuilder &B, GISelChangeObserver &Observer,
+ const std::pair<Register, Register> &MatchInfo) {
+ LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
+ unsigned ShiftAmt = SrcTy.getScalarSizeInBits() / 2;
+
+ // AArch64 comparisons produce zero or all-ones vector elements. The high
+ // half of the sum of any two such masks is their logical OR, including when
+ // both comparisons are true.
+ B.setInstrAndDebugLoc(MI);
+ auto Add = B.buildAdd(SrcTy, MatchInfo.first, MatchInfo.second);
+ auto Shift = B.buildInstr(AArch64::G_VLSHR, {SrcTy}, {Add}).addImm(ShiftAmt);
+
+ Observer.changingInstr(MI);
+ MI.getOperand(1).setReg(Shift.getReg(0));
+ Observer.changedInstr(MI);
+}
+
/// \returns True if a G_ANYEXT instruction \p MI should be mutated to a G_ZEXT
/// instruction.
bool matchMutateAnyExtToZExt(MachineInstr &MI, MachineRegisterInfo &MRI) {
diff --git a/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll b/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
index de88e10141fbf..287980b871380 100644
--- a/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
+++ b/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
@@ -521,8 +521,7 @@ define <2 x i1> @match_v2xi64_v2i64(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %ma
; CHECK-NEXT: dup v1.2d, v1.d[0]
; CHECK-NEXT: cmeq v3.2d, v0.2d, v3.2d
; CHECK-NEXT: cmeq v0.2d, v0.2d, v1.2d
-; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b
-; CHECK-NEXT: xtn v0.2s, v0.2d
+; CHECK-NEXT: addhn v0.2s, v0.2d, v3.2d
; CHECK-NEXT: and v0.8b, v0.8b, v2.8b
; CHECK-NEXT: ret
%r = tail call <2 x i1> @llvm.experimental.vector.match(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %mask)
diff --git a/llvm/test/CodeGen/AArch64/is_fpclass.ll b/llvm/test/CodeGen/AArch64/is_fpclass.ll
index 71a8f42a7b4db..bda053dd56b9c 100644
--- a/llvm/test/CodeGen/AArch64/is_fpclass.ll
+++ b/llvm/test/CodeGen/AArch64/is_fpclass.ll
@@ -619,8 +619,7 @@ define <2 x i1> @not_isfinite_v2d(<2 x double> %x) {
; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI17_0]
; CHECK-GI-NEXT: cmeq v2.2d, v0.2d, v1.2d
; CHECK-GI-NEXT: cmhi v0.2d, v0.2d, v1.2d
-; CHECK-GI-NEXT: orr v0.16b, v2.16b, v0.16b
-; CHECK-GI-NEXT: xtn v0.2s, v0.2d
+; CHECK-GI-NEXT: addhn v0.2s, v2.2d, v0.2d
; CHECK-GI-NEXT: ret
;
; CHECK-NOFP-LABEL: not_isfinite_v2d:
diff --git a/llvm/test/CodeGen/AArch64/neon-addhn.ll b/llvm/test/CodeGen/AArch64/neon-addhn.ll
index 08f4d2e593716..c7f5245f47671 100644
--- a/llvm/test/CodeGen/AArch64/neon-addhn.ll
+++ b/llvm/test/CodeGen/AArch64/neon-addhn.ll
@@ -6,8 +6,7 @@ define i1 @addhn_setcc_v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> %comparand) {
; CHECK: // %bb.0:
; CHECK-NEXT: cmeq v0.8h, v0.8h, v2.8h
; CHECK-NEXT: cmeq v1.8h, v1.8h, v2.8h
-; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: xtn v0.8b, v0.8h
+; CHECK-NEXT: addhn v0.8b, v0.8h, v1.8h
; CHECK-NEXT: umov w8, v0.b[0]
; CHECK-NEXT: and w0, w8, #0x1
; CHECK-NEXT: ret
@@ -23,8 +22,7 @@ define i1 @addhn_setcc_v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %comparand) {
; CHECK: // %bb.0:
; CHECK-NEXT: cmeq v0.4s, v0.4s, v2.4s
; CHECK-NEXT: cmeq v1.4s, v1.4s, v2.4s
-; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: xtn v0.4h, v0.4s
+; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s
; CHECK-NEXT: umov w8, v0.h[0]
; CHECK-NEXT: and w0, w8, #0x1
; CHECK-NEXT: ret
@@ -40,8 +38,7 @@ define i1 @addhn_setcc_v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> %comparand) {
; CHECK: // %bb.0:
; CHECK-NEXT: cmeq v0.2d, v0.2d, v2.2d
; CHECK-NEXT: cmeq v1.2d, v1.2d, v2.2d
-; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: xtn v0.2s, v0.2d
+; CHECK-NEXT: addhn v0.2s, v0.2d, v1.2d
; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: and w0, w8, #0x1
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/select-with-and-or.ll b/llvm/test/CodeGen/AArch64/select-with-and-or.ll
index 60f2add81b45c..50500612d3b48 100644
--- a/llvm/test/CodeGen/AArch64/select-with-and-or.ll
+++ b/llvm/test/CodeGen/AArch64/select-with-and-or.ll
@@ -119,16 +119,14 @@ define <4 x i1> @or_vec(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z, <4 x i32> %w)
; CHECK-SD: // %bb.0:
; CHECK-SD-NEXT: cmgt v2.4s, v2.4s, v3.4s
; CHECK-SD-NEXT: cmeq v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT: orr v0.16b, v0.16b, v2.16b
-; CHECK-SD-NEXT: xtn v0.4h, v0.4s
+; CHECK-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: or_vec:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: cmeq v0.4s, v0.4s, v1.4s
; CHECK-GI-NEXT: cmgt v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT: orr v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT: xtn v0.4h, v0.4s
+; CHECK-GI-NEXT: addhn v0.4h, v0.4s, v1.4s
; CHECK-GI-NEXT: ret
%a = icmp eq <4 x i32> %x, %y
%b = icmp sgt <4 x i32> %z, %w
@@ -207,16 +205,14 @@ define <4 x i1> @or_vec_undef(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z, <4 x i32
; CHECK-SD: // %bb.0:
; CHECK-SD-NEXT: cmgt v2.4s, v2.4s, v3.4s
; CHECK-SD-NEXT: cmeq v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT: orr v0.16b, v0.16b, v2.16b
-; CHECK-SD-NEXT: xtn v0.4h, v0.4s
+; CHECK-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: or_vec_undef:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: cmeq v0.4s, v0.4s, v1.4s
; CHECK-GI-NEXT: cmgt v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT: orr v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT: xtn v0.4h, v0.4s
+; CHECK-GI-NEXT: addhn v0.4h, v0.4s, v1.4s
; CHECK-GI-NEXT: ret
%a = icmp eq <4 x i32> %x, %y
%b = icmp sgt <4 x i32> %z, %w
More information about the llvm-commits
mailing list