[llvm] [AArch64] Spare N2I roundtrip when splatting float comparison (PR #141806)
Guy David via llvm-commits
llvm-commits at lists.llvm.org
Sat May 31 13:15:53 PDT 2025
https://github.com/guy-david updated https://github.com/llvm/llvm-project/pull/141806
>From 1bc6d26826843cc597d39a1b1b81938993fd828d Mon Sep 17 00:00:00 2001
From: Guy David <guyda96 at gmail.com>
Date: Wed, 28 May 2025 19:55:44 +0300
Subject: [PATCH] [AArch64] Spare N2I roundtrip when splatting float comparison
Transform `select_cc t1, t2, -1, 0` for floats into a vector comparison which
generates a mask, which is later on combined with potential vectorized DUPs.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 132 ++++++++++++------
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 4 +-
.../CodeGen/AArch64/arm64-neon-v1i1-setcc.ll | 5 +-
.../CodeGen/AArch64/build-vector-dup-simd.ll | 75 ++++++++++
4 files changed, 167 insertions(+), 49 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/build-vector-dup-simd.ll
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index ae34e6b7dcc3c..314c8d19712c3 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -11001,9 +11001,48 @@ SDValue AArch64TargetLowering::LowerSETCCCARRY(SDValue Op,
Cmp.getValue(1));
}
+/// Emit vector comparison for floating-point values, producing a mask.
+static SDValue EmitVectorComparison(SDValue LHS, SDValue RHS,
+ AArch64CC::CondCode CC, bool NoNans, EVT VT,
+ const SDLoc &dl, SelectionDAG &DAG) {
+ EVT SrcVT = LHS.getValueType();
+ assert(VT.getSizeInBits() == SrcVT.getSizeInBits() &&
+ "function only supposed to emit natural comparisons");
+
+ switch (CC) {
+ default:
+ return SDValue();
+ case AArch64CC::NE: {
+ SDValue Fcmeq = DAG.getNode(AArch64ISD::FCMEQ, dl, VT, LHS, RHS);
+ return DAG.getNOT(dl, Fcmeq, VT);
+ }
+ case AArch64CC::EQ:
+ return DAG.getNode(AArch64ISD::FCMEQ, dl, VT, LHS, RHS);
+ case AArch64CC::GE:
+ return DAG.getNode(AArch64ISD::FCMGE, dl, VT, LHS, RHS);
+ case AArch64CC::GT:
+ return DAG.getNode(AArch64ISD::FCMGT, dl, VT, LHS, RHS);
+ case AArch64CC::LE:
+ if (!NoNans)
+ return SDValue();
+ // If we ignore NaNs then we can use to the LS implementation.
+ [[fallthrough]];
+ case AArch64CC::LS:
+ return DAG.getNode(AArch64ISD::FCMGE, dl, VT, RHS, LHS);
+ case AArch64CC::LT:
+ if (!NoNans)
+ return SDValue();
+ // If we ignore NaNs then we can use to the MI implementation.
+ [[fallthrough]];
+ case AArch64CC::MI:
+ return DAG.getNode(AArch64ISD::FCMGT, dl, VT, RHS, LHS);
+ }
+}
+
SDValue AArch64TargetLowering::LowerSELECT_CC(ISD::CondCode CC, SDValue LHS,
SDValue RHS, SDValue TVal,
- SDValue FVal, const SDLoc &dl,
+ SDValue FVal, bool HasNoNaNs,
+ const SDLoc &dl,
SelectionDAG &DAG) const {
// Handle f128 first, because it will result in a comparison of some RTLIB
// call result against zero.
@@ -11187,6 +11226,28 @@ SDValue AArch64TargetLowering::LowerSELECT_CC(ISD::CondCode CC, SDValue LHS,
LHS.getValueType() == MVT::f64);
assert(LHS.getValueType() == RHS.getValueType());
EVT VT = TVal.getValueType();
+
+ // If the purpose of the comparison is to select between all ones
+ // or all zeros, use a vector comparison because the operands are already
+ // stored in SIMD registers.
+ auto *CTVal = dyn_cast<ConstantSDNode>(TVal);
+ auto *CFVal = dyn_cast<ConstantSDNode>(FVal);
+ if (Subtarget->isNeonAvailable() && CTVal && CFVal &&
+ VT.getSizeInBits() == LHS.getValueType().getSizeInBits() &&
+ ((CTVal->isAllOnes() && CFVal->isZero()) ||
+ (CTVal->isZero() && CFVal->isAllOnes()))) {
+ AArch64CC::CondCode CC1;
+ AArch64CC::CondCode CC2;
+ bool ShouldInvert = false;
+ changeVectorFPCCToAArch64CC(CC, CC1, CC2, ShouldInvert);
+ if (CTVal->isZero() ^ ShouldInvert)
+ std::swap(TVal, FVal);
+ bool NoNaNs = getTargetMachine().Options.NoNaNsFPMath || HasNoNaNs;
+ SDValue Res = EmitVectorComparison(LHS, RHS, CC1, NoNaNs, VT, dl, DAG);
+ if (Res)
+ return Res;
+ }
+
SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
// Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
@@ -11273,8 +11334,9 @@ SDValue AArch64TargetLowering::LowerSELECT_CC(SDValue Op,
SDValue RHS = Op.getOperand(1);
SDValue TVal = Op.getOperand(2);
SDValue FVal = Op.getOperand(3);
+ bool HasNoNans = Op->getFlags().hasNoNaNs();
SDLoc DL(Op);
- return LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, DL, DAG);
+ return LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, HasNoNans, DL, DAG);
}
SDValue AArch64TargetLowering::LowerSELECT(SDValue Op,
@@ -11282,6 +11344,7 @@ SDValue AArch64TargetLowering::LowerSELECT(SDValue Op,
SDValue CCVal = Op->getOperand(0);
SDValue TVal = Op->getOperand(1);
SDValue FVal = Op->getOperand(2);
+ bool HasNoNans = Op->getFlags().hasNoNaNs();
SDLoc DL(Op);
EVT Ty = Op.getValueType();
@@ -11348,7 +11411,7 @@ SDValue AArch64TargetLowering::LowerSELECT(SDValue Op,
DAG.getUNDEF(MVT::f32), FVal);
}
- SDValue Res = LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, DL, DAG);
+ SDValue Res = LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, HasNoNans, DL, DAG);
if ((Ty == MVT::f16 || Ty == MVT::bf16) && !Subtarget->hasFullFP16()) {
return DAG.getTargetExtractSubreg(AArch64::hsub, DL, Ty, Res);
@@ -15601,47 +15664,6 @@ SDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(SDValue Op,
llvm_unreachable("unexpected shift opcode");
}
-static SDValue EmitVectorComparison(SDValue LHS, SDValue RHS,
- AArch64CC::CondCode CC, bool NoNans, EVT VT,
- const SDLoc &dl, SelectionDAG &DAG) {
- EVT SrcVT = LHS.getValueType();
- assert(VT.getSizeInBits() == SrcVT.getSizeInBits() &&
- "function only supposed to emit natural comparisons");
-
- if (SrcVT.getVectorElementType().isFloatingPoint()) {
- switch (CC) {
- default:
- return SDValue();
- case AArch64CC::NE: {
- SDValue Fcmeq = DAG.getNode(AArch64ISD::FCMEQ, dl, VT, LHS, RHS);
- return DAG.getNOT(dl, Fcmeq, VT);
- }
- case AArch64CC::EQ:
- return DAG.getNode(AArch64ISD::FCMEQ, dl, VT, LHS, RHS);
- case AArch64CC::GE:
- return DAG.getNode(AArch64ISD::FCMGE, dl, VT, LHS, RHS);
- case AArch64CC::GT:
- return DAG.getNode(AArch64ISD::FCMGT, dl, VT, LHS, RHS);
- case AArch64CC::LE:
- if (!NoNans)
- return SDValue();
- // If we ignore NaNs then we can use to the LS implementation.
- [[fallthrough]];
- case AArch64CC::LS:
- return DAG.getNode(AArch64ISD::FCMGE, dl, VT, RHS, LHS);
- case AArch64CC::LT:
- if (!NoNans)
- return SDValue();
- // If we ignore NaNs then we can use to the MI implementation.
- [[fallthrough]];
- case AArch64CC::MI:
- return DAG.getNode(AArch64ISD::FCMGT, dl, VT, RHS, LHS);
- }
- }
-
- return SDValue();
-}
-
SDValue AArch64TargetLowering::LowerVSETCC(SDValue Op,
SelectionDAG &DAG) const {
if (Op.getValueType().isScalableVector())
@@ -25455,6 +25477,28 @@ static SDValue performDUPCombine(SDNode *N,
}
if (N->getOpcode() == AArch64ISD::DUP) {
+ // If the instruction is known to produce a scalar in SIMD registers, we can
+ // can duplicate it across the vector lanes using DUPLANE instead of moving
+ // it to a GPR first. For example, this allows us to handle:
+ // v4i32 = DUP (i32 (FCMGT (f32, f32)))
+ SDValue Op = N->getOperand(0);
+ // FIXME: Ideally, we should be able to handle all instructions that
+ // produce a scalar value in FPRs.
+ if (Op.getOpcode() == AArch64ISD::FCMEQ ||
+ Op.getOpcode() == AArch64ISD::FCMGE ||
+ Op.getOpcode() == AArch64ISD::FCMGT) {
+ EVT ElemVT = VT.getVectorElementType();
+ EVT ExpandedVT = VT;
+ // Insert into a 128-bit vector to match DUPLANE's pattern.
+ if (VT.getSizeInBits() != 128)
+ ExpandedVT = EVT::getVectorVT(*DCI.DAG.getContext(), ElemVT,
+ 128 / ElemVT.getSizeInBits());
+ SDValue Zero = DCI.DAG.getConstant(0, DL, MVT::i64);
+ SDValue Vec = DCI.DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ExpandedVT,
+ DCI.DAG.getUNDEF(ExpandedVT), Op, Zero);
+ return DCI.DAG.getNode(getDUPLANEOp(ElemVT), DL, VT, Vec, Zero);
+ }
+
if (DCI.isAfterLegalizeDAG()) {
// If scalar dup's operand is extract_vector_elt, try to combine them into
// duplane. For example,
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 450e2efd7d430..e29c7ac7df723 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -646,8 +646,8 @@ class AArch64TargetLowering : public TargetLowering {
SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerSELECT_CC(ISD::CondCode CC, SDValue LHS, SDValue RHS,
- SDValue TVal, SDValue FVal, const SDLoc &dl,
- SelectionDAG &DAG) const;
+ SDValue TVal, SDValue FVal, bool HasNoNans,
+ const SDLoc &dl, SelectionDAG &DAG) const;
SDValue LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/test/CodeGen/AArch64/arm64-neon-v1i1-setcc.ll b/llvm/test/CodeGen/AArch64/arm64-neon-v1i1-setcc.ll
index 6c70d19a977a5..05178c1dc291c 100644
--- a/llvm/test/CodeGen/AArch64/arm64-neon-v1i1-setcc.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-neon-v1i1-setcc.ll
@@ -174,9 +174,8 @@ define <1 x i16> @test_select_f16_i16(half %i105, half %in, <1 x i16> %x, <1 x i
; CHECK-LABEL: test_select_f16_i16:
; CHECK: // %bb.0:
; CHECK-NEXT: fcvt s0, h0
-; CHECK-NEXT: fcmp s0, s0
-; CHECK-NEXT: csetm w8, vs
-; CHECK-NEXT: dup v0.4h, w8
+; CHECK-NEXT: fcmgt s0, s0, s0
+; CHECK-NEXT: dup v0.4h, v0.h[0]
; CHECK-NEXT: bsl v0.8b, v2.8b, v3.8b
; CHECK-NEXT: ret
%i179 = fcmp uno half %i105, zeroinitializer
diff --git a/llvm/test/CodeGen/AArch64/build-vector-dup-simd.ll b/llvm/test/CodeGen/AArch64/build-vector-dup-simd.ll
new file mode 100644
index 0000000000000..c661460444bfa
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/build-vector-dup-simd.ll
@@ -0,0 +1,75 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64 | FileCheck %s
+
+define <8 x half> @dup_v8i16(half %a, half %b) {
+ ; CHECK-LABEL: dup_v8i16:
+ ; CHECK: // %bb.0: // %entry
+ ; CHECK-NEXT: fcvt s1, h1
+ ; CHECK-NEXT: fcvt s0, h0
+ ; CHECK-NEXT: fcmeq s0, s0, s1
+ ; CHECK-NEXT: ret
+ entry:
+ %0 = fcmp oeq half %a, %b
+ %vcmpd.i = sext i1 %0 to i16
+ %vecinit.i = insertelement <8 x i16> undef, i16 %vcmpd.i, i64 0
+ %1 = bitcast <8 x i16> %vecinit.i to <8 x half>
+ ret <8 x half> %1
+}
+
+define <1 x float> @dup_v1i32(float %a, float %b) {
+; CHECK-LABEL: dup_v1i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: fcmeq s0, s0, s1
+; CHECK-NEXT: ret
+entry:
+ %0 = fcmp oeq float %a, %b
+ %vcmpd.i = sext i1 %0 to i32
+ %vecinit.i = insertelement <1 x i32> undef, i32 %vcmpd.i, i64 0
+ %1 = bitcast <1 x i32> %vecinit.i to <1 x float>
+ ret <1 x float> %1
+}
+
+define <4 x float> @dup_v4i32(float %a, float %b) {
+; CHECK-LABEL: dup_v4i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: fcmge s0, s0, s1
+; CHECK-NEXT: dup v0.4s, v0.s[0]
+; CHECK-NEXT: ret
+entry:
+ %0 = fcmp oge float %a, %b
+ %vcmpd.i = sext i1 %0 to i32
+ %vecinit.i = insertelement <4 x i32> undef, i32 %vcmpd.i, i64 0
+ %1 = bitcast <4 x i32> %vecinit.i to <4 x float>
+ %2 = shufflevector <4 x float> %1, <4 x float> undef, <4 x i32> zeroinitializer
+ ret <4 x float> %2
+}
+
+define <4 x float> @dup_v4i32_reversed(float %a, float %b) {
+; CHECK-LABEL: dup_v4i32_reversed:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: fcmgt s0, s1, s0
+; CHECK-NEXT: dup v0.4s, v0.s[0]
+; CHECK-NEXT: ret
+entry:
+ %0 = fcmp ogt float %b, %a
+ %vcmpd.i = sext i1 %0 to i32
+ %vecinit.i = insertelement <4 x i32> undef, i32 %vcmpd.i, i64 0
+ %1 = bitcast <4 x i32> %vecinit.i to <4 x float>
+ %2 = shufflevector <4 x float> %1, <4 x float> undef, <4 x i32> zeroinitializer
+ ret <4 x float> %2
+}
+
+define <2 x double> @dup_v2i64(double %a, double %b) {
+; CHECK-LABEL: dup_v2i64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: fcmgt d0, d0, d1
+; CHECK-NEXT: dup v0.2d, v0.d[0]
+; CHECK-NEXT: ret
+entry:
+ %0 = fcmp ogt double %a, %b
+ %vcmpd.i = sext i1 %0 to i64
+ %vecinit.i = insertelement <2 x i64> undef, i64 %vcmpd.i, i64 0
+ %1 = bitcast <2 x i64> %vecinit.i to <2 x double>
+ %2 = shufflevector <2 x double> %1, <2 x double> undef, <2 x i32> zeroinitializer
+ ret <2 x double> %2
+}
More information about the llvm-commits
mailing list