[llvm] [AArch64] Port saturation folds to AArch64 from ARM (PR #202056)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Jun 6 09:19:35 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-arm
Author: mike-goutokuji (mike-goutokuji)
<details>
<summary>Changes</summary>
---
Full diff: https://github.com/llvm/llvm-project/pull/202056.diff
8 Files Affected:
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.cpp (+85-18)
- (modified) llvm/lib/Target/ARM/ARMISelLowering.cpp (+9-10)
- (modified) llvm/test/CodeGen/AArch64/arm64-fmax-safe.ll (+1-2)
- (modified) llvm/test/CodeGen/AArch64/arm64-fmax.ll (+1-2)
- (modified) llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll (+8-8)
- (modified) llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll (+8-8)
- (modified) llvm/test/CodeGen/AArch64/selectcc-to-shiftand.ll (+35)
- (modified) llvm/test/CodeGen/AArch64/win64_vararg.ll (+2-4)
``````````diff
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 217b0fddd247d..eca03ad524eec 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -12324,6 +12324,54 @@ static SDValue emitFloatCompareMask(SDValue LHS, SDValue RHS, SDValue TVal,
return Cmp;
}
+static bool isGTorGE(ISD::CondCode CC) {
+ return CC == ISD::SETGT || CC == ISD::SETGE;
+}
+
+static bool isLTorLE(ISD::CondCode CC) {
+ return CC == ISD::SETLT || CC == ISD::SETLE;
+}
+
+// See if a conditional (LHS CC RHS ? TrueVal : FalseVal) is lower-saturating.
+// All of these conditions (and their <= and >= counterparts) will do:
+// x < k ? k : x
+// x > k ? x : k
+// k < x ? x : k
+// k > x ? k : x
+static bool isLowerSaturate(ISD::CondCode CC, SDValue LHS, SDValue RHS,
+ SDValue TrueVal, SDValue FalseVal, SDValue K) {
+ return (isGTorGE(CC) &&
+ ((K == LHS && K == TrueVal) || (K == RHS && K == FalseVal))) ||
+ (isLTorLE(CC) &&
+ ((K == RHS && K == TrueVal) || (K == LHS && K == FalseVal)));
+}
+
+// Check if a condition of the type x < k ? k : x can be converted into a
+// bit operation instead of conditional moves when k is 0 or -1.
+static bool isLowerSaturatingConditional(ISD::CondCode CC, SDValue LHS,
+ SDValue RHS, SDValue TrueVal,
+ SDValue FalseVal, SDValue &SatValue,
+ SDValue &LowerSatConstant) {
+ SDValue *K = isa<ConstantSDNode>(LHS) ? &LHS
+ : isa<ConstantSDNode>(RHS) ? &RHS
+ : nullptr;
+ if (!K)
+ return false;
+
+ SDValue KTmp = isa<ConstantSDNode>(TrueVal) ? TrueVal : FalseVal;
+ SatValue = (KTmp == TrueVal) ? FalseVal : TrueVal;
+ SDValue VTmp = (*K == LHS) ? RHS : LHS;
+
+ if (*K != KTmp || SatValue != VTmp)
+ return false;
+
+ if (!isLowerSaturate(CC, LHS, RHS, TrueVal, FalseVal, *K))
+ return false;
+
+ LowerSatConstant = *K;
+ return true;
+}
+
SDValue AArch64TargetLowering::LowerSELECT_CC(
ISD::CondCode CC, SDValue LHS, SDValue RHS, SDValue TVal, SDValue FVal,
iterator_range<SDNode::user_iterator> Users, SDNodeFlags Flags,
@@ -12357,24 +12405,6 @@ SDValue AArch64TargetLowering::LowerSELECT_CC(
ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(TVal);
ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS);
- // Check for SMAX(lhs, 0) and SMIN(lhs, 0) patterns.
- // (SELECT_CC setgt, lhs, 0, lhs, 0) -> (BIC lhs, (SRA lhs, typesize-1))
- // (SELECT_CC setlt, lhs, 0, lhs, 0) -> (AND lhs, (SRA lhs, typesize-1))
- // Both require less instructions than compare and conditional select.
- if ((CC == ISD::SETGT || CC == ISD::SETLT) && LHS == TVal &&
- RHSC && RHSC->isZero() && CFVal && CFVal->isZero() &&
- LHS.getValueType() == RHS.getValueType()) {
- EVT VT = LHS.getValueType();
- SDValue Shift =
- DAG.getNode(ISD::SRA, DL, VT, LHS,
- DAG.getConstant(VT.getSizeInBits() - 1, DL, VT));
-
- if (CC == ISD::SETGT)
- Shift = DAG.getNOT(DL, Shift, VT);
-
- return DAG.getNode(ISD::AND, DL, VT, LHS, Shift);
- }
-
// Check for sign bit test patterns that can use TST optimization.
// (SELECT_CC setlt, sign_extend_inreg, 0, tval, fval)
// -> TST %operand, sign_bit; CSEL
@@ -12418,6 +12448,43 @@ SDValue AArch64TargetLowering::LowerSELECT_CC(
}
}
+ // Try to convert expressions of the form x < k ? k : x (and similar forms)
+ // into more efficient bit operations, which is possible when k is 0 or -1.
+ // Only allow this transformation on full-width operations.
+ SDValue LowerSatConstant;
+ SDValue SatValue;
+ if (isLowerSaturatingConditional(CC, LHS, RHS, TVal, FVal, SatValue,
+ LowerSatConstant)) {
+ EVT SatVT = SatValue.getValueType();
+ SDValue ShiftV =
+ DAG.getNode(ISD::SRA, DL, SatVT, SatValue,
+ DAG.getConstant(SatVT.getSizeInBits() - 1, DL, SatVT));
+ if (isNullConstant(LowerSatConstant)) {
+ ShiftV = DAG.getNOT(DL, ShiftV, SatVT);
+ return DAG.getNode(ISD::AND, DL, SatVT, SatValue, ShiftV);
+ }
+ if (isAllOnesConstant(LowerSatConstant))
+ return DAG.getNode(ISD::OR, DL, SatVT, SatValue, ShiftV);
+ }
+
+ // Check for SMAX(lhs, 0) and SMIN(lhs, 0) patterns.
+ // (SELECT_CC setgt, lhs, 0, lhs, 0) -> (BIC lhs, (SRA lhs, typesize-1))
+ // (SELECT_CC setlt, lhs, 0, lhs, 0) -> (AND lhs, (SRA lhs, typesize-1))
+ // Both require less instructions than compare and conditional select.
+ if ((CC == ISD::SETGT || CC == ISD::SETLT) && LHS == TVal &&
+ RHSC && RHSC->isZero() && CFVal && CFVal->isZero() &&
+ LHS.getValueType() == RHS.getValueType()) {
+ EVT VT = LHS.getValueType();
+ SDValue Shift =
+ DAG.getNode(ISD::SRA, DL, VT, LHS,
+ DAG.getConstant(VT.getSizeInBits() - 1, DL, VT));
+
+ if (CC == ISD::SETGT)
+ Shift = DAG.getNOT(DL, Shift, VT);
+
+ return DAG.getNode(ISD::AND, DL, VT, LHS, Shift);
+ }
+
unsigned Opcode = AArch64ISD::CSEL;
// If both the TVal and the FVal are constants, see if we can swap them in
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index e08a4f5ebda08..0964c4d800ba2 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -5121,8 +5121,7 @@ static SDValue LowerSaturatingConditional(SDValue Op, SelectionDAG &DAG) {
// It returns true if the transformation can be made, and in such case
// returns x in V, and k in SatK.
static bool isLowerSaturatingConditional(const SDValue &Op, SDValue &V,
- SDValue &SatK)
-{
+ SDValue &SatK) {
SDValue LHS = Op.getOperand(0);
SDValue RHS = Op.getOperand(1);
ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
@@ -5222,11 +5221,11 @@ SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
return SatValue;
// Try to convert expressions of the form x < k ? k : x (and similar forms)
- // into more efficient bit operations, which is possible when k is 0 or -1
+ // into more efficient bit operations, which is possible when k is 0 or -1.
// On ARM and Thumb-2 which have flexible operand 2 this will result in
// single instructions. On Thumb the shift and the bit operation will be two
// instructions.
- // Only allow this transformation on full-width (32-bit) operations
+ // Only allow this transformation on full-width (32-bit) operations.
SDValue LowerSatConstant;
SDValue SatValue;
if (VT == MVT::i32 &&
@@ -5250,6 +5249,12 @@ SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS);
if (Op.getValueType().isInteger()) {
+ // (SELECT_CC setlt, x, 0, 1, 0) -> SRL(x, bw-1)
+ if (CC == ISD::SETLT && isNullConstant(RHS) && isOneConstant(TrueVal) &&
+ isNullConstant(FalseVal) && LHS.getValueType() == VT)
+ return DAG.getNode(ISD::SRL, dl, VT, LHS,
+ DAG.getConstant(VT.getSizeInBits() - 1, dl, VT));
+
// Check for SMAX(lhs, 0) and SMIN(lhs, 0) patterns.
// (SELECT_CC setgt, lhs, 0, lhs, 0) -> (BIC lhs, (SRA lhs, typesize-1))
// (SELECT_CC setlt, lhs, 0, lhs, 0) -> (AND lhs, (SRA lhs, typesize-1))
@@ -5267,12 +5272,6 @@ SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
return DAG.getNode(ISD::AND, dl, VT, LHS, Shift);
}
-
- // (SELECT_CC setlt, x, 0, 1, 0) -> SRL(x, bw-1)
- if (CC == ISD::SETLT && isNullConstant(RHS) && isOneConstant(TrueVal) &&
- isNullConstant(FalseVal) && LHS.getValueType() == VT)
- return DAG.getNode(ISD::SRL, dl, VT, LHS,
- DAG.getConstant(VT.getSizeInBits() - 1, dl, VT));
}
if (LHS.getValueType() == MVT::i32) {
diff --git a/llvm/test/CodeGen/AArch64/arm64-fmax-safe.ll b/llvm/test/CodeGen/AArch64/arm64-fmax-safe.ll
index 4e3fac3650e98..f96ea0d2ed879 100644
--- a/llvm/test/CodeGen/AArch64/arm64-fmax-safe.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-fmax-safe.ll
@@ -64,8 +64,7 @@ define float @test_cross_fail(float %lhs, float %rhs) {
define i64 @test_integer(i64 %in) {
; CHECK-LABEL: test_integer:
; CHECK: // %bb.0:
-; CHECK-NEXT: cmp x0, #0
-; CHECK-NEXT: csel x0, xzr, x0, mi
+; CHECK-NEXT: bic x0, x0, x0, asr #63
; CHECK-NEXT: ret
%cmp = icmp slt i64 %in, 0
%val = select i1 %cmp, i64 0, i64 %in
diff --git a/llvm/test/CodeGen/AArch64/arm64-fmax.ll b/llvm/test/CodeGen/AArch64/arm64-fmax.ll
index f311139e193a5..ab1b978c363f7 100644
--- a/llvm/test/CodeGen/AArch64/arm64-fmax.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-fmax.ll
@@ -59,8 +59,7 @@ define float @test_cross_fail(float %lhs, float %rhs) {
define i64 @test_integer(i64 %in) {
; CHECK-LABEL: test_integer:
; CHECK: // %bb.0:
-; CHECK-NEXT: cmp x0, #0
-; CHECK-NEXT: csel x0, xzr, x0, mi
+; CHECK-NEXT: bic x0, x0, x0, asr #63
; CHECK-NEXT: ret
%cmp = icmp slt i64 %in, 0
%val = select i1 %cmp, i64 0, i64 %in
diff --git a/llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll b/llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll
index f95d338198c4d..67f598727a032 100644
--- a/llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll
+++ b/llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll
@@ -23,8 +23,8 @@ define i1 @test_signed_i1_f32(float %f) nounwind {
; CHECK-SD-LABEL: test_signed_i1_f32:
; CHECK-SD: // %bb.0:
; CHECK-SD-NEXT: fcvtzs w8, s0
-; CHECK-SD-NEXT: ands w8, w8, w8, asr #31
-; CHECK-SD-NEXT: csinv w8, w8, wzr, pl
+; CHECK-SD-NEXT: and w8, w8, w8, asr #31
+; CHECK-SD-NEXT: orr w8, w8, w8, lsr #31
; CHECK-SD-NEXT: and w0, w8, #0x1
; CHECK-SD-NEXT: ret
;
@@ -268,8 +268,8 @@ define i1 @test_signed_i1_f64(double %f) nounwind {
; CHECK-SD-LABEL: test_signed_i1_f64:
; CHECK-SD: // %bb.0:
; CHECK-SD-NEXT: fcvtzs w8, d0
-; CHECK-SD-NEXT: ands w8, w8, w8, asr #31
-; CHECK-SD-NEXT: csinv w8, w8, wzr, pl
+; CHECK-SD-NEXT: and w8, w8, w8, asr #31
+; CHECK-SD-NEXT: orr w8, w8, w8, lsr #31
; CHECK-SD-NEXT: and w0, w8, #0x1
; CHECK-SD-NEXT: ret
;
@@ -518,16 +518,16 @@ define i1 @test_signed_i1_f16(half %f) nounwind {
; CHECK-SD-CVT: // %bb.0:
; CHECK-SD-CVT-NEXT: fcvt s0, h0
; CHECK-SD-CVT-NEXT: fcvtzs w8, s0
-; CHECK-SD-CVT-NEXT: ands w8, w8, w8, asr #31
-; CHECK-SD-CVT-NEXT: csinv w8, w8, wzr, pl
+; CHECK-SD-CVT-NEXT: and w8, w8, w8, asr #31
+; CHECK-SD-CVT-NEXT: orr w8, w8, w8, lsr #31
; CHECK-SD-CVT-NEXT: and w0, w8, #0x1
; CHECK-SD-CVT-NEXT: ret
;
; CHECK-SD-FP16-LABEL: test_signed_i1_f16:
; CHECK-SD-FP16: // %bb.0:
; CHECK-SD-FP16-NEXT: fcvtzs w8, h0
-; CHECK-SD-FP16-NEXT: ands w8, w8, w8, asr #31
-; CHECK-SD-FP16-NEXT: csinv w8, w8, wzr, pl
+; CHECK-SD-FP16-NEXT: and w8, w8, w8, asr #31
+; CHECK-SD-FP16-NEXT: orr w8, w8, w8, lsr #31
; CHECK-SD-FP16-NEXT: and w0, w8, #0x1
; CHECK-SD-FP16-NEXT: ret
;
diff --git a/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll b/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
index 145aef9123a4e..892af7852e1bc 100644
--- a/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
+++ b/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
@@ -2350,14 +2350,14 @@ define <2 x i1> @test_signed_v2f64_v2i1(<2 x double> %f) {
; CHECK-SD-LABEL: test_signed_v2f64_v2i1:
; CHECK-SD: // %bb.0:
; CHECK-SD-NEXT: mov d1, v0.d[1]
-; CHECK-SD-NEXT: fcvtzs w9, d0
-; CHECK-SD-NEXT: fcvtzs w8, d1
-; CHECK-SD-NEXT: ands w8, w8, w8, asr #31
-; CHECK-SD-NEXT: csinv w8, w8, wzr, pl
-; CHECK-SD-NEXT: ands w9, w9, w9, asr #31
-; CHECK-SD-NEXT: csinv w9, w9, wzr, pl
-; CHECK-SD-NEXT: fmov s0, w9
-; CHECK-SD-NEXT: mov v0.s[1], w8
+; CHECK-SD-NEXT: fcvtzs w8, d0
+; CHECK-SD-NEXT: fcvtzs w9, d1
+; CHECK-SD-NEXT: and w8, w8, w8, asr #31
+; CHECK-SD-NEXT: orr w8, w8, w8, asr #31
+; CHECK-SD-NEXT: and w9, w9, w9, asr #31
+; CHECK-SD-NEXT: fmov s0, w8
+; CHECK-SD-NEXT: orr w9, w9, w9, asr #31
+; CHECK-SD-NEXT: mov v0.s[1], w9
; CHECK-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT: ret
;
diff --git a/llvm/test/CodeGen/AArch64/selectcc-to-shiftand.ll b/llvm/test/CodeGen/AArch64/selectcc-to-shiftand.ll
index 96a7a9d039c21..4c760e7f9d32b 100644
--- a/llvm/test/CodeGen/AArch64/selectcc-to-shiftand.ll
+++ b/llvm/test/CodeGen/AArch64/selectcc-to-shiftand.ll
@@ -329,5 +329,40 @@ define <2 x i64> @sel_shift_bool_v2i64(<2 x i1> %t) {
%shl = select <2 x i1> %t, <2 x i64> <i64 65536, i64 65536>, <2 x i64> zeroinitializer
ret <2 x i64> %shl
}
+
+; x < 0 ? 0 : x
+define i32 @sat0_inverted(i32 %x) {
+; CHECK-SD-LABEL: sat0_inverted:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: bic w0, w0, w0, asr #31
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: sat0_inverted:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: cmp w0, #0
+; CHECK-GI-NEXT: csel w0, w0, wzr, gt
+; CHECK-GI-NEXT: ret
+ %cmp = icmp slt i32 %x, 0
+ %sel = select i1 %cmp, i32 0, i32 %x
+ ret i32 %sel
+}
+
+; x < -1 ? -1 : x
+define i32 @sat1_inverted(i32 %x) {
+; CHECK-SD-LABEL: sat1_inverted:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: orr w0, w0, w0, asr #31
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: sat1_inverted:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: cmn w0, #1
+; CHECK-GI-NEXT: csinv w0, w0, wzr, gt
+; CHECK-GI-NEXT: ret
+ %cmp = icmp slt i32 %x, -1
+ %sel = select i1 %cmp, i32 -1, i32 %x
+ ret i32 %sel
+}
+
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/win64_vararg.ll b/llvm/test/CodeGen/AArch64/win64_vararg.ll
index 0f4cb44a6f73b..6b845c6163882 100644
--- a/llvm/test/CodeGen/AArch64/win64_vararg.ll
+++ b/llvm/test/CodeGen/AArch64/win64_vararg.ll
@@ -133,8 +133,7 @@ define i32 @fp(ptr, i64, ptr, ...) local_unnamed_addr #6 {
; CHECK-NEXT: mov x4, xzr
; CHECK-NEXT: orr x0, x8, #0x2
; CHECK-NEXT: bl __stdio_common_vsprintf
-; CHECK-NEXT: cmn w0, #1
-; CHECK-NEXT: csinv w0, w0, wzr, gt
+; CHECK-NEXT: orr w0, w0, w0, asr #31
; CHECK-NEXT: .seh_startepilogue
; CHECK-NEXT: ldp x29, x30, [sp, #24] // 16-byte Folded Reload
; CHECK-NEXT: .seh_save_fplr 24
@@ -268,8 +267,7 @@ define i32 @snprintf(ptr, i64, ptr, ...) local_unnamed_addr #5 {
; CHECK-NEXT: mov x4, xzr
; CHECK-NEXT: orr x0, x8, #0x2
; CHECK-NEXT: bl __stdio_common_vsprintf
-; CHECK-NEXT: cmn w0, #1
-; CHECK-NEXT: csinv w0, w0, wzr, gt
+; CHECK-NEXT: orr w0, w0, w0, asr #31
; CHECK-NEXT: .seh_startepilogue
; CHECK-NEXT: ldp x21, x30, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT: .seh_save_lrpair x21, 32
``````````
</details>
https://github.com/llvm/llvm-project/pull/202056
More information about the llvm-commits
mailing list