[llvm] [ARM] Custom Lowering for SADDO_CARRY an SSUBO_CARRY (PR #154419)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 19 13:12:34 PDT 2025
https://github.com/AZero13 created https://github.com/llvm/llvm-project/pull/154419
To do this, I did refactoring to mirror what goes on with AArch64, including having the carryFlagToValue do the inversion.
While the patterns are not the best, with pattern matching, I hope to at make it as good as AArch64 on Thumb2 where we have CSEL.
>From 7610c97e240d4da28e87acead2cf0abeb531370d Mon Sep 17 00:00:00 2001
From: AZero13 <gfunni234 at gmail.com>
Date: Tue, 19 Aug 2025 16:04:49 -0400
Subject: [PATCH] [ARM] Custom Lowering for SADDO_CARRY an SSUBO_CARRY
To do this, I did refactoring to mirror what goes on with AArch64, including having the carryFlagToValue do the inversion.
While the patterns are not the best, with pattern matching, I hope to at make it as good as AArch64 on Thumb2 where we have CSEL.
---
llvm/lib/Target/ARM/ARMISelLowering.cpp | 121 ++++++++++++------------
1 file changed, 58 insertions(+), 63 deletions(-)
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 830156359e9e8..875531bc54cfe 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -988,6 +988,8 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
setOperationAction(ISD::UADDO_CARRY, MVT::i32, Custom);
setOperationAction(ISD::USUBO_CARRY, MVT::i32, Custom);
+ setOperationAction(ISD::SADDO_CARRY, MVT::i32, Custom);
+ setOperationAction(ISD::SSUBO_CARRY, MVT::i32, Custom);
if (Subtarget->hasDSP()) {
setOperationAction(ISD::SADDSAT, MVT::i8, Custom);
setOperationAction(ISD::SSUBSAT, MVT::i8, Custom);
@@ -4923,23 +4925,30 @@ ARMTargetLowering::LowerSignedALUO(SDValue Op, SelectionDAG &DAG) const {
return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow);
}
-static SDValue ConvertBooleanCarryToCarryFlag(SDValue BoolCarry,
- SelectionDAG &DAG) {
- SDLoc DL(BoolCarry);
- EVT CarryVT = BoolCarry.getValueType();
-
- // This converts the boolean value carry into the carry flag by doing
- // ARMISD::SUBC Carry, 1
- SDValue Carry = DAG.getNode(ARMISD::SUBC, DL,
- DAG.getVTList(CarryVT, MVT::i32),
- BoolCarry, DAG.getConstant(1, DL, CarryVT));
- return Carry.getValue(1);
+static SDValue valueToCarryFlag(SDValue Value, SelectionDAG &DAG, bool Invert) {
+ SDLoc DL(Value);
+ EVT VT = Value.getValueType();
+ SDValue Op0 = Invert ? DAG.getConstant(0, DL, VT) : Value;
+ SDValue Op1 = Invert ? Value : DAG.getConstant(1, DL, VT);
+ SDValue Cmp =
+ DAG.getNode(ARMISD::SUBC, DL, DAG.getVTList(VT, MVT::i32), Op0, Op1);
+ return Cmp.getValue(1);
}
-static SDValue ConvertCarryFlagToBooleanCarry(SDValue Flags, EVT VT,
- SelectionDAG &DAG) {
+static SDValue carryFlagToValue(SDValue Flags, EVT VT,
+ SelectionDAG &DAG, bool Invert) {
SDLoc DL(Flags);
+ if (Invert) {
+ // Now convert the carry flag into a boolean carry. We do this
+ // using ARMISD::SUBE 1, 0, Carry
+
+ // TODO: Use CSEL on Thumbv8 once the patterns are matched.
+ return DAG.getNode(ARMISD::SUBE, DL, DAG.getVTList(VT, MVT::i32),
+ DAG.getConstant(1, DL, MVT::i32),
+ DAG.getConstant(0, DL, MVT::i32), Flags);
+ }
+
// Now convert the carry flag into a boolean carry. We do this
// using ARMISD:ADDE 0, 0, Carry
return DAG.getNode(ARMISD::ADDE, DL, DAG.getVTList(VT, MVT::i32),
@@ -4947,6 +4956,15 @@ static SDValue ConvertCarryFlagToBooleanCarry(SDValue Flags, EVT VT,
DAG.getConstant(0, DL, MVT::i32), Flags);
}
+// Value is 1 if 'V' bit of NZCV is 1, else 0
+static SDValue overflowFlagToValue(SDValue Flags, EVT VT, SelectionDAG &DAG) {
+ SDLoc DL(Flags);
+ SDValue Zero = DAG.getConstant(0, DL, VT);
+ SDValue One = DAG.getConstant(1, DL, VT);
+ SDValue ARMcc = DAG.getConstant(ARMCC::VS, DL, MVT::i32);
+ return DAG.getNode(ARMISD::CMOV, DL, VT, Zero, One, ARMcc, Flags);
+}
+
SDValue ARMTargetLowering::LowerUnsignedALUO(SDValue Op,
SelectionDAG &DAG) const {
// Let legalize expand this if it isn't a legal type yet.
@@ -4967,16 +4985,12 @@ SDValue ARMTargetLowering::LowerUnsignedALUO(SDValue Op,
case ISD::UADDO:
Value = DAG.getNode(ARMISD::ADDC, dl, VTs, LHS, RHS);
// Convert the carry flag into a boolean value.
- Overflow = ConvertCarryFlagToBooleanCarry(Value.getValue(1), VT, DAG);
+ Overflow = carryFlagToValue(Value.getValue(1), VT, DAG, false);
break;
case ISD::USUBO: {
Value = DAG.getNode(ARMISD::SUBC, dl, VTs, LHS, RHS);
// Convert the carry flag into a boolean value.
- Overflow = ConvertCarryFlagToBooleanCarry(Value.getValue(1), VT, DAG);
- // ARMISD::SUBC returns 0 when we have to borrow, so make it an overflow
- // value. So compute 1 - C.
- Overflow = DAG.getNode(ISD::SUB, dl, MVT::i32,
- DAG.getConstant(1, dl, MVT::i32), Overflow);
+ Overflow = carryFlagToValue(Value.getValue(1), VT, DAG, true);
break;
}
}
@@ -6859,21 +6873,19 @@ static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG,
static SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) {
SDValue LHS = Op.getOperand(0);
SDValue RHS = Op.getOperand(1);
+
+ assert(LHS.getSimpleValueType().isInteger() && "SETCCCARRY is integer only.");
+
SDValue Carry = Op.getOperand(2);
SDValue Cond = Op.getOperand(3);
SDLoc DL(Op);
- assert(LHS.getSimpleValueType().isInteger() && "SETCCCARRY is integer only.");
-
// ARMISD::SUBE expects a carry not a borrow like ISD::USUBO_CARRY so we
// have to invert the carry first.
- Carry = DAG.getNode(ISD::SUB, DL, MVT::i32,
- DAG.getConstant(1, DL, MVT::i32), Carry);
- // This converts the boolean value carry into the carry flag.
- Carry = ConvertBooleanCarryToCarryFlag(Carry, DAG);
+ SDValue InvCarry = valueToCarryFlag(Carry, DAG, true);
SDVTList VTs = DAG.getVTList(LHS.getValueType(), MVT::i32);
- SDValue Cmp = DAG.getNode(ARMISD::SUBE, DL, VTs, LHS, RHS, Carry);
+ SDValue Cmp = DAG.getNode(ARMISD::SUBE, DL, VTs, LHS, RHS, InvCarry);
SDValue FVal = DAG.getConstant(0, DL, MVT::i32);
SDValue TVal = DAG.getConstant(1, DL, MVT::i32);
@@ -9766,48 +9778,26 @@ static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG,
return N0;
}
-static SDValue LowerUADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG) {
- SDNode *N = Op.getNode();
- EVT VT = N->getValueType(0);
- SDVTList VTs = DAG.getVTList(VT, MVT::i32);
+static SDValue LowerADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG,
+ unsigned Opcode, bool IsSigned) {
+ EVT VT0 = Op.getValue(0).getValueType();
+ EVT VT1 = Op.getValue(1).getValueType();
- SDValue Carry = Op.getOperand(2);
+ bool InvertCarry = Opcode == ARMISD::SUBE;
+ SDValue OpLHS = Op.getOperand(0);
+ SDValue OpRHS = Op.getOperand(1);
+ SDValue OpCarryIn = valueToCarryFlag(Op.getOperand(2), DAG, InvertCarry);
SDLoc DL(Op);
- SDValue Result;
- if (Op.getOpcode() == ISD::UADDO_CARRY) {
- // This converts the boolean value carry into the carry flag.
- Carry = ConvertBooleanCarryToCarryFlag(Carry, DAG);
-
- // Do the addition proper using the carry flag we wanted.
- Result = DAG.getNode(ARMISD::ADDE, DL, VTs, Op.getOperand(0),
- Op.getOperand(1), Carry);
-
- // Now convert the carry flag into a boolean value.
- Carry = ConvertCarryFlagToBooleanCarry(Result.getValue(1), VT, DAG);
- } else {
- // ARMISD::SUBE expects a carry not a borrow like ISD::USUBO_CARRY so we
- // have to invert the carry first.
- Carry = DAG.getNode(ISD::SUB, DL, MVT::i32,
- DAG.getConstant(1, DL, MVT::i32), Carry);
- // This converts the boolean value carry into the carry flag.
- Carry = ConvertBooleanCarryToCarryFlag(Carry, DAG);
-
- // Do the subtraction proper using the carry flag we wanted.
- Result = DAG.getNode(ARMISD::SUBE, DL, VTs, Op.getOperand(0),
- Op.getOperand(1), Carry);
+ SDValue Sum = DAG.getNode(Opcode, DL, DAG.getVTList(VT0, MVT::i32), OpLHS,
+ OpRHS, OpCarryIn);
- // Now convert the carry flag into a boolean value.
- Carry = ConvertCarryFlagToBooleanCarry(Result.getValue(1), VT, DAG);
- // But the carry returned by ARMISD::SUBE is not a borrow as expected
- // by ISD::USUBO_CARRY, so compute 1 - C.
- Carry = DAG.getNode(ISD::SUB, DL, MVT::i32,
- DAG.getConstant(1, DL, MVT::i32), Carry);
- }
+ SDValue OutFlag =
+ IsSigned ? overflowFlagToValue(Sum.getValue(1), VT1, DAG)
+ : carryFlagToValue(Sum.getValue(1), VT1, DAG, InvertCarry);
- // Return both values.
- return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, Carry);
+ return DAG.getMergeValues({Sum, OutFlag}, DL);
}
SDValue ARMTargetLowering::LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const {
@@ -10651,8 +10641,13 @@ SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
return LowerDIV_Windows(Op, DAG, /* Signed */ false);
return LowerUDIV(Op, DAG, Subtarget);
case ISD::UADDO_CARRY:
+ return LowerADDSUBO_CARRY(Op, DAG, ARMISD::ADDE, false /*unsigned*/);
case ISD::USUBO_CARRY:
- return LowerUADDSUBO_CARRY(Op, DAG);
+ return LowerADDSUBO_CARRY(Op, DAG, ARMISD::SUBE, false /*unsigned*/);
+ case ISD::SADDO_CARRY:
+ return LowerADDSUBO_CARRY(Op, DAG, ARMISD::ADDE, true /*signed*/);
+ case ISD::SSUBO_CARRY:
+ return LowerADDSUBO_CARRY(Op, DAG, ARMISD::SUBE, true /*signed*/);
case ISD::SADDO:
case ISD::SSUBO:
return LowerSignedALUO(Op, DAG);
More information about the llvm-commits
mailing list