[clang] [llvm] Add support for flag output operand "=@cc" for SystemZ. (PR #125970)
Ulrich Weigand via cfe-commits
cfe-commits at lists.llvm.org
Tue Jun 24 09:49:17 PDT 2025
================
@@ -8781,8 +8961,445 @@ static bool combineCCMask(SDValue &CCReg, int &CCValid, int &CCMask) {
return false;
}
-SDValue SystemZTargetLowering::combineBR_CCMASK(
- SDNode *N, DAGCombinerInfo &DCI) const {
+// Combine (select_cc_a (select_cc_b)), where select_cc_a has one of TrueVal
+// or FalseVal has nested select_cc_b(already been combined sequence)
+SDValue
+SystemZTargetLowering::combineSELECT_CC_CCIPMMask(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ SelectionDAG &DAG = DCI.DAG;
+ // Check if CCOp1 and CCOp2 refers to the same CC condition node.
+ const auto isSameCCIPMOp = [](SDValue &CCOp1, SDValue &CCOp2,
+ int &CCValidVal) {
+ // Already combined sequence.
+ if (CCValidVal != SystemZ::CCMASK_ANY)
+ return false;
+ SDNode *N1 = CCOp1.getNode(), *N2 = CCOp2.getNode();
+ return N1 && N2 && N1 == N2;
+ };
+
+ auto *OuterTrueVal = dyn_cast<ConstantSDNode>(N->getOperand(0));
+ auto *OuterFalseVal = dyn_cast<ConstantSDNode>(N->getOperand(1));
+
+ // Already handled the case both operands constant in combineCCMask.
+ // Not yet encountered the case where both operands are sub-expressions,
+ // that case can be handled by removing this condition.
+ if (!((OuterTrueVal != nullptr) ^ (OuterFalseVal != nullptr)))
+ return SDValue();
+
+ SDValue NestedCCOp = OuterTrueVal ? N->getOperand(1) : N->getOperand(0);
+ auto *NestedCCNode = NestedCCOp.getNode();
+ // check if nested select_cc_b has already been combined.
+ if (!NestedCCNode || NestedCCNode->getOpcode() != SystemZISD::SELECT_CCMASK)
+ return SDValue();
+
+ auto *NestedTrueVal = dyn_cast<ConstantSDNode>(NestedCCNode->getOperand(0));
+ auto *NestedFalseVal = dyn_cast<ConstantSDNode>(NestedCCNode->getOperand(1));
+ if (!NestedTrueVal || !NestedFalseVal)
+ return SDValue();
+ bool Invert = false;
+ // Check if outer select_cc_a and nested select_cc_b True/False matching
+ // or inverted.
+ if (OuterTrueVal) {
+ // OuterFalseVal points to already combined nested select_cc_b.
+ if (OuterTrueVal->getZExtValue() == NestedFalseVal->getZExtValue())
+ Invert = !Invert; // Inverted.
+ else if (OuterTrueVal->getZExtValue() != NestedTrueVal->getZExtValue())
+ return SDValue();
+ } else if (OuterFalseVal) {
+ // OuterTrueVal points to already combined nested select_cc_b.
+ if (OuterFalseVal->getZExtValue() == NestedTrueVal->getZExtValue())
+ Invert = !Invert; // Inverted.
+ else if (OuterFalseVal->getZExtValue() != NestedFalseVal->getZExtValue())
+ return SDValue();
+ }
+ auto *OuterCCValid = dyn_cast<ConstantSDNode>(N->getOperand(2));
+ auto *OuterCCMask = dyn_cast<ConstantSDNode>(N->getOperand(3));
+ auto *NestedCCValid = dyn_cast<ConstantSDNode>(NestedCCOp->getOperand(2));
+ auto *NestedCCMask = dyn_cast<ConstantSDNode>(NestedCCOp->getOperand(3));
+ if (!OuterCCValid || !OuterCCMask || !NestedCCValid || !NestedCCMask)
+ return SDValue();
+
+ int OuterCCValidVal = OuterCCValid->getZExtValue();
+ int OuterCCMaskVal = OuterCCMask->getZExtValue();
+ int NestedCCValidVal = NestedCCValid->getZExtValue();
+ int NestedCCMaskVal = NestedCCMask->getZExtValue();
+ int CCMask = OuterCCMaskVal;
+ SDValue OuterCCReg = N->getOperand(4);
+ SDValue NestedCCReg = NestedCCOp->getOperand(4);
+
+ // Combine two already combined (select_cc_a (select_cc_b)), where TrueVal
+ // of select_cc_a points to select_cc_b. We return select_cc with TrueVal
+ // and FalseVal from select_cc_b with combined CCMask.
+ // One of OuterTrueVal or OuterFalseVal has select_cc_b.
+ if ((OuterTrueVal != nullptr) ^ (OuterFalseVal != nullptr)) {
+ // Both OuterCCValidVal and NestedCCValidVal have already been combined.
+ if (OuterCCValidVal == SystemZ::CCMASK_ANY &&
+ // And both points to the same CC.
+ isSameCCIPMOp(OuterCCReg, NestedCCReg, NestedCCValidVal)) {
+ CCMask |= NestedCCMaskVal;
+ // NestedCCOp has both operands constants.
+ auto Op0 = NestedCCOp->getOperand(0);
+ auto Op1 = NestedCCOp->getOperand(1);
+ // Return combined select_cc.
+ return DAG.getNode(
+ SystemZISD::SELECT_CCMASK, SDLoc(N), N->getValueType(0), Op0, Op1,
+ DAG.getTargetConstant(OuterCCValidVal, SDLoc(N), MVT::i32),
+ DAG.getTargetConstant(CCMask, SDLoc(N), MVT::i32), NestedCCReg);
+ }
+ }
+ // Now handles the case where outer select_cc_a has not yet been combined.
+ // Combine outer select_cc and check if it corresponds to the same
+ // CC as nested CC.
+ // Outer select_cc has yet not been combined.
+ if (OuterCCValidVal != SystemZ::CCMASK_ICMP ||
+ // Try combining outer select_cc.
+ !combineCCMask(OuterCCReg, OuterCCValidVal, OuterCCMaskVal) ||
+ // Check nested select_cc has already been combined and points to
+ // same Condtiion code as outer select_cc.
+ !isSameCCIPMOp(OuterCCReg, NestedCCReg, NestedCCValidVal))
+ return SDValue();
+ // Check if nested select_cc original CCMask was CCMASK_CMP_EQ.
+ // Only one-bit is set in NestedCCMaskVal for CCMASK_CMP_EQ.
+ bool IsNestedCMP_EQ =
+ NestedCCMaskVal && !(NestedCCMaskVal & (NestedCCMaskVal - 1));
+
+ // Outer select_cc is inverted with respect to nested select_cc.
+ if (Invert)
+ OuterCCMaskVal ^= SystemZ::CCMASK_ANY;
+
+ // Intersection of masks.
+ if (CCMask != SystemZ::CCMASK_CMP_EQ && !IsNestedCMP_EQ)
+ OuterCCMaskVal &= NestedCCMaskVal;
+ else // Union the masks.
+ OuterCCMaskVal |= NestedCCMaskVal;
+
+ SDValue Op0 = NestedCCOp->getOperand(0);
+ SDValue Op1 = NestedCCOp->getOperand(1);
+
+ return DAG.getNode(
+ SystemZISD::SELECT_CCMASK, SDLoc(N), N->getValueType(0), Op0, Op1,
+ DAG.getTargetConstant(OuterCCValidVal, SDLoc(N), MVT::i32),
+ DAG.getTargetConstant(OuterCCMaskVal, SDLoc(N), MVT::i32), OuterCCReg);
+}
+
+// Merging versus split in multiple branches cost.
+TargetLoweringBase::CondMergingParams
+SystemZTargetLowering::getJumpConditionMergingParams(Instruction::BinaryOps Opc,
+ const Value *Lhs,
+ const Value *Rhs) const {
+ const auto isFlagOutOpCC = [](const Value *V) {
+ using namespace llvm::PatternMatch;
+ const Value *RHSVal;
+ const APInt *RHSC;
+ if (const auto *I = dyn_cast<Instruction>(V)) {
+ // PatternMatch.h provides concise tree-based pattern match of llvm IR.
+ if (match(I->getOperand(0), m_And(m_Value(RHSVal), m_APInt(RHSC))) ||
+ match(I, m_Cmp(m_Value(RHSVal), m_APInt(RHSC)))) {
+ if (const auto *CB = dyn_cast<CallBase>(RHSVal)) {
+ if (CB->isInlineAsm()) {
+ const InlineAsm *IA = cast<InlineAsm>(CB->getCalledOperand());
+ return IA &&
+ IA->getConstraintString().find("{@cc}") != std::string::npos;
+ }
+ }
+ }
+ }
+ return false;
+ };
+ // Pattern (ICmp %asm) or (ICmp (And %asm)).
+ // Cost of longest dependency chain (ICmp, And) is 2. CostThreshold or
+ // BaseCost can be set >=2. If cost of instruction <= CostThreshold
+ // conditionals will be merged or else conditionals will be split.
+ if (isFlagOutOpCC(Lhs) && isFlagOutOpCC(Rhs))
+ return {3, 0, -1};
+ // Default.
+ return {-1, -1, -1};
+}
+
+SDValue SystemZTargetLowering::combineTM(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ SelectionDAG &DAG = DCI.DAG;
+ auto *TMOp0Node = N->getOperand(0).getNode();
+ auto *TMOp1Const = dyn_cast<ConstantSDNode>(N->getOperand(1));
+ auto *TMOp2Const = dyn_cast<ConstantSDNode>(N->getOperand(2));
+ if (!TMOp0Node || !TMOp1Const || !TMOp2Const ||
+ // Third operand of TM is false.
+ TMOp2Const->getZExtValue() != 0)
+ return SDValue();
+ auto TMOp1ConstVal = TMOp1Const->getZExtValue();
+ // Optimize (TM (IPM)).
+ if (TMOp0Node->getOpcode() == SystemZISD::IPM) {
+ int CCMask, CCValid;
+ if (TMOp1ConstVal == (1 << SystemZ::IPM_CC))
+ CCMask = SystemZ::CCMASK_CMP_GE;
+ else if (TMOp1ConstVal == (1 << (SystemZ::IPM_CC + 1)))
+ CCMask = SystemZ::CCMASK_CMP_LE;
+ else
+ return SDValue();
+ SDValue CCReg = TMOp0Node->getOperand(0);
+ CCValid = SystemZ::CCMASK_ANY;
+ // Return combined node.
+ return DAG.getNode(SystemZISD::SELECT_CCMASK, SDLoc(N), N->getValueType(0),
+ N->getOperand(0), N->getOperand(1),
+ DAG.getTargetConstant(CCValid, SDLoc(N), MVT::i32),
+ DAG.getTargetConstant(CCMask, SDLoc(N), MVT::i32),
+ CCReg);
+ }
+ // Optimize (TM (XOR (Op0 Op1))).
+ if (TMOp0Node->getOpcode() == ISD::XOR) {
+ auto *XorOp0 = TMOp0Node->getOperand(0).getNode();
+ // Op0. (SELECT_CCMASK (ICMP (SRL (IPM)))).
+ // Op1. (SRL (IPM (CC))).
+ if (XorOp0 && XorOp0->getOpcode() == SystemZISD::SELECT_CCMASK) {
+ auto *XorOp0CCValid = dyn_cast<ConstantSDNode>(XorOp0->getOperand(2));
+ auto *XorOp0CCMask = dyn_cast<ConstantSDNode>(XorOp0->getOperand(3));
+ if (!XorOp0CCValid || !XorOp0CCMask)
+ return SDValue();
+ SDValue XorOp0CCReg = XorOp0->getOperand(4);
+ int XorOp0CCMaskVal = XorOp0CCMask->getZExtValue();
+ int XorOp0CCValidVal = XorOp0CCValid->getZExtValue();
+ int CCMask = SystemZ::CCMASK_CMP_EQ, CCValid, TMCCMask;
+ SDValue CCReg = TMOp0Node->getOperand(1);
+ // (SELECT_CCMASK (ICMP (SRL (IPM)))).
+ if (!combineCCMask(XorOp0CCReg, XorOp0CCValidVal, XorOp0CCMaskVal) ||
+ // (SRL (IPM (CC))).
+ !combineCCMask(CCReg, CCValid, CCMask))
+ return SDValue();
+ auto *N0 = XorOp0CCReg.getNode(), *N1 = CCReg.getNode();
+ // Check if Op0 and Op1 point to the same CC.
+ if (!N0 || !N1 || N0 != N1)
+ return SDValue();
+ if (TMOp1ConstVal == 1)
+ TMCCMask = SystemZ::CCMASK_CMP_GE;
+ else
+ return SDValue();
+ // CCMask ^ XorOp0CCMaskVal = TMCCMask..
+ CCMask = XorOp0CCMaskVal ^ TMCCMask;
+ // Returned combined node with evaluated CCMask.
+ return DAG.getNode(
+ SystemZISD::SELECT_CCMASK, SDLoc(N), N->getValueType(0),
+ XorOp0->getOperand(0), XorOp0->getOperand(1),
+ DAG.getTargetConstant(XorOp0CCValidVal, SDLoc(N), MVT::i32),
+ DAG.getTargetConstant(CCMask, SDLoc(N), MVT::i32), CCReg);
+ }
+ }
+ return SDValue();
+}
+
+SDValue SystemZTargetLowering::combineAND(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ SelectionDAG &DAG = DCI.DAG;
+ auto *AndOp0 = N->getOperand(0).getNode();
+ auto *AndOp1 = N->getOperand(1).getNode();
+ if (!AndOp0 || !AndOp1)
+ return SDValue();
+ // Both Operands of ISD::AND are SystemZISD::SELECT_CCMASK.
+ // And CCMask of both operands and check if they points to the
+ // same CC and update CCReg. Return combined SDNode.
+ if (AndOp0->getOpcode() == SystemZISD::SELECT_CCMASK &&
+ AndOp1->getOpcode() == SystemZISD::SELECT_CCMASK) {
+ auto *Op0CCValid = dyn_cast<ConstantSDNode>(AndOp0->getOperand(2));
+ auto *Op0CCMask = dyn_cast<ConstantSDNode>(AndOp0->getOperand(3));
+ auto *Op1CCValid = dyn_cast<ConstantSDNode>(AndOp1->getOperand(2));
+ auto *Op1CCMask = dyn_cast<ConstantSDNode>(AndOp1->getOperand(3));
+ if (!Op0CCValid || !Op1CCValid || !Op0CCMask || !Op1CCMask)
+ return SDValue();
+ int Op0CCValidVal = Op0CCValid->getZExtValue();
+ int Op1CCValidVal = Op1CCValid->getZExtValue();
+ // Check if both AndOp0 and AndOp1 have aleady been combined.
+ if (Op0CCValidVal != SystemZ::CCMASK_ANY ||
+ Op1CCValidVal != SystemZ::CCMASK_ANY)
+ return SDValue();
+ SDValue Op0CCReg = AndOp0->getOperand(4), Op1CCReg = AndOp1->getOperand(4);
+ auto *CCNode0 = Op0CCReg.getNode(), *CCNode1 = Op1CCReg.getNode();
+ // Check if AndOp0 and AndOp1 refers to same CC.
+ if (!CCNode0 || !CCNode1 || CCNode0 != CCNode1)
+ return SDValue();
+ int Op0CCMaskVal = Op0CCMask->getZExtValue();
+ int Op1CCMaskVal = Op1CCMask->getZExtValue();
+ int CCMask = Op0CCMaskVal & Op1CCMaskVal;
+ return DAG.getNode(SystemZISD::SELECT_CCMASK, SDLoc(N), N->getValueType(0),
+ AndOp0->getOperand(0), AndOp0->getOperand(1),
+ DAG.getTargetConstant(Op0CCValidVal, SDLoc(N), MVT::i32),
+ DAG.getTargetConstant(CCMask, SDLoc(N), MVT::i32),
+ Op0CCReg);
+ } else if (AndOp0->getOpcode() == SystemZISD::SELECT_CCMASK) {
+ // check AndOp1 for (SRL (IPM (CC))) pattern. Second operand is CC.
+ SDValue CCReg = N->getOperand(1);
+ int CCMask = SystemZ::CCMASK_CMP_EQ, CCValid;
+ if (!combineCCMask(CCReg, CCValid, CCMask))
+ return SDValue();
+ SDValue Op0CCReg = AndOp0->getOperand(4);
+ auto *CCNode0 = Op0CCReg.getNode(), *CCNode1 = CCReg.getNode();
+ // Check if AndOp0 and AndOp1 refers to same CC.
+ if (!CCNode0 || !CCNode1 || CCNode0 != CCNode1)
+ return SDValue();
+ auto *Op0CCValid = dyn_cast<ConstantSDNode>(AndOp0->getOperand(2));
+ auto *Op0CCMask = dyn_cast<ConstantSDNode>(AndOp0->getOperand(3));
+ int Op0CCValidVal = Op0CCValid->getZExtValue();
+ if (!Op0CCMask || !Op0CCValid || Op0CCValidVal != SystemZ::CCMASK_ANY)
+ return SDValue();
+ int Op0CCMaskVal = Op0CCMask->getZExtValue();
+ // Op0CCMaskVal & CCMask = 0. Invert Op0CCMaskVal.
+ int CCMaskVal = Op0CCMaskVal ^ 0xf;
+ assert(CCMaskVal < 4 && "CC out of range");
+ CCMask = 1 << (3 - CCMaskVal);
+ return DAG.getNode(SystemZISD::SELECT_CCMASK, SDLoc(N), N->getValueType(0),
+ AndOp0->getOperand(0), AndOp0->getOperand(1),
+ DAG.getTargetConstant(Op0CCValidVal, SDLoc(N), MVT::i32),
+ DAG.getTargetConstant(CCMask, SDLoc(N), MVT::i32),
+ CCReg);
+ }
+ return SDValue();
+}
+
+SDValue SystemZTargetLowering::combineOR(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ SelectionDAG &DAG = DCI.DAG;
+ auto *OrOp0 = N->getOperand(0).getNode();
+ auto *OrOp1 = N->getOperand(1).getNode();
+ if (!OrOp0 || !OrOp1)
+ return SDValue();
+ // Both Operands of ISD::OR are SystemZISD::SELECT_CCMASK.
+ // And CCMask of both operands and check if they points to the
+ // same CC and update CCReg. Return combined SDNode.
+ if (OrOp0->getOpcode() == SystemZISD::SELECT_CCMASK &&
+ OrOp1->getOpcode() == SystemZISD::SELECT_CCMASK) {
+ auto *Op0CCValid = dyn_cast<ConstantSDNode>(OrOp0->getOperand(2));
+ auto *Op0CCMask = dyn_cast<ConstantSDNode>(OrOp0->getOperand(3));
+ auto *Op1CCValid = dyn_cast<ConstantSDNode>(OrOp1->getOperand(2));
+ auto *Op1CCMask = dyn_cast<ConstantSDNode>(OrOp1->getOperand(3));
+ if (!Op0CCValid || !Op1CCValid || !Op0CCMask || !Op1CCMask)
+ return SDValue();
+ int Op0CCValidVal = Op0CCValid->getZExtValue();
+ int Op1CCValidVal = Op1CCValid->getZExtValue();
+ // Check if both OrOp0 and OrOp1 have aleady been combined.
+ if (Op0CCValidVal != SystemZ::CCMASK_ANY ||
+ Op1CCValidVal != SystemZ::CCMASK_ANY)
+ return SDValue();
+ SDValue Op0CCReg = OrOp0->getOperand(4), Op1CCReg = OrOp1->getOperand(4);
+ auto *CCNode0 = Op0CCReg.getNode(), *CCNode1 = Op1CCReg.getNode();
+ // Check if OrOp0 and OrOp0 refers to same CC.
+ if (!CCNode0 || !CCNode1 || CCNode0 != CCNode1)
+ return SDValue();
+ int Op0CCMaskVal = Op0CCMask->getZExtValue();
+ int Op1CCMaskVal = Op1CCMask->getZExtValue();
+ // Oring Masks.
+ int CCMask = Op0CCMaskVal | Op1CCMaskVal;
+ return DAG.getNode(SystemZISD::SELECT_CCMASK, SDLoc(N), N->getValueType(0),
+ OrOp0->getOperand(0), OrOp1->getOperand(1),
+ DAG.getTargetConstant(Op0CCValidVal, SDLoc(N), MVT::i32),
+ DAG.getTargetConstant(CCMask, SDLoc(N), MVT::i32),
+ Op0CCReg);
+ }
+ return SDValue();
+}
+
+SDValue SystemZTargetLowering::combineICMP(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ SelectionDAG &DAG = DCI.DAG;
+ auto *LHS = N->getOperand(0).getNode();
+ if (!LHS)
+ return SDValue();
+ auto *RHS = dyn_cast<ConstantSDNode>(N->getOperand(1));
+ // Handle the case where RHS is const.
+ // Remove redundant ICMP left after operand 0 pattern has been combined by
----------------
uweigand wrote:
Please don't talk in terms of what "has been combined" or not. This function should do a correct transformation on whatever DAG it sees. It does not matter at all how this DAG came to be or what past transformations it might have gone through. Either the transformation is correct or not.
https://github.com/llvm/llvm-project/pull/125970
More information about the cfe-commits
mailing list