[llvm] [X86] Lower i512 ADD/SUB using Kogge-Stone on AVX512 (PR #174761)

[Simon Pilgrim via llvm-commits]

================
@@ -34031,6 +34034,65 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
     Results.push_back(DAG.getBitcast(VT, Op));
     return;
   }
+  case ISD::ADD:
+  case ISD::SUB: {
+    // Use Kogge-Stone parallel carry/borrow propagation for i512 add/sub.
+    // TODO: ISD::UADDO_CARRY
+    SDValue LHS = N->getOperand(0);
+    SDValue RHS = N->getOperand(1);
+    EVT VT = N->getValueType(0);
+    bool IsAdd = Opc == ISD::ADD;
+
+    assert(Subtarget.useAVX512Regs() && "AVX512 required");
+    assert(VT == MVT::i512 && "Unexpected VT!");
+
+    if (!mayFoldIntoVector(LHS, Subtarget) ||
+        !mayFoldIntoVector(RHS, Subtarget))
+      return;
+
+    MVT VecVT = MVT::getVectorVT(MVT::i64, 8);
+    MVT BoolVT = MVT::getVectorVT(MVT::i1, 8);
+    SDValue Vec0 = DAG.getBitcast(VecVT, LHS);
+    SDValue Vec1 = DAG.getBitcast(VecVT, RHS);
+    SDValue AllOnes = DAG.getAllOnesConstant(dl, VecVT);
+
+    // Compute partial sum/difference (per-lane, no carry propagation).
+    SDValue Partial = DAG.getNode(Opc, dl, VecVT, Vec0, Vec1);
+
+    // Detect carry/borrow generation.
+    ISD::CondCode CarryCC = IsAdd ? ISD::SETULT : ISD::SETUGT;
+    SDValue Carry = DAG.getSetCC(dl, BoolVT, Partial, Vec0, CarryCC);
+
+    // Detect propagate lanes.
+    SDValue PropCmp = IsAdd ? AllOnes : DAG.getConstant(0, dl, VecVT);
+    SDValue Propagate = DAG.getSetCC(dl, BoolVT, Partial, PropCmp, ISD::SETEQ);
+
+    // Convert masks to scalar for Kogge-Stone propagation.
+    SDValue CarryIn = DAG.getNode(ISD::BITCAST, dl, MVT::i8, Carry);
+    SDValue PropIn = DAG.getNode(ISD::BITCAST, dl, MVT::i8, Propagate);
----------------
RKSimon wrote:

```suggestion
    SDValue PropIn = DAG.getBitcast(MVT::i8, Propagate);
```

https://github.com/llvm/llvm-project/pull/174761