[llvm] [AVX-512] make vpternlogq more aggressive for longer chains of bitmanipulations (PR #189971)

[Simon Pilgrim via llvm-commits]

================
@@ -4829,118 +4848,170 @@ bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
   if (!(Subtarget->hasVLX() || NVT.is512BitVector()))
     return false;
 
-  auto getFoldableLogicOp = [](SDValue Op) {
-    // Peek through single use bitcast.
-    if (Op.getOpcode() == ISD::BITCAST && Op.hasOneUse())
-      Op = Op.getOperand(0);
-
-    if (!Op.hasOneUse())
-      return SDValue();
-
-    unsigned Opc = Op.getOpcode();
-    if (Opc == ISD::AND || Opc == ISD::OR || Opc == ISD::XOR ||
-        Opc == X86ISD::ANDNP)
-      return Op;
-
-    return SDValue();
+  auto IsLogic = [](unsigned Opc) {
+    return Opc == ISD::AND || Opc == ISD::OR || Opc == ISD::XOR ||
+           Opc == X86ISD::ANDNP;
+  };
+  auto IsNot = [](SDValue V) {
+    return V.getOpcode() == ISD::XOR &&
+           ISD::isBuildVectorAllOnes(V.getOperand(1).getNode());
+  };
+  auto PeelBitcast = [](SDValue V) {
+    if (V.getOpcode() == ISD::BITCAST && V.hasOneUse())
+      return V.getOperand(0);
+    return V;
+  };
+  auto IsLoadLike = [](SDValue V) {
+    return isa<LoadSDNode>(V.getNode()) ||
+           V.getOpcode() == X86ISD::VBROADCAST_LOAD;
   };
 
-  SDValue N0, N1, A, FoldableOp;
-
-  // Identify and (optionally) peel an outer NOT that wraps a pure logic tree
-  auto tryPeelOuterNotWrappingLogic = [&](SDNode *Op) {
-    if (Op->getOpcode() == ISD::XOR && Op->hasOneUse() &&
-        ISD::isBuildVectorAllOnes(Op->getOperand(1).getNode())) {
-      SDValue InnerOp = getFoldableLogicOp(Op->getOperand(0));
-
-      if (!InnerOp)
-        return SDValue();
+  struct Leaf {
+    SDValue V;
+    SDNode *Parent;
+  };
 
-      N0 = InnerOp.getOperand(0);
-      N1 = InnerOp.getOperand(1);
-      if ((FoldableOp = getFoldableLogicOp(N1))) {
-        A = N0;
-        return InnerOp;
-      }
-      if ((FoldableOp = getFoldableLogicOp(N0))) {
-        A = N1;
-        return InnerOp;
+  // Symbolically evaluate the tree rooted at Root.  Opaque, if non-null, is
+  // treated as a leaf even when it is a logic op (used for cascading).  On
+  // success fills Leaves (the distinct inputs in seed order, at most three),
+  // Imm and NumOps (the number of logic/NOT nodes folded - a profitability
+  // signal), and returns true.  Returns false when more than three distinct
+  // leaves are required.  Single-use is required to fold a node; bitcasts are
+  // peeled.
+  auto Evaluate = [&](SDValue Root, SDNode *Opaque,
+                      SmallVectorImpl<Leaf> &Leaves, uint8_t &Imm,
+                      unsigned &NumOps) -> bool {
+    static constexpr uint8_t Seeds[] = {0xF0, 0xCC, 0xAA};
+    NumOps = 0;
+    std::function<int(SDValue, SDNode *, bool)> Eval =
----------------
RKSimon wrote:

(style) `auto Eval = [&](SDValue Op, SDNode *Parent, bool IsRoot) -> int {`

https://github.com/llvm/llvm-project/pull/189971