[llvm] [X86][ISel] Improve VPTERNLOG matching for negated logic trees (PR #164863)

[Yi-Chi Lee via llvm-commits]

https://github.com/yichi170 updated https://github.com/llvm/llvm-project/pull/164863

>From f66a0cb105e46692f147c453c79ef60f38b93861 Mon Sep 17 00:00:00 2001
From: Yi-Chi Lee <yichi170 at gmail.com>
Date: Thu, 23 Oct 2025 12:45:49 -0500
Subject: [PATCH 1/2] [X86][ISel] Improve VPTERNLOG matching for negated logic
 trees

---
 llvm/lib/Target/X86/X86ISelDAGToDAG.cpp | 39 ++++++++++++++++++++++---
 llvm/test/CodeGen/X86/issue163738.ll    | 13 +++++++++
 2 files changed, 48 insertions(+), 4 deletions(-)
 create mode 100644 llvm/test/CodeGen/X86/issue163738.ll

diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index 62073ec125e8f..06a691d73c749 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -4721,9 +4721,6 @@ bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
   if (!(Subtarget->hasVLX() || NVT.is512BitVector()))
     return false;
 
-  SDValue N0 = N->getOperand(0);
-  SDValue N1 = N->getOperand(1);
-
   auto getFoldableLogicOp = [](SDValue Op) {
     // Peek through single use bitcast.
     if (Op.getOpcode() == ISD::BITCAST && Op.hasOneUse())
@@ -4740,6 +4737,37 @@ bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
     return SDValue();
   };
 
+  // Identify and (optionally) peel an outer NOT that wraps a pure logic tree
+  auto tryPeelOuterNotWrappingLogic = [&](SDNode *Op) {
+    if (Op->getOpcode() == ISD::XOR && Op->hasOneUse() &&
+        ISD::isBuildVectorAllOnes(Op->getOperand(1).getNode())) {
+      SDNode *InnerN = Op->getOperand(0).getNode();
+
+      unsigned InnerOpc = InnerN->getOpcode();
+      if (InnerOpc != ISD::AND && InnerOpc != ISD::OR &&
+          InnerOpc != ISD::XOR && InnerOpc != X86ISD::ANDNP) {
+        return Op;
+      }
+
+      SDValue InnerN0 = InnerN->getOperand(0);
+      SDValue InnerN1 = InnerN->getOperand(1);
+      SDValue FoldableOp;
+      if (getFoldableLogicOp(InnerN1) || getFoldableLogicOp(InnerN0))
+        return InnerN;
+    }
+    return Op;
+  };
+
+  SDNode *OriN = N;
+  bool PeeledOuterNot = false;
+  N = tryPeelOuterNotWrappingLogic(N);
+  if (N != OriN)
+    PeeledOuterNot = true;
+    
+
+  SDValue N0 = N->getOperand(0);
+  SDValue N1 = N->getOperand(1);
+
   SDValue A, FoldableOp;
   if ((FoldableOp = getFoldableLogicOp(N1))) {
     A = N0;
@@ -4798,7 +4826,10 @@ bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
   case ISD::XOR: Imm ^= TernlogMagicA; break;
   }
 
-  return matchVPTERNLOG(N, ParentA, ParentB, ParentC, A, B, C, Imm);
+  if (PeeledOuterNot)
+    Imm = ~Imm;
+
+  return matchVPTERNLOG(OriN, ParentA, ParentB, ParentC, A, B, C, Imm);
 }
 
 /// If the high bits of an 'and' operand are known zero, try setting the
diff --git a/llvm/test/CodeGen/X86/issue163738.ll b/llvm/test/CodeGen/X86/issue163738.ll
new file mode 100644
index 0000000000000..61fe043a970dd
--- /dev/null
+++ b/llvm/test/CodeGen/X86/issue163738.ll
@@ -0,0 +1,13 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=CHECK
+
+define <8 x i64> @foo(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpternlogq {{.*#+}} zmm0 = ~(zmm0 | zmm2 | zmm1)
+; CHECK-NEXT:    retq
+  %and.demorgan = or <8 x i64> %b, %a
+  %and3.demorgan = or <8 x i64> %and.demorgan, %c
+  %and3 = xor <8 x i64> %and3.demorgan, splat (i64 -1)
+  ret <8 x i64> %and3
+}

>From 0cc7cd915cfbd8e7b813018f458b2ff81d2fc51f Mon Sep 17 00:00:00 2001
From: Yi-Chi Lee <yichi170 at gmail.com>
Date: Thu, 23 Oct 2025 13:08:55 -0500
Subject: [PATCH 2/2] clang-format

---
 llvm/lib/Target/X86/X86ISelDAGToDAG.cpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index 06a691d73c749..ab86901c262b0 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -4744,8 +4744,8 @@ bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
       SDNode *InnerN = Op->getOperand(0).getNode();
 
       unsigned InnerOpc = InnerN->getOpcode();
-      if (InnerOpc != ISD::AND && InnerOpc != ISD::OR &&
-          InnerOpc != ISD::XOR && InnerOpc != X86ISD::ANDNP) {
+      if (InnerOpc != ISD::AND && InnerOpc != ISD::OR && InnerOpc != ISD::XOR &&
+          InnerOpc != X86ISD::ANDNP) {
         return Op;
       }
 
@@ -4763,7 +4763,6 @@ bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
   N = tryPeelOuterNotWrappingLogic(N);
   if (N != OriN)
     PeeledOuterNot = true;
-    
 
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);