[llvm] [InstCombine][X86] Fold add(vpmadd52(0, a, b), x) -> vpmadd52(x, a, b) (PR #195937)

Adam Scott via llvm-commits llvm-commits at lists.llvm.org
Mon Jun 8 06:30:35 PDT 2026


https://github.com/as4230 updated https://github.com/llvm/llvm-project/pull/195937

>From 210794115119e9c50f770b42f1c3647153fe70ed Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Tue, 5 May 2026 16:48:34 -0400
Subject: [PATCH 1/4] [InstCombine][X86] Fold add(vpmadd52(0, a, b), x) ->
 vpmadd52(x, a, b)

vpmadd52 computes acc + P(a, b) mod 2^64 where P is independent of acc
so add(vpmadd52(0, a, b), x) = vpmadd52(x, a, b).

Applies to all six vpmadd52{h,l}.uq.{128,256,512} variants.

Fixes #194697.
---
 .../Target/X86/X86InstCombineIntrinsic.cpp    |  26 ++++
 .../InstCombine/X86/x86-vpmadd52.ll           | 127 ++++++++++++++++++
 2 files changed, 153 insertions(+)
 create mode 100644 llvm/test/Transforms/InstCombine/X86/x86-vpmadd52.ll

diff --git a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
index 932b4a416a8d3..393e1fb98eca8 100644
--- a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
@@ -3204,6 +3204,32 @@ X86TTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
     }
     break;
 
+  case Intrinsic::x86_avx512_vpmadd52h_uq_128:
+  case Intrinsic::x86_avx512_vpmadd52l_uq_128:
+  case Intrinsic::x86_avx512_vpmadd52h_uq_256:
+  case Intrinsic::x86_avx512_vpmadd52l_uq_256:
+  case Intrinsic::x86_avx512_vpmadd52h_uq_512:
+  case Intrinsic::x86_avx512_vpmadd52l_uq_512: {
+    // Fold add(vpmadd52(<zero>, a, b), x) -> vpmadd52(x, a, b)
+    Value *Acc = II.getArgOperand(0);
+    if (!match(Acc, m_Zero()) || !II.hasOneUse())
+      break;
+
+    auto *Add = dyn_cast<BinaryOperator>(*II.user_begin());
+    if (!Add || Add->getOpcode() != Instruction::Add)
+      break;
+
+    Value *X =
+        Add->getOperand(0) == &II ? Add->getOperand(1) : Add->getOperand(0);
+
+    Value *NewCall = IC.Builder.CreateIntrinsic(
+        IID, {}, {X, II.getArgOperand(1), II.getArgOperand(2)});
+
+    IC.replaceInstUsesWith(*Add, NewCall);
+    IC.eraseInstFromFunction(*Add);
+    return IC.eraseInstFromFunction(II);
+  }
+
   default:
     break;
   }
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-vpmadd52.ll b/llvm/test/Transforms/InstCombine/X86/x86-vpmadd52.ll
new file mode 100644
index 0000000000000..fcfb412dfd90b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-vpmadd52.ll
@@ -0,0 +1,127 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=instcombine -mtriple=x86_64-- -mattr=+avx512ifma -S | FileCheck %s
+
+declare <2 x i64> @llvm.x86.avx512.vpmadd52h.uq.128(<2 x i64>, <2 x i64>, <2 x i64>)
+declare <2 x i64> @llvm.x86.avx512.vpmadd52l.uq.128(<2 x i64>, <2 x i64>, <2 x i64>)
+declare <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64>, <4 x i64>, <4 x i64>)
+declare <4 x i64> @llvm.x86.avx512.vpmadd52l.uq.256(<4 x i64>, <4 x i64>, <4 x i64>)
+declare <8 x i64> @llvm.x86.avx512.vpmadd52h.uq.512(<8 x i64>, <8 x i64>, <8 x i64>)
+declare <8 x i64> @llvm.x86.avx512.vpmadd52l.uq.512(<8 x i64>, <8 x i64>, <8 x i64>)
+
+; ---- Positive cases should fold ----
+
+define <2 x i64> @fold_h_128(<2 x i64> %x, <2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: @fold_h_128(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i64> @llvm.x86.avx512.vpmadd52h.uq.128(<2 x i64> [[X:%.*]], <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]])
+; CHECK-NEXT:    ret <2 x i64> [[R]]
+;
+  %m = call <2 x i64> @llvm.x86.avx512.vpmadd52h.uq.128(<2 x i64> zeroinitializer, <2 x i64> %a, <2 x i64> %b)
+  %r = add <2 x i64> %m, %x
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @fold_l_128(<2 x i64> %x, <2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: @fold_l_128(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i64> @llvm.x86.avx512.vpmadd52l.uq.128(<2 x i64> [[X:%.*]], <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]])
+; CHECK-NEXT:    ret <2 x i64> [[R]]
+;
+  %m = call <2 x i64> @llvm.x86.avx512.vpmadd52l.uq.128(<2 x i64> zeroinitializer, <2 x i64> %a, <2 x i64> %b)
+  %r = add <2 x i64> %m, %x
+  ret <2 x i64> %r
+}
+
+define <4 x i64> @fold_h_256(<4 x i64> %x, <4 x i64> %a, <4 x i64> %b) {
+; CHECK-LABEL: @fold_h_256(
+; CHECK-NEXT:    [[R:%.*]] = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> [[X:%.*]], <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]])
+; CHECK-NEXT:    ret <4 x i64> [[R]]
+;
+  %m = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> %a, <4 x i64> %b)
+  %r = add <4 x i64> %m, %x
+  ret <4 x i64> %r
+}
+
+define <4 x i64> @fold_l_256(<4 x i64> %x, <4 x i64> %a, <4 x i64> %b) {
+; CHECK-LABEL: @fold_l_256(
+; CHECK-NEXT:    [[R:%.*]] = call <4 x i64> @llvm.x86.avx512.vpmadd52l.uq.256(<4 x i64> [[X:%.*]], <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]])
+; CHECK-NEXT:    ret <4 x i64> [[R]]
+;
+  %m = call <4 x i64> @llvm.x86.avx512.vpmadd52l.uq.256(<4 x i64> zeroinitializer, <4 x i64> %a, <4 x i64> %b)
+  %r = add <4 x i64> %m, %x
+  ret <4 x i64> %r
+}
+
+define <8 x i64> @fold_h_512(<8 x i64> %x, <8 x i64> %a, <8 x i64> %b) {
+; CHECK-LABEL: @fold_h_512(
+; CHECK-NEXT:    [[R:%.*]] = call <8 x i64> @llvm.x86.avx512.vpmadd52h.uq.512(<8 x i64> [[X:%.*]], <8 x i64> [[A:%.*]], <8 x i64> [[B:%.*]])
+; CHECK-NEXT:    ret <8 x i64> [[R]]
+;
+  %m = call <8 x i64> @llvm.x86.avx512.vpmadd52h.uq.512(<8 x i64> zeroinitializer, <8 x i64> %a, <8 x i64> %b)
+  %r = add <8 x i64> %m, %x
+  ret <8 x i64> %r
+}
+
+define <8 x i64> @fold_l_512(<8 x i64> %x, <8 x i64> %a, <8 x i64> %b) {
+; CHECK-LABEL: @fold_l_512(
+; CHECK-NEXT:    [[R:%.*]] = call <8 x i64> @llvm.x86.avx512.vpmadd52l.uq.512(<8 x i64> [[X:%.*]], <8 x i64> [[A:%.*]], <8 x i64> [[B:%.*]])
+; CHECK-NEXT:    ret <8 x i64> [[R]]
+;
+  %m = call <8 x i64> @llvm.x86.avx512.vpmadd52l.uq.512(<8 x i64> zeroinitializer, <8 x i64> %a, <8 x i64> %b)
+  %r = add <8 x i64> %m, %x
+  ret <8 x i64> %r
+}
+
+define <4 x i64> @fold_commuted(<4 x i64> %x, <4 x i64> %a, <4 x i64> %b) {
+; CHECK-LABEL: @fold_commuted(
+; CHECK-NEXT:    [[R:%.*]] = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> [[X:%.*]], <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]])
+; CHECK-NEXT:    ret <4 x i64> [[R]]
+;
+  %m = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> %a, <4 x i64> %b)
+  %r = add <4 x i64> %x, %m
+  ret <4 x i64> %r
+}
+
+; ---- Negative cases should NOT fold ----
+
+define <4 x i64> @nofold_nonzero_acc(<4 x i64> %x, <4 x i64> %acc, <4 x i64> %a, <4 x i64> %b) {
+; CHECK-LABEL: @nofold_nonzero_acc(
+; CHECK-NEXT:    [[M:%.*]] = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> [[ACC:%.*]], <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = add <4 x i64> [[M]], [[X:%.*]]
+; CHECK-NEXT:    ret <4 x i64> [[R]]
+;
+  %m = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> %acc, <4 x i64> %a, <4 x i64> %b)
+  %r = add <4 x i64> %m, %x
+  ret <4 x i64> %r
+}
+
+define <4 x i64> @nofold_multi_use(<4 x i64> %x, <4 x i64> %a, <4 x i64> %b, ptr %p) {
+; CHECK-LABEL: @nofold_multi_use(
+; CHECK-NEXT:    [[M:%.*]] = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]])
+; CHECK-NEXT:    store <4 x i64> [[M]], ptr [[P:%.*]], align 32
+; CHECK-NEXT:    [[R:%.*]] = add <4 x i64> [[M]], [[X:%.*]]
+; CHECK-NEXT:    ret <4 x i64> [[R]]
+;
+  %m = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> %a, <4 x i64> %b)
+  store <4 x i64> %m, ptr %p
+  %r = add <4 x i64> %m, %x
+  ret <4 x i64> %r
+}
+
+define <4 x i64> @nofold_sub(<4 x i64> %x, <4 x i64> %a, <4 x i64> %b) {
+; CHECK-LABEL: @nofold_sub(
+; CHECK-NEXT:    [[M:%.*]] = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = sub <4 x i64> [[M]], [[X:%.*]]
+; CHECK-NEXT:    ret <4 x i64> [[R]]
+;
+  %m = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> %a, <4 x i64> %b)
+  %r = sub <4 x i64> %m, %x
+  ret <4 x i64> %r
+}
+
+define <4 x i64> @nofold_no_add(<4 x i64> %a, <4 x i64> %b) {
+; CHECK-LABEL: @nofold_no_add(
+; CHECK-NEXT:    [[R:%.*]] = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]])
+; CHECK-NEXT:    ret <4 x i64> [[R]]
+;
+  %r = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> %a, <4 x i64> %b)
+  ret <4 x i64> %r
+}

>From a8a0cc1aa7a8d89b7d165746818cb391392eaf5d Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Wed, 6 May 2026 23:33:39 -0400
Subject: [PATCH 2/4] use m_c_Add and add repeated-use test

---
 llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp      |  6 ++----
 llvm/test/Transforms/InstCombine/X86/x86-vpmadd52.ll | 11 +++++++++++
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
index 393e1fb98eca8..b4634d7705eb3 100644
--- a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
@@ -3216,12 +3216,10 @@ X86TTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
       break;
 
     auto *Add = dyn_cast<BinaryOperator>(*II.user_begin());
-    if (!Add || Add->getOpcode() != Instruction::Add)
+    Value *X;
+    if (!Add || !match(Add, m_c_Add(m_Specific(&II), m_Value(X))))
       break;
 
-    Value *X =
-        Add->getOperand(0) == &II ? Add->getOperand(1) : Add->getOperand(0);
-
     Value *NewCall = IC.Builder.CreateIntrinsic(
         IID, {}, {X, II.getArgOperand(1), II.getArgOperand(2)});
 
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-vpmadd52.ll b/llvm/test/Transforms/InstCombine/X86/x86-vpmadd52.ll
index fcfb412dfd90b..613298dc98380 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-vpmadd52.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-vpmadd52.ll
@@ -106,6 +106,17 @@ define <4 x i64> @nofold_multi_use(<4 x i64> %x, <4 x i64> %a, <4 x i64> %b, ptr
   ret <4 x i64> %r
 }
 
+define <4 x i64> @nofold_madd_double_use(<4 x i64> %a, <4 x i64> %b) {
+; CHECK-LABEL: @nofold_madd_double_use(
+; CHECK-NEXT:    [[M:%.*]] = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = shl <4 x i64> [[M]], splat (i64 1)
+; CHECK-NEXT:    ret <4 x i64> [[R]]
+;
+  %m = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> %a, <4 x i64> %b)
+  %r = add <4 x i64> %m, %m
+  ret <4 x i64> %r
+}
+
 define <4 x i64> @nofold_sub(<4 x i64> %x, <4 x i64> %a, <4 x i64> %b) {
 ; CHECK-LABEL: @nofold_sub(
 ; CHECK-NEXT:    [[M:%.*]] = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]])

>From 56a991708ba08c62dc463dd65fb017000946dd2f Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Thu, 7 May 2026 05:56:41 -0400
Subject: [PATCH 3/4] add multi-call test. insert new madd52 at Add instead of
 II

---
 .../Target/X86/X86InstCombineIntrinsic.cpp    |  1 +
 .../InstCombine/X86/x86-vpmadd52.ll           | 27 +++++++++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
index b4634d7705eb3..4999581489e82 100644
--- a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
@@ -3220,6 +3220,7 @@ X86TTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
     if (!Add || !match(Add, m_c_Add(m_Specific(&II), m_Value(X))))
       break;
 
+    IC.Builder.SetInsertPoint(Add);
     Value *NewCall = IC.Builder.CreateIntrinsic(
         IID, {}, {X, II.getArgOperand(1), II.getArgOperand(2)});
 
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-vpmadd52.ll b/llvm/test/Transforms/InstCombine/X86/x86-vpmadd52.ll
index 613298dc98380..be832ba5b8e71 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-vpmadd52.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-vpmadd52.ll
@@ -80,6 +80,33 @@ define <4 x i64> @fold_commuted(<4 x i64> %x, <4 x i64> %a, <4 x i64> %b) {
   ret <4 x i64> %r
 }
 
+define <4 x i64> @fold_two_madd52_calls(<4 x i64> %a, <4 x i64> %b) {
+; CHECK-LABEL: @fold_two_madd52_calls(
+; CHECK-NEXT:    [[M2:%.*]] = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> [[M2]], <4 x i64> [[A]], <4 x i64> [[B]])
+; CHECK-NEXT:    ret <4 x i64> [[R]]
+;
+  %m1 = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> %a, <4 x i64> %b)
+  %m2 = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> %a, <4 x i64> %b)
+  %r = add <4 x i64> %m1, %m2
+  ret <4 x i64> %r
+}
+
+define <4 x i64> @fold_three_madd52_chain(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c, <4 x i64> %d, <4 x i64> %e, <4 x i64> %f) {
+; CHECK-LABEL: @fold_three_madd52_chain(
+; CHECK-NEXT:    [[M2:%.*]] = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> [[C:%.*]], <4 x i64> [[D:%.*]])
+; CHECK-NEXT:    [[T1:%.*]] = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> [[M2]], <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> [[T1]], <4 x i64> [[E:%.*]], <4 x i64> [[F:%.*]])
+; CHECK-NEXT:    ret <4 x i64> [[R]]
+;
+  %m1 = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> %a, <4 x i64> %b)
+  %m2 = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> %c, <4 x i64> %d)
+  %m3 = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> %e, <4 x i64> %f)
+  %t1 = add <4 x i64> %m1, %m2
+  %r  = add <4 x i64> %t1, %m3
+  ret <4 x i64> %r
+}
+
 ; ---- Negative cases should NOT fold ----
 
 define <4 x i64> @nofold_nonzero_acc(<4 x i64> %x, <4 x i64> %acc, <4 x i64> %a, <4 x i64> %b) {

>From 66ef635c7ba2d56e232d9f4e733cb70697444281 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Mon, 8 Jun 2026 12:54:53 +0000
Subject: [PATCH 4/4] [InstCombine][X86] Drop redundant declares from test

---
 llvm/test/Transforms/InstCombine/X86/x86-vpmadd52.ll | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/llvm/test/Transforms/InstCombine/X86/x86-vpmadd52.ll b/llvm/test/Transforms/InstCombine/X86/x86-vpmadd52.ll
index be832ba5b8e71..77b01f27bd244 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-vpmadd52.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-vpmadd52.ll
@@ -1,13 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -passes=instcombine -mtriple=x86_64-- -mattr=+avx512ifma -S | FileCheck %s
 
-declare <2 x i64> @llvm.x86.avx512.vpmadd52h.uq.128(<2 x i64>, <2 x i64>, <2 x i64>)
-declare <2 x i64> @llvm.x86.avx512.vpmadd52l.uq.128(<2 x i64>, <2 x i64>, <2 x i64>)
-declare <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64>, <4 x i64>, <4 x i64>)
-declare <4 x i64> @llvm.x86.avx512.vpmadd52l.uq.256(<4 x i64>, <4 x i64>, <4 x i64>)
-declare <8 x i64> @llvm.x86.avx512.vpmadd52h.uq.512(<8 x i64>, <8 x i64>, <8 x i64>)
-declare <8 x i64> @llvm.x86.avx512.vpmadd52l.uq.512(<8 x i64>, <8 x i64>, <8 x i64>)
-
 ; ---- Positive cases should fold ----
 
 define <2 x i64> @fold_h_128(<2 x i64> %x, <2 x i64> %a, <2 x i64> %b) {



More information about the llvm-commits mailing list