[llvm] [InstCombine][X86] Fold add(vpmadd52(0, a, b), x) -> vpmadd52(x, a, b) (PR #195937)

[via llvm-commits]

llvmorg-github-actions[bot] wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-llvm-transforms

Author: as4230

<details>
<summary>Changes</summary>

vpmadd52 computes acc + P(a, b) mod 2^64 where P is independent of acc so add(vpmadd52(0, a, b), x) = vpmadd52(x, a, b).

Applies to all six vpmadd52{h,l}.uq.{128,256,512} variants.

Fixes #<!-- -->194697.

---
Full diff: https://github.com/llvm/llvm-project/pull/195937.diff


2 Files Affected:

- (modified) llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp (+26) 
- (added) llvm/test/Transforms/InstCombine/X86/x86-vpmadd52.ll (+127) 


``````````diff
diff --git a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
index 932b4a416a8d3..393e1fb98eca8 100644
--- a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
@@ -3204,6 +3204,32 @@ X86TTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
     }
     break;
 
+  case Intrinsic::x86_avx512_vpmadd52h_uq_128:
+  case Intrinsic::x86_avx512_vpmadd52l_uq_128:
+  case Intrinsic::x86_avx512_vpmadd52h_uq_256:
+  case Intrinsic::x86_avx512_vpmadd52l_uq_256:
+  case Intrinsic::x86_avx512_vpmadd52h_uq_512:
+  case Intrinsic::x86_avx512_vpmadd52l_uq_512: {
+    // Fold add(vpmadd52(<zero>, a, b), x) -> vpmadd52(x, a, b)
+    Value *Acc = II.getArgOperand(0);
+    if (!match(Acc, m_Zero()) || !II.hasOneUse())
+      break;
+
+    auto *Add = dyn_cast<BinaryOperator>(*II.user_begin());
+    if (!Add || Add->getOpcode() != Instruction::Add)
+      break;
+
+    Value *X =
+        Add->getOperand(0) == &II ? Add->getOperand(1) : Add->getOperand(0);
+
+    Value *NewCall = IC.Builder.CreateIntrinsic(
+        IID, {}, {X, II.getArgOperand(1), II.getArgOperand(2)});
+
+    IC.replaceInstUsesWith(*Add, NewCall);
+    IC.eraseInstFromFunction(*Add);
+    return IC.eraseInstFromFunction(II);
+  }
+
   default:
     break;
   }
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-vpmadd52.ll b/llvm/test/Transforms/InstCombine/X86/x86-vpmadd52.ll
new file mode 100644
index 0000000000000..fcfb412dfd90b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-vpmadd52.ll
@@ -0,0 +1,127 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=instcombine -mtriple=x86_64-- -mattr=+avx512ifma -S | FileCheck %s
+
+declare <2 x i64> @llvm.x86.avx512.vpmadd52h.uq.128(<2 x i64>, <2 x i64>, <2 x i64>)
+declare <2 x i64> @llvm.x86.avx512.vpmadd52l.uq.128(<2 x i64>, <2 x i64>, <2 x i64>)
+declare <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64>, <4 x i64>, <4 x i64>)
+declare <4 x i64> @llvm.x86.avx512.vpmadd52l.uq.256(<4 x i64>, <4 x i64>, <4 x i64>)
+declare <8 x i64> @llvm.x86.avx512.vpmadd52h.uq.512(<8 x i64>, <8 x i64>, <8 x i64>)
+declare <8 x i64> @llvm.x86.avx512.vpmadd52l.uq.512(<8 x i64>, <8 x i64>, <8 x i64>)
+
+; ---- Positive cases should fold ----
+
+define <2 x i64> @fold_h_128(<2 x i64> %x, <2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: @fold_h_128(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i64> @llvm.x86.avx512.vpmadd52h.uq.128(<2 x i64> [[X:%.*]], <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]])
+; CHECK-NEXT:    ret <2 x i64> [[R]]
+;
+  %m = call <2 x i64> @llvm.x86.avx512.vpmadd52h.uq.128(<2 x i64> zeroinitializer, <2 x i64> %a, <2 x i64> %b)
+  %r = add <2 x i64> %m, %x
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @fold_l_128(<2 x i64> %x, <2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: @fold_l_128(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i64> @llvm.x86.avx512.vpmadd52l.uq.128(<2 x i64> [[X:%.*]], <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]])
+; CHECK-NEXT:    ret <2 x i64> [[R]]
+;
+  %m = call <2 x i64> @llvm.x86.avx512.vpmadd52l.uq.128(<2 x i64> zeroinitializer, <2 x i64> %a, <2 x i64> %b)
+  %r = add <2 x i64> %m, %x
+  ret <2 x i64> %r
+}
+
+define <4 x i64> @fold_h_256(<4 x i64> %x, <4 x i64> %a, <4 x i64> %b) {
+; CHECK-LABEL: @fold_h_256(
+; CHECK-NEXT:    [[R:%.*]] = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> [[X:%.*]], <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]])
+; CHECK-NEXT:    ret <4 x i64> [[R]]
+;
+  %m = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> %a, <4 x i64> %b)
+  %r = add <4 x i64> %m, %x
+  ret <4 x i64> %r
+}
+
+define <4 x i64> @fold_l_256(<4 x i64> %x, <4 x i64> %a, <4 x i64> %b) {
+; CHECK-LABEL: @fold_l_256(
+; CHECK-NEXT:    [[R:%.*]] = call <4 x i64> @llvm.x86.avx512.vpmadd52l.uq.256(<4 x i64> [[X:%.*]], <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]])
+; CHECK-NEXT:    ret <4 x i64> [[R]]
+;
+  %m = call <4 x i64> @llvm.x86.avx512.vpmadd52l.uq.256(<4 x i64> zeroinitializer, <4 x i64> %a, <4 x i64> %b)
+  %r = add <4 x i64> %m, %x
+  ret <4 x i64> %r
+}
+
+define <8 x i64> @fold_h_512(<8 x i64> %x, <8 x i64> %a, <8 x i64> %b) {
+; CHECK-LABEL: @fold_h_512(
+; CHECK-NEXT:    [[R:%.*]] = call <8 x i64> @llvm.x86.avx512.vpmadd52h.uq.512(<8 x i64> [[X:%.*]], <8 x i64> [[A:%.*]], <8 x i64> [[B:%.*]])
+; CHECK-NEXT:    ret <8 x i64> [[R]]
+;
+  %m = call <8 x i64> @llvm.x86.avx512.vpmadd52h.uq.512(<8 x i64> zeroinitializer, <8 x i64> %a, <8 x i64> %b)
+  %r = add <8 x i64> %m, %x
+  ret <8 x i64> %r
+}
+
+define <8 x i64> @fold_l_512(<8 x i64> %x, <8 x i64> %a, <8 x i64> %b) {
+; CHECK-LABEL: @fold_l_512(
+; CHECK-NEXT:    [[R:%.*]] = call <8 x i64> @llvm.x86.avx512.vpmadd52l.uq.512(<8 x i64> [[X:%.*]], <8 x i64> [[A:%.*]], <8 x i64> [[B:%.*]])
+; CHECK-NEXT:    ret <8 x i64> [[R]]
+;
+  %m = call <8 x i64> @llvm.x86.avx512.vpmadd52l.uq.512(<8 x i64> zeroinitializer, <8 x i64> %a, <8 x i64> %b)
+  %r = add <8 x i64> %m, %x
+  ret <8 x i64> %r
+}
+
+define <4 x i64> @fold_commuted(<4 x i64> %x, <4 x i64> %a, <4 x i64> %b) {
+; CHECK-LABEL: @fold_commuted(
+; CHECK-NEXT:    [[R:%.*]] = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> [[X:%.*]], <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]])
+; CHECK-NEXT:    ret <4 x i64> [[R]]
+;
+  %m = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> %a, <4 x i64> %b)
+  %r = add <4 x i64> %x, %m
+  ret <4 x i64> %r
+}
+
+; ---- Negative cases should NOT fold ----
+
+define <4 x i64> @nofold_nonzero_acc(<4 x i64> %x, <4 x i64> %acc, <4 x i64> %a, <4 x i64> %b) {
+; CHECK-LABEL: @nofold_nonzero_acc(
+; CHECK-NEXT:    [[M:%.*]] = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> [[ACC:%.*]], <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = add <4 x i64> [[M]], [[X:%.*]]
+; CHECK-NEXT:    ret <4 x i64> [[R]]
+;
+  %m = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> %acc, <4 x i64> %a, <4 x i64> %b)
+  %r = add <4 x i64> %m, %x
+  ret <4 x i64> %r
+}
+
+define <4 x i64> @nofold_multi_use(<4 x i64> %x, <4 x i64> %a, <4 x i64> %b, ptr %p) {
+; CHECK-LABEL: @nofold_multi_use(
+; CHECK-NEXT:    [[M:%.*]] = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]])
+; CHECK-NEXT:    store <4 x i64> [[M]], ptr [[P:%.*]], align 32
+; CHECK-NEXT:    [[R:%.*]] = add <4 x i64> [[M]], [[X:%.*]]
+; CHECK-NEXT:    ret <4 x i64> [[R]]
+;
+  %m = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> %a, <4 x i64> %b)
+  store <4 x i64> %m, ptr %p
+  %r = add <4 x i64> %m, %x
+  ret <4 x i64> %r
+}
+
+define <4 x i64> @nofold_sub(<4 x i64> %x, <4 x i64> %a, <4 x i64> %b) {
+; CHECK-LABEL: @nofold_sub(
+; CHECK-NEXT:    [[M:%.*]] = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = sub <4 x i64> [[M]], [[X:%.*]]
+; CHECK-NEXT:    ret <4 x i64> [[R]]
+;
+  %m = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> %a, <4 x i64> %b)
+  %r = sub <4 x i64> %m, %x
+  ret <4 x i64> %r
+}
+
+define <4 x i64> @nofold_no_add(<4 x i64> %a, <4 x i64> %b) {
+; CHECK-LABEL: @nofold_no_add(
+; CHECK-NEXT:    [[R:%.*]] = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]])
+; CHECK-NEXT:    ret <4 x i64> [[R]]
+;
+  %r = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> %a, <4 x i64> %b)
+  ret <4 x i64> %r
+}

``````````

</details>


https://github.com/llvm/llvm-project/pull/195937