[llvm] [AArch64][InstCombine] Combine add/sub with mull intrinsics (PR #207415)

via llvm-commits llvm-commits at lists.llvm.org
Fri Jul 3 08:33:27 PDT 2026


llvmorg-github-actions[bot] wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-backend-aarch64

Author: Gaƫtan Bossu (gbossu)

<details>
<summary>Changes</summary>

E.g. add(smullb(a, b), wide_acc) -> smlalb(wide_acc, a, b) where both smullb and smlalb are part of SVE2/SME.

---
Full diff: https://github.com/llvm/llvm-project/pull/207415.diff


2 Files Affected:

- (modified) llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp (+60) 
- (added) llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-mull-acc.ll (+110) 


``````````diff
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index edd95625e13ea..e38384fc171ad 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -2624,6 +2624,61 @@ instCombineSVEPairwiseAddLong(InstCombiner &IC, IntrinsicInst &II) {
   return &II; // II is now trivially dead and will get erased.
 }
 
+// Combine long mull and accumulate into fused mlal/lmsl
+// E.g. add(smullb(a, b), wide_acc) -> smlalb(wide_acc, a, b)
+static std::optional<Instruction *>
+instCombineSVEMultiplyAccumulateLong(InstCombiner &IC, IntrinsicInst &II) {
+  auto GetOpcodeForMLA = [](Intrinsic::ID IID) {
+    switch (IID) {
+    case Intrinsic::aarch64_sve_smullb:
+      return Intrinsic::aarch64_sve_smlalb;
+    case Intrinsic::aarch64_sve_smullt:
+      return Intrinsic::aarch64_sve_smlalt;
+    case Intrinsic::aarch64_sve_umullb:
+      return Intrinsic::aarch64_sve_umlalb;
+    case Intrinsic::aarch64_sve_umullt:
+      return Intrinsic::aarch64_sve_umlalt;
+    default:
+      llvm_unreachable("Expected SVE MULL intrinsic");
+    }
+  };
+  auto GetOpcodeForMLS = [](Intrinsic::ID IID) {
+    switch (IID) {
+    case Intrinsic::aarch64_sve_smullb:
+      return Intrinsic::aarch64_sve_smlslb;
+    case Intrinsic::aarch64_sve_smullt:
+      return Intrinsic::aarch64_sve_smlslt;
+    case Intrinsic::aarch64_sve_umullb:
+      return Intrinsic::aarch64_sve_umlslb;
+    case Intrinsic::aarch64_sve_umullt:
+      return Intrinsic::aarch64_sve_umlslt;
+    default:
+      llvm_unreachable("Expected SVE MULL intrinsic");
+    }
+  };
+
+  auto *User = dyn_cast_or_null<Instruction>(II.getUniqueUndroppableUser());
+  if (!User)
+    return std::nullopt;
+
+  Value *Acc;
+  Intrinsic::ID FuseIID;
+  if (match(User, m_c_Add(m_Specific(&II), m_Value(Acc))))
+    FuseIID = GetOpcodeForMLA(II.getIntrinsicID());
+  else if (match(User, m_Sub(m_Value(Acc), m_Specific(&II))))
+    FuseIID = GetOpcodeForMLS(II.getIntrinsicID());
+  else
+    return std::nullopt;
+
+  IC.Builder.SetInsertPoint(User);
+  Value *MulAcc = IC.Builder.CreateIntrinsic(
+      FuseIID, {II.getType()}, {Acc, II.getArgOperand(0), II.getArgOperand(1)});
+
+  IC.replaceInstUsesWith(*User, MulAcc);
+  IC.eraseInstFromFunction(*User);
+  return &II; // II is now trivially dead and will get erased.
+}
+
 static std::optional<Instruction *> instCombineSVEVectorAdd(InstCombiner &IC,
                                                             IntrinsicInst &II) {
   if (auto MLA = instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul,
@@ -3194,6 +3249,11 @@ AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC,
   case Intrinsic::aarch64_sve_sadalp:
   case Intrinsic::aarch64_sve_uadalp:
     return instCombineSVEPairwiseAddLong(IC, II);
+  case Intrinsic::aarch64_sve_smullb:
+  case Intrinsic::aarch64_sve_smullt:
+  case Intrinsic::aarch64_sve_umullb:
+  case Intrinsic::aarch64_sve_umullt:
+    return instCombineSVEMultiplyAccumulateLong(IC, II);
   case Intrinsic::aarch64_sve_sub:
     return instCombineSVEVectorSub(IC, II);
   case Intrinsic::aarch64_sve_sub_u:
diff --git a/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-mull-acc.ll b/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-mull-acc.ll
new file mode 100644
index 0000000000000..4cab4c2568ebe
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-mull-acc.ll
@@ -0,0 +1,110 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -passes=instcombine < %s | FileCheck %s
+
+target triple = "aarch64"
+
+; Tests for the instCombineSVEMultiplyAccumulateLong combine.
+; It is type-agnostic, so test for <vscale x 4 x i32> only.
+
+define <vscale x 4 x i32> @smlalb_from_add_smullb(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: @smlalb_from_add_smullb(
+; CHECK-NEXT:    [[R:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.smlalb.nxv4i32(<vscale x 4 x i32> [[ACC:%.*]], <vscale x 8 x i16> [[A:%.*]], <vscale x 8 x i16> [[B:%.*]])
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[R]]
+;
+  %mul = call <vscale x 4 x i32> @llvm.aarch64.sve.smullb.nxv4i32(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+  %r = add <vscale x 4 x i32> %mul, %acc
+  ret <vscale x 4 x i32> %r
+}
+
+; Check commutativity for mla combines
+define <vscale x 4 x i32> @smlalb_from_add_smullb_commuted(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: @smlalb_from_add_smullb_commuted(
+; CHECK-NEXT:    [[R:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.smlalb.nxv4i32(<vscale x 4 x i32> [[ACC:%.*]], <vscale x 8 x i16> [[A:%.*]], <vscale x 8 x i16> [[B:%.*]])
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[R]]
+;
+  %mul = call <vscale x 4 x i32> @llvm.aarch64.sve.smullb.nxv4i32(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+  %r = add <vscale x 4 x i32> %acc, %mul
+  ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @umlalb_from_add_umullb(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: @umlalb_from_add_umullb(
+; CHECK-NEXT:    [[R:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.umlalb.nxv4i32(<vscale x 4 x i32> [[ACC:%.*]], <vscale x 8 x i16> [[A:%.*]], <vscale x 8 x i16> [[B:%.*]])
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[R]]
+;
+  %mul = call <vscale x 4 x i32> @llvm.aarch64.sve.umullb.nxv4i32(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+  %r = add <vscale x 4 x i32> %acc, %mul
+  ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @smlslb_from_sub_smullb(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: @smlslb_from_sub_smullb(
+; CHECK-NEXT:    [[R:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.smlslb.nxv4i32(<vscale x 4 x i32> [[ACC:%.*]], <vscale x 8 x i16> [[A:%.*]], <vscale x 8 x i16> [[B:%.*]])
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[R]]
+;
+  %mul = call <vscale x 4 x i32> @llvm.aarch64.sve.smullb.nxv4i32(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+  %r = sub <vscale x 4 x i32> %acc, %mul
+  ret <vscale x 4 x i32> %r
+}
+
+; Check mls combines only work if the accumulator is the first operand in sub.
+define <vscale x 4 x i32> @no_smlslb_from_sub_smullb_commuted(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: @no_smlslb_from_sub_smullb_commuted(
+; CHECK-NEXT:    [[MUL:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.smullb.nxv4i32(<vscale x 8 x i16> [[A:%.*]], <vscale x 8 x i16> [[B:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = sub <vscale x 4 x i32> [[MUL]], [[ACC:%.*]]
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[R]]
+;
+  %mul = call <vscale x 4 x i32> @llvm.aarch64.sve.smullb.nxv4i32(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+  %r = sub <vscale x 4 x i32> %mul, %acc
+  ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @umlslb_from_sub_umullb(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: @umlslb_from_sub_umullb(
+; CHECK-NEXT:    [[R:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.umlslb.nxv4i32(<vscale x 4 x i32> [[ACC:%.*]], <vscale x 8 x i16> [[A:%.*]], <vscale x 8 x i16> [[B:%.*]])
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[R]]
+;
+  %mul = call <vscale x 4 x i32> @llvm.aarch64.sve.umullb.nxv4i32(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+  %r = sub <vscale x 4 x i32> %acc, %mul
+  ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @smlalt_from_add_smullt(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: @smlalt_from_add_smullt(
+; CHECK-NEXT:    [[R:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.smlalt.nxv4i32(<vscale x 4 x i32> [[ACC:%.*]], <vscale x 8 x i16> [[A:%.*]], <vscale x 8 x i16> [[B:%.*]])
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[R]]
+;
+  %mul = call <vscale x 4 x i32> @llvm.aarch64.sve.smullt.nxv4i32(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+  %r = add <vscale x 4 x i32> %mul, %acc
+  ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @umlalt_from_add_umullt(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: @umlalt_from_add_umullt(
+; CHECK-NEXT:    [[R:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.umlalt.nxv4i32(<vscale x 4 x i32> [[ACC:%.*]], <vscale x 8 x i16> [[A:%.*]], <vscale x 8 x i16> [[B:%.*]])
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[R]]
+;
+  %mul = call <vscale x 4 x i32> @llvm.aarch64.sve.umullt.nxv4i32(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+  %r = add <vscale x 4 x i32> %acc, %mul
+  ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @smlslt_from_sub_smullt(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: @smlslt_from_sub_smullt(
+; CHECK-NEXT:    [[R:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.smlslt.nxv4i32(<vscale x 4 x i32> [[ACC:%.*]], <vscale x 8 x i16> [[A:%.*]], <vscale x 8 x i16> [[B:%.*]])
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[R]]
+;
+  %mul = call <vscale x 4 x i32> @llvm.aarch64.sve.smullt.nxv4i32(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+  %r = sub <vscale x 4 x i32> %acc, %mul
+  ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @umlslt_from_sub_umullt(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: @umlslt_from_sub_umullt(
+; CHECK-NEXT:    [[R:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.umlslt.nxv4i32(<vscale x 4 x i32> [[ACC:%.*]], <vscale x 8 x i16> [[A:%.*]], <vscale x 8 x i16> [[B:%.*]])
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[R]]
+;
+  %mul = call <vscale x 4 x i32> @llvm.aarch64.sve.umullt.nxv4i32(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+  %r = sub <vscale x 4 x i32> %acc, %mul
+  ret <vscale x 4 x i32> %r
+}

``````````

</details>


https://github.com/llvm/llvm-project/pull/207415


More information about the llvm-commits mailing list