[llvm] [AArch64][InstCombine] Combine add/sub with mull intrinsics (PR #207415)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 3 08:33:27 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Gaƫtan Bossu (gbossu)
<details>
<summary>Changes</summary>
E.g. add(smullb(a, b), wide_acc) -> smlalb(wide_acc, a, b) where both smullb and smlalb are part of SVE2/SME.
---
Full diff: https://github.com/llvm/llvm-project/pull/207415.diff
2 Files Affected:
- (modified) llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp (+60)
- (added) llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-mull-acc.ll (+110)
``````````diff
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index edd95625e13ea..e38384fc171ad 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -2624,6 +2624,61 @@ instCombineSVEPairwiseAddLong(InstCombiner &IC, IntrinsicInst &II) {
return &II; // II is now trivially dead and will get erased.
}
+// Combine long mull and accumulate into fused mlal/lmsl
+// E.g. add(smullb(a, b), wide_acc) -> smlalb(wide_acc, a, b)
+static std::optional<Instruction *>
+instCombineSVEMultiplyAccumulateLong(InstCombiner &IC, IntrinsicInst &II) {
+ auto GetOpcodeForMLA = [](Intrinsic::ID IID) {
+ switch (IID) {
+ case Intrinsic::aarch64_sve_smullb:
+ return Intrinsic::aarch64_sve_smlalb;
+ case Intrinsic::aarch64_sve_smullt:
+ return Intrinsic::aarch64_sve_smlalt;
+ case Intrinsic::aarch64_sve_umullb:
+ return Intrinsic::aarch64_sve_umlalb;
+ case Intrinsic::aarch64_sve_umullt:
+ return Intrinsic::aarch64_sve_umlalt;
+ default:
+ llvm_unreachable("Expected SVE MULL intrinsic");
+ }
+ };
+ auto GetOpcodeForMLS = [](Intrinsic::ID IID) {
+ switch (IID) {
+ case Intrinsic::aarch64_sve_smullb:
+ return Intrinsic::aarch64_sve_smlslb;
+ case Intrinsic::aarch64_sve_smullt:
+ return Intrinsic::aarch64_sve_smlslt;
+ case Intrinsic::aarch64_sve_umullb:
+ return Intrinsic::aarch64_sve_umlslb;
+ case Intrinsic::aarch64_sve_umullt:
+ return Intrinsic::aarch64_sve_umlslt;
+ default:
+ llvm_unreachable("Expected SVE MULL intrinsic");
+ }
+ };
+
+ auto *User = dyn_cast_or_null<Instruction>(II.getUniqueUndroppableUser());
+ if (!User)
+ return std::nullopt;
+
+ Value *Acc;
+ Intrinsic::ID FuseIID;
+ if (match(User, m_c_Add(m_Specific(&II), m_Value(Acc))))
+ FuseIID = GetOpcodeForMLA(II.getIntrinsicID());
+ else if (match(User, m_Sub(m_Value(Acc), m_Specific(&II))))
+ FuseIID = GetOpcodeForMLS(II.getIntrinsicID());
+ else
+ return std::nullopt;
+
+ IC.Builder.SetInsertPoint(User);
+ Value *MulAcc = IC.Builder.CreateIntrinsic(
+ FuseIID, {II.getType()}, {Acc, II.getArgOperand(0), II.getArgOperand(1)});
+
+ IC.replaceInstUsesWith(*User, MulAcc);
+ IC.eraseInstFromFunction(*User);
+ return &II; // II is now trivially dead and will get erased.
+}
+
static std::optional<Instruction *> instCombineSVEVectorAdd(InstCombiner &IC,
IntrinsicInst &II) {
if (auto MLA = instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul,
@@ -3194,6 +3249,11 @@ AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC,
case Intrinsic::aarch64_sve_sadalp:
case Intrinsic::aarch64_sve_uadalp:
return instCombineSVEPairwiseAddLong(IC, II);
+ case Intrinsic::aarch64_sve_smullb:
+ case Intrinsic::aarch64_sve_smullt:
+ case Intrinsic::aarch64_sve_umullb:
+ case Intrinsic::aarch64_sve_umullt:
+ return instCombineSVEMultiplyAccumulateLong(IC, II);
case Intrinsic::aarch64_sve_sub:
return instCombineSVEVectorSub(IC, II);
case Intrinsic::aarch64_sve_sub_u:
diff --git a/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-mull-acc.ll b/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-mull-acc.ll
new file mode 100644
index 0000000000000..4cab4c2568ebe
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-mull-acc.ll
@@ -0,0 +1,110 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -passes=instcombine < %s | FileCheck %s
+
+target triple = "aarch64"
+
+; Tests for the instCombineSVEMultiplyAccumulateLong combine.
+; It is type-agnostic, so test for <vscale x 4 x i32> only.
+
+define <vscale x 4 x i32> @smlalb_from_add_smullb(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: @smlalb_from_add_smullb(
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.smlalb.nxv4i32(<vscale x 4 x i32> [[ACC:%.*]], <vscale x 8 x i16> [[A:%.*]], <vscale x 8 x i16> [[B:%.*]])
+; CHECK-NEXT: ret <vscale x 4 x i32> [[R]]
+;
+ %mul = call <vscale x 4 x i32> @llvm.aarch64.sve.smullb.nxv4i32(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+ %r = add <vscale x 4 x i32> %mul, %acc
+ ret <vscale x 4 x i32> %r
+}
+
+; Check commutativity for mla combines
+define <vscale x 4 x i32> @smlalb_from_add_smullb_commuted(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: @smlalb_from_add_smullb_commuted(
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.smlalb.nxv4i32(<vscale x 4 x i32> [[ACC:%.*]], <vscale x 8 x i16> [[A:%.*]], <vscale x 8 x i16> [[B:%.*]])
+; CHECK-NEXT: ret <vscale x 4 x i32> [[R]]
+;
+ %mul = call <vscale x 4 x i32> @llvm.aarch64.sve.smullb.nxv4i32(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+ %r = add <vscale x 4 x i32> %acc, %mul
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @umlalb_from_add_umullb(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: @umlalb_from_add_umullb(
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.umlalb.nxv4i32(<vscale x 4 x i32> [[ACC:%.*]], <vscale x 8 x i16> [[A:%.*]], <vscale x 8 x i16> [[B:%.*]])
+; CHECK-NEXT: ret <vscale x 4 x i32> [[R]]
+;
+ %mul = call <vscale x 4 x i32> @llvm.aarch64.sve.umullb.nxv4i32(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+ %r = add <vscale x 4 x i32> %acc, %mul
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @smlslb_from_sub_smullb(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: @smlslb_from_sub_smullb(
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.smlslb.nxv4i32(<vscale x 4 x i32> [[ACC:%.*]], <vscale x 8 x i16> [[A:%.*]], <vscale x 8 x i16> [[B:%.*]])
+; CHECK-NEXT: ret <vscale x 4 x i32> [[R]]
+;
+ %mul = call <vscale x 4 x i32> @llvm.aarch64.sve.smullb.nxv4i32(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+ %r = sub <vscale x 4 x i32> %acc, %mul
+ ret <vscale x 4 x i32> %r
+}
+
+; Check mls combines only work if the accumulator is the first operand in sub.
+define <vscale x 4 x i32> @no_smlslb_from_sub_smullb_commuted(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: @no_smlslb_from_sub_smullb_commuted(
+; CHECK-NEXT: [[MUL:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.smullb.nxv4i32(<vscale x 8 x i16> [[A:%.*]], <vscale x 8 x i16> [[B:%.*]])
+; CHECK-NEXT: [[R:%.*]] = sub <vscale x 4 x i32> [[MUL]], [[ACC:%.*]]
+; CHECK-NEXT: ret <vscale x 4 x i32> [[R]]
+;
+ %mul = call <vscale x 4 x i32> @llvm.aarch64.sve.smullb.nxv4i32(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+ %r = sub <vscale x 4 x i32> %mul, %acc
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @umlslb_from_sub_umullb(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: @umlslb_from_sub_umullb(
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.umlslb.nxv4i32(<vscale x 4 x i32> [[ACC:%.*]], <vscale x 8 x i16> [[A:%.*]], <vscale x 8 x i16> [[B:%.*]])
+; CHECK-NEXT: ret <vscale x 4 x i32> [[R]]
+;
+ %mul = call <vscale x 4 x i32> @llvm.aarch64.sve.umullb.nxv4i32(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+ %r = sub <vscale x 4 x i32> %acc, %mul
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @smlalt_from_add_smullt(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: @smlalt_from_add_smullt(
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.smlalt.nxv4i32(<vscale x 4 x i32> [[ACC:%.*]], <vscale x 8 x i16> [[A:%.*]], <vscale x 8 x i16> [[B:%.*]])
+; CHECK-NEXT: ret <vscale x 4 x i32> [[R]]
+;
+ %mul = call <vscale x 4 x i32> @llvm.aarch64.sve.smullt.nxv4i32(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+ %r = add <vscale x 4 x i32> %mul, %acc
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @umlalt_from_add_umullt(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: @umlalt_from_add_umullt(
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.umlalt.nxv4i32(<vscale x 4 x i32> [[ACC:%.*]], <vscale x 8 x i16> [[A:%.*]], <vscale x 8 x i16> [[B:%.*]])
+; CHECK-NEXT: ret <vscale x 4 x i32> [[R]]
+;
+ %mul = call <vscale x 4 x i32> @llvm.aarch64.sve.umullt.nxv4i32(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+ %r = add <vscale x 4 x i32> %acc, %mul
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @smlslt_from_sub_smullt(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: @smlslt_from_sub_smullt(
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.smlslt.nxv4i32(<vscale x 4 x i32> [[ACC:%.*]], <vscale x 8 x i16> [[A:%.*]], <vscale x 8 x i16> [[B:%.*]])
+; CHECK-NEXT: ret <vscale x 4 x i32> [[R]]
+;
+ %mul = call <vscale x 4 x i32> @llvm.aarch64.sve.smullt.nxv4i32(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+ %r = sub <vscale x 4 x i32> %acc, %mul
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @umlslt_from_sub_umullt(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: @umlslt_from_sub_umullt(
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.umlslt.nxv4i32(<vscale x 4 x i32> [[ACC:%.*]], <vscale x 8 x i16> [[A:%.*]], <vscale x 8 x i16> [[B:%.*]])
+; CHECK-NEXT: ret <vscale x 4 x i32> [[R]]
+;
+ %mul = call <vscale x 4 x i32> @llvm.aarch64.sve.umullt.nxv4i32(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+ %r = sub <vscale x 4 x i32> %acc, %mul
+ ret <vscale x 4 x i32> %r
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/207415
More information about the llvm-commits
mailing list