[llvm] [GlobalISel] TableGen memcpy-like prelegalizer combines (PR #203235)
Cullen Rhodes via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 11 03:17:34 PDT 2026
https://github.com/c-rhodes created https://github.com/llvm/llvm-project/pull/203235
This removes the corresponding handwritten C++ combine handling from the AArch64 prelegalizer combiners.
Assisted-by: codex
>From 672396b1ede9284d5da5fc96530add53387ce1a2 Mon Sep 17 00:00:00 2001
From: Cullen Rhodes <cullen.rhodes at arm.com>
Date: Mon, 8 Jun 2026 06:00:10 +0000
Subject: [PATCH] [GlobalISel] TableGen memcpy-like prelegalizer combines
This removes the corresponding handwritten C++ combine handling from the
AArch64 prelegalizer combiners.
Assisted-by: codex
---
.../llvm/CodeGen/GlobalISel/CombinerHelper.h | 12 +++----
.../include/llvm/Target/GlobalISel/Combine.td | 33 +++++++++++++++++++
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 19 +++++++++--
llvm/lib/Target/AArch64/AArch64Combine.td | 16 +++++++--
.../AArch64/GISel/AArch64GlobalISelUtils.cpp | 30 +++++++++--------
.../AArch64/GISel/AArch64GlobalISelUtils.h | 9 +++--
.../GISel/AArch64O0PreLegalizerCombiner.cpp | 19 -----------
.../GISel/AArch64PreLegalizerCombiner.cpp | 20 -----------
.../Target/Mips/MipsPreLegalizerCombiner.cpp | 2 +-
9 files changed, 93 insertions(+), 67 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index aa61310994a67..878cf28ae239e 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -358,6 +358,12 @@ class CombinerHelper {
/// $whatever = COPY $addr
LLVM_ABI bool tryCombineMemCpyFamily(MachineInstr &MI,
unsigned MaxLen = 0) const;
+ LLVM_ABI bool matchCombineMemCpyFamily(MachineInstr &MI,
+ MemCpyFamilyLoweringInfo &MatchInfo,
+ unsigned MaxLen = 0) const;
+ LLVM_ABI void
+ applyCombineMemCpyFamily(MachineInstr &MI,
+ MemCpyFamilyLoweringInfo &MatchInfo) const;
LLVM_ABI bool matchPtrAddImmedChain(MachineInstr &MI,
PtrAddChain &MatchInfo) const;
@@ -860,12 +866,6 @@ class CombinerHelper {
/// combine functions. Returns true if changed.
LLVM_ABI bool tryCombine(MachineInstr &MI) const;
- /// Emit loads and stores that perform the given memcpy.
- /// Assumes \p MI is a G_MEMCPY_INLINE
- /// TODO: implement dynamically sized inline memcpy,
- /// and rename: s/bool tryEmit/void emit/
- LLVM_ABI bool tryEmitMemcpyInline(MachineInstr &MI) const;
-
/// Match:
/// (G_UMULO x, 2) -> (G_UADDO x, x)
/// (G_SMULO x, 2) -> (G_SADDO x, x)
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index f3474eb95c436..a6aa19712845f 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -296,6 +296,39 @@ def combine_indexed_load_store : GICombineRule<
[{ return Helper.matchCombineIndexedLoadStore(*${root}, ${matchinfo}); }]),
(apply [{ Helper.applyCombineIndexedLoadStore(*${root}, ${matchinfo}); }])>;
+def memcpy_family_matchinfo : GIDefMatchData<"MemCpyFamilyLoweringInfo">;
+def combine_memcpy_inline : GICombineRule<
+ (defs root:$root, memcpy_family_matchinfo:$matchinfo),
+ (match (G_MEMCPY_INLINE $dst_addr, $src_addr, $size):$root,
+ [{ return Helper.matchCombineMemCpyFamily(*${root}, ${matchinfo}); }]),
+ (apply [{ Helper.applyCombineMemCpyFamily(*${root}, ${matchinfo}); }])>;
+
+def combine_memcpy : GICombineRule<
+ (defs root:$root, memcpy_family_matchinfo:$matchinfo),
+ (match (G_MEMCPY $dst_addr, $src_addr, $size, $tailcall):$root,
+ [{ return Helper.matchCombineMemCpyFamily(*${root}, ${matchinfo},
+ CInfo.EnableOpt ? 0 : 32); }]),
+ (apply [{ Helper.applyCombineMemCpyFamily(*${root}, ${matchinfo}); }])>;
+
+def combine_memmove : GICombineRule<
+ (defs root:$root, memcpy_family_matchinfo:$matchinfo),
+ (match (G_MEMMOVE $dst_addr, $src_addr, $size, $tailcall):$root,
+ [{ return Helper.matchCombineMemCpyFamily(*${root}, ${matchinfo},
+ CInfo.EnableOpt ? 0 : 32); }]),
+ (apply [{ Helper.applyCombineMemCpyFamily(*${root}, ${matchinfo}); }])>;
+
+def combine_memset : GICombineRule<
+ (defs root:$root, memcpy_family_matchinfo:$matchinfo),
+ (match (G_MEMSET $dst_addr, $value, $size, $tailcall):$root,
+ [{ return Helper.matchCombineMemCpyFamily(*${root}, ${matchinfo},
+ CInfo.EnableOpt ? 0 : 32); }]),
+ (apply [{ Helper.applyCombineMemCpyFamily(*${root}, ${matchinfo}); }])>;
+
+def memcpy_family_combines : GICombineGroup<[combine_memcpy_inline,
+ combine_memcpy,
+ combine_memmove,
+ combine_memset]>;
+
def opt_brcond_by_inverting_cond_matchdata : GIDefMatchData<"MachineInstr *">;
def opt_brcond_by_inverting_cond : GICombineRule<
(defs root:$root, opt_brcond_by_inverting_cond_matchdata:$matchinfo),
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 88b68d7685c63..393792809e53b 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -1721,12 +1721,25 @@ void CombinerHelper::applyOptBrCondByInvertingCond(
Observer.changedInstr(*BrCond);
}
-bool CombinerHelper::tryEmitMemcpyInline(MachineInstr &MI) const {
+bool CombinerHelper::matchCombineMemCpyFamily(
+ MachineInstr &MI, MemCpyFamilyLoweringInfo &MatchInfo,
+ unsigned MaxLen) const {
+ auto &[Dst, Src, KnownLen, Alignment, DstAlignCanChange, MemOps] = MatchInfo;
+ return canLowerMemCpyFamily(MI, MRI, MaxLen, Dst, Src, KnownLen, Alignment,
+ DstAlignCanChange, MemOps);
+}
+
+void CombinerHelper::applyCombineMemCpyFamily(
+ MachineInstr &MI, MemCpyFamilyLoweringInfo &MatchInfo) const {
+ auto &[Dst, Src, KnownLen, Alignment, DstAlignCanChange, MemOps] = MatchInfo;
MachineIRBuilder HelperBuilder(MI);
GISelObserverWrapper DummyObserver;
LegalizerHelper Helper(HelperBuilder.getMF(), DummyObserver, HelperBuilder);
- return Helper.lowerMemCpyFamily(MI) ==
- LegalizerHelper::LegalizeResult::Legalized;
+ bool Changed = Helper.lowerMemCpyFamily(MI, Dst, Src, KnownLen, Alignment,
+ DstAlignCanChange, MemOps) ==
+ LegalizerHelper::LegalizeResult::Legalized;
+ assert(Changed && "expected memcpy-family instruction to lower");
+ (void)Changed;
}
bool CombinerHelper::tryCombineMemCpyFamily(MachineInstr &MI,
diff --git a/llvm/lib/Target/AArch64/AArch64Combine.td b/llvm/lib/Target/AArch64/AArch64Combine.td
index a9c447336cd5e..82b9cec960d89 100644
--- a/llvm/lib/Target/AArch64/AArch64Combine.td
+++ b/llvm/lib/Target/AArch64/AArch64Combine.td
@@ -68,6 +68,14 @@ def simplify_uaddo : GICombineRule<
(apply [{ applySimplifyUADDO(*${root}, MRI, B, Observer, Helper,
${matchinfo}); }])>;
+def emit_bzero : GICombineRule<
+ (defs root:$root),
+ (match (G_MEMSET $dst_addr, $value, $size, $tailcall):$root,
+ [{ return llvm::AArch64GISelUtils::matchEmitBZero(*${root}, MRI,
+ Libcalls,
+ CInfo.EnableMinSize); }]),
+ (apply [{ llvm::AArch64GISelUtils::applyEmitBZero(*${root}, B); }])>;
+
def AArch64PreLegalizerCombiner: GICombiner<
"AArch64PreLegalizerCombinerImpl", [all_combines,
select_zero_true,
@@ -82,13 +90,17 @@ def AArch64PreLegalizerCombiner: GICombiner<
push_sub_through_sext,
push_add_through_sext,
push_mul_through_sext,
- simplify_uaddo]> {
+ simplify_uaddo,
+ memcpy_family_combines,
+ emit_bzero]> {
let CombineAllMethodName = "tryCombineAllImpl";
}
def AArch64O0PreLegalizerCombiner: GICombiner<
"AArch64O0PreLegalizerCombinerImpl", [optnone_combines,
- combine_shuffle_vector]> {
+ combine_shuffle_vector,
+ memcpy_family_combines,
+ emit_bzero]> {
let CombineAllMethodName = "tryCombineAllImpl";
}
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.cpp b/llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.cpp
index 74cb5e9bb0729..d0ef3ae72b02d 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.cpp
@@ -60,15 +60,27 @@ bool AArch64GISelUtils::isCMN(const MachineInstr *MaybeSub,
return MaybeZero && MaybeZero->Value.getZExtValue() == 0;
}
-bool AArch64GISelUtils::tryEmitBZero(MachineInstr &MI,
- MachineIRBuilder &MIRBuilder,
- const LibcallLoweringInfo &Libcalls,
- bool MinSize) {
+void AArch64GISelUtils::applyEmitBZero(MachineInstr &MI,
+ MachineIRBuilder &MIRBuilder) {
+ assert(MI.getOpcode() == TargetOpcode::G_MEMSET);
+
+ MIRBuilder.setInstrAndDebugLoc(MI);
+ MIRBuilder
+ .buildInstr(TargetOpcode::G_BZERO, {},
+ {MI.getOperand(0), MI.getOperand(2)})
+ .addImm(MI.getOperand(3).getImm())
+ .addMemOperand(*MI.memoperands_begin());
+ MI.eraseFromParent();
+}
+
+bool AArch64GISelUtils::matchEmitBZero(const MachineInstr &MI,
+ const MachineRegisterInfo &MRI,
+ const LibcallLoweringInfo &Libcalls,
+ bool MinSize) {
assert(MI.getOpcode() == TargetOpcode::G_MEMSET);
if (Libcalls.getLibcallImpl(RTLIB::BZERO) == RTLIB::Unsupported)
return false;
- MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
auto Zero =
getIConstantVRegValWithLookThrough(MI.getOperand(1).getReg(), MRI);
if (!Zero || Zero->Value.getSExtValue() != 0)
@@ -86,14 +98,6 @@ bool AArch64GISelUtils::tryEmitBZero(MachineInstr &MI,
return false;
}
}
-
- MIRBuilder.setInstrAndDebugLoc(MI);
- MIRBuilder
- .buildInstr(TargetOpcode::G_BZERO, {},
- {MI.getOperand(0), MI.getOperand(2)})
- .addImm(MI.getOperand(3).getImm())
- .addMemOperand(*MI.memoperands_begin());
- MI.eraseFromParent();
return true;
}
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.h b/llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.h
index 1d28f9f0d47e7..973cf9d07bd91 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.h
+++ b/llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.h
@@ -50,9 +50,12 @@ bool isCMN(const MachineInstr *MaybeSub, const CmpInst::Predicate &Pred,
///
/// \note This only applies on Darwin.
///
-/// \returns true if \p MI was replaced with a G_BZERO.
-bool tryEmitBZero(MachineInstr &MI, MachineIRBuilder &MIRBuilder,
- const LibcallLoweringInfo &Libcalls, bool MinSize);
+/// \returns true if \p MI can be replaced with a G_BZERO.
+bool matchEmitBZero(const MachineInstr &MI, const MachineRegisterInfo &MRI,
+ const LibcallLoweringInfo &Libcalls, bool MinSize);
+///
+/// Replace \p MI with a G_BZERO.
+void applyEmitBZero(MachineInstr &MI, MachineIRBuilder &MIRBuilder);
/// Analyze a ptrauth discriminator value to try to find the constant integer
/// and address parts, cracking a ptrauth_blend intrinsic if there is one.
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64O0PreLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64O0PreLegalizerCombiner.cpp
index 613cf24ed0aa6..74e8c682df482 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64O0PreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64O0PreLegalizerCombiner.cpp
@@ -92,25 +92,6 @@ bool AArch64O0PreLegalizerCombinerImpl::tryCombineAll(MachineInstr &MI) const {
if (tryCombineAllImpl(MI))
return true;
- unsigned Opc = MI.getOpcode();
- switch (Opc) {
- case TargetOpcode::G_MEMCPY_INLINE:
- return Helper.tryEmitMemcpyInline(MI);
- case TargetOpcode::G_MEMCPY:
- case TargetOpcode::G_MEMMOVE:
- case TargetOpcode::G_MEMSET: {
- // At -O0 set a maxlen of 32 to inline;
- unsigned MaxLen = 32;
- // Try to inline memcpy type calls if optimizations are enabled.
- if (Helper.tryCombineMemCpyFamily(MI, MaxLen))
- return true;
- if (Opc == TargetOpcode::G_MEMSET)
- return llvm::AArch64GISelUtils::tryEmitBZero(MI, B, Libcalls,
- CInfo.EnableMinSize);
- return false;
- }
- }
-
return false;
}
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
index d8aae8bf420aa..91c75a06c84cd 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
@@ -789,26 +789,6 @@ bool AArch64PreLegalizerCombinerImpl::tryCombineAll(MachineInstr &MI) const {
if (tryCombineAllImpl(MI))
return true;
- unsigned Opc = MI.getOpcode();
- switch (Opc) {
- case TargetOpcode::G_MEMCPY_INLINE:
- return Helper.tryEmitMemcpyInline(MI);
- case TargetOpcode::G_MEMCPY:
- case TargetOpcode::G_MEMMOVE:
- case TargetOpcode::G_MEMSET: {
- // If we're at -O0 set a maxlen of 32 to inline, otherwise let the other
- // heuristics decide.
- unsigned MaxLen = CInfo.EnableOpt ? 0 : 32;
- // Try to inline memcpy type calls if optimizations are enabled.
- if (Helper.tryCombineMemCpyFamily(MI, MaxLen))
- return true;
- if (Opc == TargetOpcode::G_MEMSET)
- return llvm::AArch64GISelUtils::tryEmitBZero(MI, B, Libcalls,
- CInfo.EnableMinSize);
- return false;
- }
- }
-
return false;
}
diff --git a/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp b/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
index 5ada356de7e44..2d488ddcecfa0 100644
--- a/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
@@ -58,7 +58,7 @@ class MipsPreLegalizerCombinerImpl : public Combiner {
default:
return false;
case TargetOpcode::G_MEMCPY_INLINE:
- return Helper.tryEmitMemcpyInline(MI);
+ return Helper.tryCombineMemCpyFamily(MI);
case TargetOpcode::G_LOAD:
case TargetOpcode::G_SEXTLOAD:
case TargetOpcode::G_ZEXTLOAD: {
More information about the llvm-commits
mailing list