[llvm] [GlobalISel] Implement `llvm.memset.inline` (PR #203198)
Ömer Sinan Ağacan via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 12 05:00:43 PDT 2026
https://github.com/osa1 updated https://github.com/llvm/llvm-project/pull/203198
>From 303949d0a02ddfd66e17776d4551c672faebe8e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= <omeragacan at gmail.com>
Date: Wed, 10 Jun 2026 17:09:36 +0100
Subject: [PATCH 01/11] Implement G_MEMSET_INLINE, legalize in AArch64
---
.../llvm/CodeGen/GlobalISel/CombinerHelper.h | 4 +++
.../llvm/CodeGen/GlobalISel/LegalizerHelper.h | 2 +-
llvm/include/llvm/Support/TargetOpcodes.def | 3 +++
llvm/include/llvm/Target/GenericOpcodes.td | 7 ++++++
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 8 ++++++
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 5 +++-
.../CodeGen/GlobalISel/LegalizerHelper.cpp | 25 ++++++++++++-------
.../AArch64/GISel/AArch64LegalizerInfo.cpp | 3 +++
.../GISel/AArch64PreLegalizerCombiner.cpp | 2 ++
9 files changed, 48 insertions(+), 11 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index aa61310994a67..f744a61285c4a 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -866,6 +866,10 @@ class CombinerHelper {
/// and rename: s/bool tryEmit/void emit/
LLVM_ABI bool tryEmitMemcpyInline(MachineInstr &MI) const;
+ /// Emit stores that perform the given memset.
+ /// Assumes \p MI is a G_MEMSET_INLINE
+ LLVM_ABI bool tryEmitMemsetInline(MachineInstr &MI) const;
+
/// Match:
/// (G_UMULO x, 2) -> (G_UADDO x, x)
/// (G_SMULO x, 2) -> (G_SADDO x, x)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index 47d7da7a0eb52..e33b534067229 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -304,7 +304,7 @@ class LegalizerHelper {
// Memcpy family legalization helpers.
LegalizeResult lowerMemset(MachineInstr &MI, Register Dst, Register Val,
- uint64_t KnownLen, Align Alignment,
+ uint64_t KnownLen, uint64_t Limit, Align Alignment,
bool IsVolatile);
LegalizeResult lowerMemcpy(MachineInstr &MI, Register Dst, Register Src,
uint64_t KnownLen, uint64_t Limit, Align DstAlign,
diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def
index 19b48cb9765ff..051cea764e5a0 100644
--- a/llvm/include/llvm/Support/TargetOpcodes.def
+++ b/llvm/include/llvm/Support/TargetOpcodes.def
@@ -959,6 +959,9 @@ HANDLE_TARGET_OPCODE(G_MEMMOVE)
HANDLE_TARGET_OPCODE(G_MEMSET)
HANDLE_TARGET_OPCODE(G_BZERO)
+/// llvm.memset.inline intrinsic
+HANDLE_TARGET_OPCODE(G_MEMSET_INLINE)
+
/// llvm.trap, llvm.debugtrap and llvm.ubsantrap intrinsics
HANDLE_TARGET_OPCODE(G_TRAP)
HANDLE_TARGET_OPCODE(G_DEBUGTRAP)
diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index b0edb462f0f9d..e22813021b409 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -1863,6 +1863,13 @@ def G_BZERO : GenericInstruction {
let mayStore = true;
}
+def G_MEMSET_INLINE : GenericInstruction {
+ let OutOperandList = (outs);
+ let InOperandList = (ins ptype0:$dst_addr, type1:$value, type2:$size, untyped_imm_0:$tailcall);
+ let hasSideEffects = false;
+ let mayStore = true;
+}
+
//------------------------------------------------------------------------------
// Trap intrinsics
//------------------------------------------------------------------------------
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 88b68d7685c63..f7159ad0617b6 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -1729,6 +1729,14 @@ bool CombinerHelper::tryEmitMemcpyInline(MachineInstr &MI) const {
LegalizerHelper::LegalizeResult::Legalized;
}
+bool CombinerHelper::tryEmitMemsetInline(MachineInstr &MI) const {
+ MachineIRBuilder HelperBuilder(MI);
+ GISelObserverWrapper DummyObserver;
+ LegalizerHelper Helper(HelperBuilder.getMF(), DummyObserver, HelperBuilder);
+ return Helper.lowerMemCpyFamily(MI) ==
+ LegalizerHelper::LegalizeResult::Legalized;
+}
+
bool CombinerHelper::tryCombineMemCpyFamily(MachineInstr &MI,
unsigned MaxLen) const {
MachineIRBuilder HelperBuilder(MI);
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 28755da8327aa..7e4e209c727e5 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1835,7 +1835,8 @@ bool IRTranslator::translateMemFunc(const CallInst &CI,
ICall.addMemOperand(
MF->getMachineMemOperand(MachinePointerInfo(CI.getArgOperand(0)),
StoreFlags, 1, DstAlign, AAInfo));
- if (Opcode != TargetOpcode::G_MEMSET)
+ if (Opcode != TargetOpcode::G_MEMSET &&
+ Opcode != TargetOpcode::G_MEMSET_INLINE)
ICall.addMemOperand(MF->getMachineMemOperand(
MachinePointerInfo(SrcPtr), LoadFlags, 1, SrcAlign, AAInfo));
@@ -2445,6 +2446,8 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMMOVE);
case Intrinsic::memset:
return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMSET);
+ case Intrinsic::memset_inline:
+ return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMSET_INLINE);
case Intrinsic::eh_typeid_for: {
GlobalValue *GV = ExtractTypeInfo(CI.getArgOperand(0));
Register Reg = getOrCreateVReg(CI);
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index afe93f617a119..8dc6e637334cb 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -10811,7 +10811,7 @@ static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB) {
LegalizerHelper::LegalizeResult
LegalizerHelper::lowerMemset(MachineInstr &MI, Register Dst, Register Val,
- uint64_t KnownLen, Align Alignment,
+ uint64_t KnownLen, uint64_t Limit, Align Alignment,
bool IsVolatile) {
auto &MF = *MI.getParent()->getParent();
const auto &TLI = *MF.getSubtarget().getTargetLowering();
@@ -10822,13 +10822,11 @@ LegalizerHelper::lowerMemset(MachineInstr &MI, Register Dst, Register Val,
bool DstAlignCanChange = false;
MachineFrameInfo &MFI = MF.getFrameInfo();
- bool OptSize = shouldLowerMemFuncForSize(MF);
MachineInstr *FIDef = getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Dst, MRI);
if (FIDef && !MFI.isFixedObjectIndex(FIDef->getOperand(1).getIndex()))
DstAlignCanChange = true;
- unsigned Limit = TLI.getMaxStoresPerMemset(OptSize);
std::vector<LLT> MemOps;
const auto &DstMMO = **MI.memoperands_begin();
@@ -11143,7 +11141,8 @@ LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen) {
// matcher function.
assert((Opc == TargetOpcode::G_MEMCPY ||
Opc == TargetOpcode::G_MEMCPY_INLINE ||
- Opc == TargetOpcode::G_MEMMOVE || Opc == TargetOpcode::G_MEMSET) &&
+ Opc == TargetOpcode::G_MEMMOVE || Opc == TargetOpcode::G_MEMSET ||
+ Opc == TargetOpcode::G_MEMSET_INLINE) &&
"Expected memcpy like instruction");
auto MMOIt = MI.memoperands_begin();
@@ -11153,7 +11152,7 @@ LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen) {
Align SrcAlign;
auto [Dst, Src, Len] = MI.getFirst3Regs();
- if (Opc != TargetOpcode::G_MEMSET) {
+ if (Opc != TargetOpcode::G_MEMSET && Opc != TargetOpcode::G_MEMSET_INLINE) {
assert(MMOIt != MI.memoperands_end() && "Expected a second MMO on MI");
MemOp = *(++MMOIt);
SrcAlign = MemOp->getBaseAlign();
@@ -11162,8 +11161,9 @@ LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen) {
// See if this is a constant length copy
auto LenVRegAndVal = getIConstantVRegValWithLookThrough(Len, MRI);
if (!LenVRegAndVal) {
- // FIXME: support dynamically sized G_MEMCPY_INLINE
+ // FIXME: support dynamically sized G_MEMCPY_INLINE and G_MEMSET_INLINE
assert(Opc != TargetOpcode::G_MEMCPY_INLINE &&
+ Opc != TargetOpcode::G_MEMSET_INLINE &&
"inline memcpy with dynamic size is not yet supported");
return UnableToLegalize;
}
@@ -11174,7 +11174,8 @@ LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen) {
return Legalized;
}
- if (Opc != TargetOpcode::G_MEMCPY_INLINE && MaxLen && KnownLen > MaxLen)
+ if (Opc != TargetOpcode::G_MEMCPY_INLINE &&
+ Opc != TargetOpcode::G_MEMSET_INLINE && MaxLen && KnownLen > MaxLen)
return UnableToLegalize;
bool IsVolatile = MemOp->isVolatile();
@@ -11190,7 +11191,13 @@ LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen) {
}
if (Opc == TargetOpcode::G_MEMMOVE)
return lowerMemmove(MI, Dst, Src, KnownLen, DstAlign, SrcAlign, IsVolatile);
- if (Opc == TargetOpcode::G_MEMSET)
- return lowerMemset(MI, Dst, Src, KnownLen, DstAlign, IsVolatile);
+ if (Opc == TargetOpcode::G_MEMSET || Opc == TargetOpcode::G_MEMSET_INLINE) {
+ auto &MF = *MI.getParent()->getParent();
+ bool OptSize = shouldLowerMemFuncForSize(MF);
+ uint64_t Limit = Opc == TargetOpcode::G_MEMSET_INLINE
+ ? std::numeric_limits<uint64_t>::max()
+ : TLI.getMaxStoresPerMemset(OptSize);
+ return lowerMemset(MI, Dst, Src, KnownLen, Limit, DstAlign, IsVolatile);
+ }
return UnableToLegalize;
}
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index c0756ab66cb49..eca93c51760ac 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -1376,6 +1376,9 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
getActionDefinitionsBuilder(G_MEMCPY_INLINE)
.legalForCartesianProduct({p0}, {p0}, {s64});
+ getActionDefinitionsBuilder(G_MEMSET_INLINE)
+ .legalForCartesianProduct({p0}, {s64}, {s64})
+ .customForCartesianProduct({p0}, {s8}, {s64});
} else {
getActionDefinitionsBuilder({G_BZERO, G_MEMCPY, G_MEMMOVE, G_MEMSET})
.libcall();
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
index d8aae8bf420aa..cf4c1546efdfd 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
@@ -793,6 +793,8 @@ bool AArch64PreLegalizerCombinerImpl::tryCombineAll(MachineInstr &MI) const {
switch (Opc) {
case TargetOpcode::G_MEMCPY_INLINE:
return Helper.tryEmitMemcpyInline(MI);
+ case TargetOpcode::G_MEMSET_INLINE:
+ return Helper.tryEmitMemsetInline(MI);
case TargetOpcode::G_MEMCPY:
case TargetOpcode::G_MEMMOVE:
case TargetOpcode::G_MEMSET: {
>From b8b9d8c33d33583a83bea48e12fdd30f636d9a31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= <omeragacan at gmail.com>
Date: Wed, 10 Jun 2026 17:22:01 +0100
Subject: [PATCH 02/11] Add more handling around existing G_MEMCPY_INLINE code
---
llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h | 2 ++
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 3 ++-
llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp | 1 +
llvm/lib/CodeGen/MachineVerifier.cpp | 4 +++-
llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp | 2 ++
.../Target/AArch64/GISel/AArch64O0PreLegalizerCombiner.cpp | 2 ++
llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 3 ++-
llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp | 2 ++
llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp | 2 ++
llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp | 1 +
llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp | 2 +-
11 files changed, 20 insertions(+), 4 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index f744a61285c4a..da127baeff08c 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -868,6 +868,8 @@ class CombinerHelper {
/// Emit stores that perform the given memset.
/// Assumes \p MI is a G_MEMSET_INLINE
+ /// TODO: implement dynamically sized inline memset,
+ /// and rename: s/bool tryEmit/void emit/
LLVM_ABI bool tryEmitMemsetInline(MachineInstr &MI) const;
/// Match:
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 7e4e209c727e5..4547b0c41cb96 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1805,7 +1805,8 @@ bool IRTranslator::translateMemFunc(const CallInst &CI,
DstAlign = MSI->getDestAlign().valueOrOne();
}
- if (Opcode != TargetOpcode::G_MEMCPY_INLINE) {
+ if (Opcode != TargetOpcode::G_MEMCPY_INLINE &&
+ Opcode != TargetOpcode::G_MEMSET_INLINE) {
// We need to propagate the tail call flag from the IR inst as an argument.
// Otherwise, we have to pessimize and assume later that we cannot tail call
// any memory intrinsics.
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 8dc6e637334cb..4dc2fcb951c8a 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -4947,6 +4947,7 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) {
case G_MEMCPY:
case G_MEMMOVE:
case G_MEMCPY_INLINE:
+ case G_MEMSET_INLINE:
return lowerMemCpyFamily(MI);
case G_ZEXT:
case G_SEXT:
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index c611b837a86b4..5c68851c2e122 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -2128,7 +2128,9 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
break;
}
case TargetOpcode::G_BZERO:
- case TargetOpcode::G_MEMSET: {
+ case TargetOpcode::G_MEMSET:
+ case TargetOpcode::G_MEMSET_INLINE:
+ {
ArrayRef<MachineMemOperand *> MMOs = MI->memoperands();
std::string Name = Opc == TargetOpcode::G_MEMSET ? "memset" : "bzero";
if (MMOs.size() != 1) {
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index cf650fd5c4e72..4f4c999ab244d 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -3603,6 +3603,7 @@ bool AArch64InstructionSelector::select(MachineInstr &I) {
case TargetOpcode::G_MEMCPY_INLINE:
case TargetOpcode::G_MEMMOVE:
case TargetOpcode::G_MEMSET:
+ case TargetOpcode::G_MEMSET_INLINE:
assert(STI.hasMOPS() && "Shouldn't get here without +mops feature");
return selectMOPS(I, MRI);
}
@@ -3629,6 +3630,7 @@ bool AArch64InstructionSelector::selectMOPS(MachineInstr &GI,
Mopcode = AArch64::MOPSMemoryMovePseudo;
break;
case TargetOpcode::G_MEMSET:
+ case TargetOpcode::G_MEMSET_INLINE:
// For tagged memset see llvm.aarch64.mops.memset.tag
Mopcode = AArch64::MOPSMemorySetPseudo;
break;
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64O0PreLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64O0PreLegalizerCombiner.cpp
index 613cf24ed0aa6..3bca9053ca429 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64O0PreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64O0PreLegalizerCombiner.cpp
@@ -96,6 +96,8 @@ bool AArch64O0PreLegalizerCombinerImpl::tryCombineAll(MachineInstr &MI) const {
switch (Opc) {
case TargetOpcode::G_MEMCPY_INLINE:
return Helper.tryEmitMemcpyInline(MI);
+ case TargetOpcode::G_MEMSET_INLINE:
+ return Helper.tryEmitMemsetInline(MI);
case TargetOpcode::G_MEMCPY:
case TargetOpcode::G_MEMMOVE:
case TargetOpcode::G_MEMSET: {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index e341392ae068a..84ed533863fd9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -2238,7 +2238,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
.lower();
}
- getActionDefinitionsBuilder({G_MEMCPY, G_MEMCPY_INLINE, G_MEMMOVE, G_MEMSET})
+ getActionDefinitionsBuilder(
+ {G_MEMCPY, G_MEMCPY_INLINE, G_MEMMOVE, G_MEMSET, G_MEMSET_INLINE})
.lower();
getActionDefinitionsBuilder({G_TRAP, G_DEBUGTRAP}).custom();
diff --git a/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp b/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
index 5ada356de7e44..43cf18517026b 100644
--- a/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
@@ -59,6 +59,8 @@ class MipsPreLegalizerCombinerImpl : public Combiner {
return false;
case TargetOpcode::G_MEMCPY_INLINE:
return Helper.tryEmitMemcpyInline(MI);
+ case TargetOpcode::G_MEMSET_INLINE:
+ return Helper.tryEmitMemsetInline(MI);
case TargetOpcode::G_LOAD:
case TargetOpcode::G_SEXTLOAD:
case TargetOpcode::G_ZEXTLOAD: {
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index 716fab95eb943..305f977d14914 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -529,6 +529,8 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
getActionDefinitionsBuilder(G_MEMCPY_INLINE).lower();
+ getActionDefinitionsBuilder(G_MEMSET_INLINE).lower();
+
getActionDefinitionsBuilder({G_DYN_STACKALLOC, G_STACKSAVE, G_STACKRESTORE})
.lower();
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 79577505157fd..e87cb961a83a8 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -1042,6 +1042,7 @@ bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
case TargetOpcode::G_MEMCPY:
case TargetOpcode::G_MEMCPY_INLINE:
case TargetOpcode::G_MEMSET:
+ case TargetOpcode::G_MEMSET_INLINE:
return selectMemOperation(ResVReg, I);
case TargetOpcode::G_ICMP:
diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
index 6c9b7eb2ef37e..116f861d341c8 100644
--- a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
@@ -307,7 +307,7 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
.unsupportedIf(LegalityPredicates::any(typeIs(0, p9), typeIs(1, p9)))
.legalIf(all(typeInSet(0, allPtrs), typeInSet(1, allPtrs)));
- getActionDefinitionsBuilder(G_MEMSET)
+ getActionDefinitionsBuilder(G_MEMSET, G_MEMSET_INLINE)
.unsupportedIf(typeIs(0, p9))
.legalIf(all(typeInSet(0, allPtrs), typeInSet(1, allIntScalars)));
>From a7474f737307b9e201a39e0e731aa45065ed8a5e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= <omeragacan at gmail.com>
Date: Wed, 10 Jun 2026 17:27:52 +0100
Subject: [PATCH 03/11] Fix errors
---
llvm/include/llvm/Target/GenericOpcodes.td | 2 +-
llvm/lib/CodeGen/MachineVerifier.cpp | 4 +++-
llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp | 2 +-
3 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index e22813021b409..8270ae7690953 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -1865,7 +1865,7 @@ def G_BZERO : GenericInstruction {
def G_MEMSET_INLINE : GenericInstruction {
let OutOperandList = (outs);
- let InOperandList = (ins ptype0:$dst_addr, type1:$value, type2:$size, untyped_imm_0:$tailcall);
+ let InOperandList = (ins ptype0:$dst_addr, type1:$value, type2:$size);
let hasSideEffects = false;
let mayStore = true;
}
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index 5c68851c2e122..313c1d305c922 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -2132,7 +2132,9 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
case TargetOpcode::G_MEMSET_INLINE:
{
ArrayRef<MachineMemOperand *> MMOs = MI->memoperands();
- std::string Name = Opc == TargetOpcode::G_MEMSET ? "memset" : "bzero";
+ std::string Name = Opc == TargetOpcode::G_MEMSET ? "memset"
+ : Opc == TargetOpcode::G_MEMSET_INLINE ? "memset_inline"
+ : "bzero";
if (MMOs.size() != 1) {
report(Twine(Name, " must have 1 memory operand"), MI);
break;
diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
index 116f861d341c8..512d1494a7b36 100644
--- a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
@@ -307,7 +307,7 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
.unsupportedIf(LegalityPredicates::any(typeIs(0, p9), typeIs(1, p9)))
.legalIf(all(typeInSet(0, allPtrs), typeInSet(1, allPtrs)));
- getActionDefinitionsBuilder(G_MEMSET, G_MEMSET_INLINE)
+ getActionDefinitionsBuilder({G_MEMSET, G_MEMSET_INLINE})
.unsupportedIf(typeIs(0, p9))
.legalIf(all(typeInSet(0, allPtrs), typeInSet(1, allIntScalars)));
>From 68da09702a287ebae834172e13070be3eaee2bd8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= <omeragacan at gmail.com>
Date: Wed, 10 Jun 2026 17:31:45 +0100
Subject: [PATCH 04/11] More missing memset_inline handling
---
llvm/lib/CodeGen/MachineVerifier.cpp | 7 ++++---
llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp | 3 ++-
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index 313c1d305c922..f68ff34654ffe 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -2154,9 +2154,10 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
if (DstPtrTy.getAddressSpace() != MMOs[0]->getAddrSpace())
report("inconsistent " + Twine(Name, " address space"), MI);
- if (!MI->getOperand(MI->getNumOperands() - 1).isImm() ||
- (MI->getOperand(MI->getNumOperands() - 1).getImm() & ~1LL))
- report("'tail' flag (last operand) must be an immediate 0 or 1", MI);
+ if (Opc != TargetOpcode::G_MEMSET_INLINE)
+ if (!MI->getOperand(MI->getNumOperands() - 1).isImm() ||
+ (MI->getOperand(MI->getNumOperands() - 1).getImm() & ~1LL))
+ report("'tail' flag (last operand) must be an immediate 0 or 1", MI);
break;
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index e87cb961a83a8..2841fd38f5b04 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -2330,7 +2330,8 @@ bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg,
return true;
Register SrcReg = I.getOperand(1).getReg();
- if (I.getOpcode() == TargetOpcode::G_MEMSET) {
+ if (I.getOpcode() == TargetOpcode::G_MEMSET ||
+ I.getOpcode() == TargetOpcode::G_MEMSET_INLINE) {
Register VarReg = getOrCreateMemSetGlobal(I);
if (!VarReg.isValid())
return false;
>From 9f5a4cf0d0a05bc445ea0420b45af00e106e2a1b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= <omeragacan at gmail.com>
Date: Wed, 10 Jun 2026 18:11:50 +0100
Subject: [PATCH 05/11] Tests
---
.../GlobalISel/inline-memset-forced.mir | 75 ++++++++++
.../GlobalISel/irtranslator-memset-inline.ll | 141 ++++++++++++++++++
.../GlobalISel/legalize-memsetinline.mir | 59 ++++++++
.../AMDGPU/GlobalISel/llvm.memset.inline.ll | 20 +++
.../inline-memset.mir | 57 +++++++
.../legalizer/legalize-memset-inline.mir | 44 ++++++
.../CodeGen/RISCV/GlobalISel/memset-inline.ll | 69 +++++++++
7 files changed, 465 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/inline-memset-forced.mir
create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-memset-inline.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-memsetinline.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memset.inline.ll
create mode 100644 llvm/test/CodeGen/Mips/GlobalISel/mips-prelegalizer-combiner/inline-memset.mir
create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-memset-inline.mir
create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/memset-inline.ll
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/inline-memset-forced.mir b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memset-forced.mir
new file mode 100644
index 0000000000000..bce7d6fe3c094
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memset-forced.mir
@@ -0,0 +1,75 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=aarch64 -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+--- |
+ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+ target triple = "arm64-apple-darwin"
+
+ declare void @llvm.memset.inline.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg) #0
+
+ define void @test_memset_inline(ptr nocapture %dst, i8 %val) local_unnamed_addr {
+ entry:
+ tail call void @llvm.memset.inline.p0.i64(ptr align 4 %dst, i8 %val, i64 143, i1 false)
+ ret void
+ }
+
+ attributes #0 = { argmemonly nounwind }
+
+...
+---
+name: test_memset_inline
+alignment: 4
+tracksRegLiveness: true
+registers:
+ - { id: 0, class: _ }
+ - { id: 1, class: _ }
+ - { id: 2, class: _ }
+ - { id: 3, class: _ }
+machineFunctionInfo: {}
+body: |
+ bb.1.entry:
+ liveins: $w1, $x0
+
+ ; CHECK-LABEL: name: test_memset_inline
+ ; CHECK: liveins: $w1, $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $w1
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i8) = G_TRUNC [[COPY1]](i32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i64) = G_ZEXT [[TRUNC]](i8)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 72340172838076673
+ ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(i64) = G_MUL [[ZEXT]], [[C]]
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x i64>) = G_BUILD_VECTOR [[MUL]](i64), [[MUL]](i64)
+ ; CHECK-NEXT: G_STORE [[BUILD_VECTOR]](<2 x i64>), [[COPY]](p0) :: (store (<2 x i64>) into %ir.dst, align 4)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+ ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](s64)
+ ; CHECK-NEXT: G_STORE [[BUILD_VECTOR]](<2 x i64>), [[PTR_ADD]](p0) :: (store (<2 x i64>) into %ir.dst + 16, align 4)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
+ ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](s64)
+ ; CHECK-NEXT: G_STORE [[BUILD_VECTOR]](<2 x i64>), [[PTR_ADD1]](p0) :: (store (<2 x i64>) into %ir.dst + 32, align 4)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+ ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](s64)
+ ; CHECK-NEXT: G_STORE [[BUILD_VECTOR]](<2 x i64>), [[PTR_ADD2]](p0) :: (store (<2 x i64>) into %ir.dst + 48, align 4)
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
+ ; CHECK-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](s64)
+ ; CHECK-NEXT: G_STORE [[BUILD_VECTOR]](<2 x i64>), [[PTR_ADD3]](p0) :: (store (<2 x i64>) into %ir.dst + 64, align 4)
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 80
+ ; CHECK-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](s64)
+ ; CHECK-NEXT: G_STORE [[BUILD_VECTOR]](<2 x i64>), [[PTR_ADD4]](p0) :: (store (<2 x i64>) into %ir.dst + 80, align 4)
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 96
+ ; CHECK-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C6]](s64)
+ ; CHECK-NEXT: G_STORE [[BUILD_VECTOR]](<2 x i64>), [[PTR_ADD5]](p0) :: (store (<2 x i64>) into %ir.dst + 96, align 4)
+ ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 112
+ ; CHECK-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C7]](s64)
+ ; CHECK-NEXT: G_STORE [[BUILD_VECTOR]](<2 x i64>), [[PTR_ADD6]](p0) :: (store (<2 x i64>) into %ir.dst + 112, align 4)
+ ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s64) = G_CONSTANT i64 127
+ ; CHECK-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C8]](s64)
+ ; CHECK-NEXT: G_STORE [[BUILD_VECTOR]](<2 x i64>), [[PTR_ADD7]](p0) :: (store (<2 x i64>) into %ir.dst + 127, align 1, basealign 4)
+ ; CHECK-NEXT: RET_ReallyLR
+ %0:_(p0) = COPY $x0
+ %1:_(i32) = COPY $w1
+ %2:_(i8) = G_TRUNC %1(i32)
+ %3:_(s64) = G_CONSTANT i64 143
+ G_MEMSET_INLINE %0(p0), %2(i8), %3(s64) :: (store (s8) into %ir.dst, align 4)
+ RET_ReallyLR
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-memset-inline.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-memset-inline.ll
new file mode 100644
index 0000000000000..cd67cc8af4367
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-memset-inline.ll
@@ -0,0 +1,141 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -mtriple=aarch64-unknown-unknown -global-isel -global-isel-abort=1 -verify-machineinstrs -stop-after=irtranslator %s -o - | FileCheck %s
+
+define void @memset(ptr %dst, i8 %val) {
+ ; CHECK-LABEL: name: memset
+ ; CHECK: bb.1.entry:
+ ; CHECK-NEXT: liveins: $w1, $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $w1
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i8) = G_TRUNC [[COPY1]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i64) = G_ZEXT [[C]](i32)
+ ; CHECK-NEXT: G_MEMSET [[COPY]](p0), [[TRUNC]](i8), [[ZEXT]](i64), 0 :: (store (s8) into %ir.dst)
+ ; CHECK-NEXT: RET_ReallyLR
+entry:
+ call void @llvm.memset.p0.i32(ptr %dst, i8 %val, i32 4, i1 false)
+ ret void
+}
+
+define void @inline_memset(ptr %dst, i8 %val) {
+ ; CHECK-LABEL: name: inline_memset
+ ; CHECK: bb.1.entry:
+ ; CHECK-NEXT: liveins: $w1, $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $w1
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i8) = G_TRUNC [[COPY1]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i64) = G_ZEXT [[C]](i32)
+ ; CHECK-NEXT: G_MEMSET_INLINE [[COPY]](p0), [[TRUNC]](i8), [[ZEXT]](i64) :: (store (s8) into %ir.dst)
+ ; CHECK-NEXT: RET_ReallyLR
+entry:
+ call void @llvm.memset.inline.p0.i32(ptr %dst, i8 %val, i32 4, i1 false)
+ ret void
+}
+
+define void @memset_volatile(ptr %dst, i8 %val) {
+ ; CHECK-LABEL: name: memset_volatile
+ ; CHECK: bb.1.entry:
+ ; CHECK-NEXT: liveins: $w1, $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $w1
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i8) = G_TRUNC [[COPY1]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i64) = G_ZEXT [[C]](i32)
+ ; CHECK-NEXT: G_MEMSET [[COPY]](p0), [[TRUNC]](i8), [[ZEXT]](i64), 0 :: (volatile store (s8) into %ir.dst)
+ ; CHECK-NEXT: RET_ReallyLR
+entry:
+ call void @llvm.memset.p0.i32(ptr %dst, i8 %val, i32 4, i1 true)
+ ret void
+}
+
+define void @inline_memset_volatile(ptr %dst, i8 %val) {
+ ; CHECK-LABEL: name: inline_memset_volatile
+ ; CHECK: bb.1.entry:
+ ; CHECK-NEXT: liveins: $w1, $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $w1
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i8) = G_TRUNC [[COPY1]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i64) = G_ZEXT [[C]](i32)
+ ; CHECK-NEXT: G_MEMSET_INLINE [[COPY]](p0), [[TRUNC]](i8), [[ZEXT]](i64) :: (volatile store (s8) into %ir.dst)
+ ; CHECK-NEXT: RET_ReallyLR
+entry:
+ call void @llvm.memset.inline.p0.i32(ptr %dst, i8 %val, i32 4, i1 true)
+ ret void
+}
+
+define void @tail_memset(ptr %dst, i8 %val) {
+ ; CHECK-LABEL: name: tail_memset
+ ; CHECK: bb.1.entry:
+ ; CHECK-NEXT: liveins: $w1, $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $w1
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i8) = G_TRUNC [[COPY1]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i64) = G_ZEXT [[C]](i32)
+ ; CHECK-NEXT: G_MEMSET [[COPY]](p0), [[TRUNC]](i8), [[ZEXT]](i64), 1 :: (store (s8) into %ir.dst)
+ ; CHECK-NEXT: RET_ReallyLR
+entry:
+ tail call void @llvm.memset.p0.i32(ptr %dst, i8 %val, i32 4, i1 false)
+ ret void
+}
+
+define void @tail_inline_memset(ptr %dst, i8 %val) {
+ ; CHECK-LABEL: name: tail_inline_memset
+ ; CHECK: bb.1.entry:
+ ; CHECK-NEXT: liveins: $w1, $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $w1
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i8) = G_TRUNC [[COPY1]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i64) = G_ZEXT [[C]](i32)
+ ; CHECK-NEXT: G_MEMSET_INLINE [[COPY]](p0), [[TRUNC]](i8), [[ZEXT]](i64) :: (store (s8) into %ir.dst)
+ ; CHECK-NEXT: RET_ReallyLR
+entry:
+ tail call void @llvm.memset.inline.p0.i32(ptr %dst, i8 %val, i32 4, i1 false)
+ ret void
+}
+
+define void @tail_memset_volatile(ptr %dst, i8 %val) {
+ ; CHECK-LABEL: name: tail_memset_volatile
+ ; CHECK: bb.1.entry:
+ ; CHECK-NEXT: liveins: $w1, $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $w1
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i8) = G_TRUNC [[COPY1]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i64) = G_ZEXT [[C]](i32)
+ ; CHECK-NEXT: G_MEMSET [[COPY]](p0), [[TRUNC]](i8), [[ZEXT]](i64), 1 :: (volatile store (s8) into %ir.dst)
+ ; CHECK-NEXT: RET_ReallyLR
+entry:
+ tail call void @llvm.memset.p0.i32(ptr %dst, i8 %val, i32 4, i1 true)
+ ret void
+}
+
+define void @tail_inline_memset_volatile(ptr %dst, i8 %val) {
+ ; CHECK-LABEL: name: tail_inline_memset_volatile
+ ; CHECK: bb.1.entry:
+ ; CHECK-NEXT: liveins: $w1, $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $w1
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i8) = G_TRUNC [[COPY1]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i64) = G_ZEXT [[C]](i32)
+ ; CHECK-NEXT: G_MEMSET_INLINE [[COPY]](p0), [[TRUNC]](i8), [[ZEXT]](i64) :: (volatile store (s8) into %ir.dst)
+ ; CHECK-NEXT: RET_ReallyLR
+entry:
+ tail call void @llvm.memset.inline.p0.i32(ptr %dst, i8 %val, i32 4, i1 true)
+ ret void
+}
+
+declare void @llvm.memset.p0.i32(ptr nocapture writeonly, i8, i32, i1) nounwind
+declare void @llvm.memset.inline.p0.i32(ptr nocapture writeonly, i8, i32, i1) nounwind
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-memsetinline.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-memsetinline.mir
new file mode 100644
index 0000000000000..e689bc107a07b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-memsetinline.mir
@@ -0,0 +1,59 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-- -run-pass=legalizer -verify-machineinstrs -o - %s | FileCheck %s
+
+---
+name: memsetinline_test
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1, $vgpr2
+
+ ; CHECK-LABEL: name: memsetinline_test
+ ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY2]](s32)
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s8) = COPY [[TRUNC]](s8)
+ ; CHECK-NEXT: G_STORE [[COPY2]](s32), [[MV]](p0) :: (store (s8))
+ ; CHECK-NEXT: S_ENDPGM 0
+ %0:_(s32) = COPY $vgpr0
+ %1:_(s32) = COPY $vgpr1
+ %2:_(p0) = G_MERGE_VALUES %0:_(s32), %1:_(s32)
+ %3:_(s32) = COPY $vgpr2
+ %4:_(s8) = G_TRUNC %3:_(s32)
+ %5:_(s32) = G_CONSTANT i32 1
+ %6:_(s64) = G_ZEXT %5:_(s32)
+ G_MEMSET_INLINE %2:_(p0), %4:_(s8), %6:_(s64) :: (store (s8))
+ S_ENDPGM 0
+
+...
+---
+name: memsetinline_test_volatile
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1, $vgpr2
+
+ ; CHECK-LABEL: name: memsetinline_test_volatile
+ ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY2]](s32)
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s8) = COPY [[TRUNC]](s8)
+ ; CHECK-NEXT: G_STORE [[COPY2]](s32), [[MV]](p0) :: (volatile store (s8))
+ ; CHECK-NEXT: S_ENDPGM 0
+ %0:_(s32) = COPY $vgpr0
+ %1:_(s32) = COPY $vgpr1
+ %2:_(p0) = G_MERGE_VALUES %0:_(s32), %1:_(s32)
+ %3:_(s32) = COPY $vgpr2
+ %4:_(s8) = G_TRUNC %3:_(s32)
+ %5:_(s32) = G_CONSTANT i32 1
+ %6:_(s64) = G_ZEXT %5:_(s32)
+ G_MEMSET_INLINE %2:_(p0), %4:_(s8), %6:_(s64) :: (volatile store (s8))
+ S_ENDPGM 0
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memset.inline.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memset.inline.ll
new file mode 100644
index 0000000000000..1e73bc416f795
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memset.inline.ll
@@ -0,0 +1,20 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mem-intrinsic-expand-size=3 %s -o - | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mem-intrinsic-expand-size=5 %s -o - | FileCheck -check-prefix=GCN %s
+
+declare void @llvm.memset.inline.p1.i32(ptr addrspace(1), i8, i32, i1 immarg)
+
+define amdgpu_cs void @test(ptr addrspace(1) %dst, i8 %val) {
+; GCN-LABEL: test:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_mov_b32 s2, 0
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b64 s[0:1], 0
+; GCN-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64
+; GCN-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:1
+; GCN-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:2
+; GCN-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:3
+; GCN-NEXT: s_endpgm
+ call void @llvm.memset.inline.p1.i32(ptr addrspace(1) %dst, i8 %val, i32 4, i1 false)
+ ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/mips-prelegalizer-combiner/inline-memset.mir b/llvm/test/CodeGen/Mips/GlobalISel/mips-prelegalizer-combiner/inline-memset.mir
new file mode 100644
index 0000000000000..48a8052da0d72
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/mips-prelegalizer-combiner/inline-memset.mir
@@ -0,0 +1,57 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=mips-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+--- |
+ target datalayout = "e-m:m-p:32:32-i8:8:32-i16:16:32-i64:64-n32-S64"
+ target triple = "mipsel-pc-linux-gnu"
+
+ declare void @llvm.memset.inline.p0.i64(ptr noalias nocapture writeonly, i8, i64 immarg, i1 immarg) #0
+
+ define void @test_memset_inline(ptr nocapture %dst, i8 %val) local_unnamed_addr {
+ entry:
+ tail call void @llvm.memset.inline.p0.i64(ptr align 4 %dst, i8 %val, i64 2, i1 false)
+ ret void
+ }
+
+ attributes #0 = { argmemonly nofree nounwind willreturn }
+
+...
+---
+name: test_memset_inline
+alignment: 4
+tracksRegLiveness: true
+registers:
+ - { id: 0, class: _ }
+ - { id: 1, class: _ }
+ - { id: 2, class: _ }
+ - { id: 3, class: _ }
+ - { id: 4, class: _ }
+liveins:
+ - { reg: '$a0' }
+ - { reg: '$a1' }
+frameInfo:
+ maxAlignment: 1
+machineFunctionInfo: {}
+body: |
+ bb.1.entry:
+ liveins: $a0, $a1
+
+ ; MIPS32-LABEL: name: test_memset_inline
+ ; MIPS32: liveins: $a0, $a1
+ ; MIPS32-NEXT: {{ $}}
+ ; MIPS32-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+ ; MIPS32-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
+ ; MIPS32-NEXT: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32)
+ ; MIPS32-NEXT: G_STORE [[TRUNC]](s8), [[COPY]](p0) :: (store (s8) into %ir.dst, align 4)
+ ; MIPS32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+ ; MIPS32-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](s32)
+ ; MIPS32-NEXT: G_STORE [[TRUNC]](s8), [[PTR_ADD]](p0) :: (store (s8) into %ir.dst + 1, basealign 4)
+ ; MIPS32-NEXT: RetRA
+ %0:_(p0) = COPY $a0
+ %1:_(s32) = COPY $a1
+ %2:_(s8) = G_TRUNC %1(s32)
+ %3:_(s64) = G_CONSTANT i64 2
+ %4:_(s32) = G_TRUNC %3(s64)
+ G_MEMSET_INLINE %0(p0), %2(s8), %4(s32) :: (store (s8) into %ir.dst, align 4)
+ RetRA
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-memset-inline.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-memset-inline.mir
new file mode 100644
index 0000000000000..dd4b9a57c9b32
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-memset-inline.mir
@@ -0,0 +1,44 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=riscv32 -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=RISCV32
+# RUN: llc -mtriple=riscv64 -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=RISCV64
+
+---
+name: test_memset_inline
+alignment: 4
+liveins:
+ - { reg: '$x10', virtual-reg: '' }
+body: |
+ bb.0.entry:
+ liveins: $x10
+ ; RISCV32-LABEL: name: test_memset_inline
+ ; RISCV32: liveins: $x10
+ ; RISCV32-NEXT: {{ $}}
+ ; RISCV32-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+ ; RISCV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
+ ; RISCV32-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[C]](s32)
+ ; RISCV32-NEXT: G_STORE [[TRUNC]](s16), [[COPY]](p0) :: (store (s8), align 4)
+ ; RISCV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+ ; RISCV32-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](s32)
+ ; RISCV32-NEXT: G_STORE [[TRUNC]](s16), [[PTR_ADD]](p0) :: (store (s8) into unknown-address + 1)
+ ; RISCV32-NEXT: PseudoRET
+ ;
+ ; RISCV64-LABEL: name: test_memset_inline
+ ; RISCV64: liveins: $x10
+ ; RISCV64-NEXT: {{ $}}
+ ; RISCV64-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+ ; RISCV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 7
+ ; RISCV64-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[C]](s64)
+ ; RISCV64-NEXT: G_STORE [[TRUNC]](s16), [[COPY]](p0) :: (store (s8), align 4)
+ ; RISCV64-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+ ; RISCV64-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](s64)
+ ; RISCV64-NEXT: G_STORE [[TRUNC]](s16), [[PTR_ADD]](p0) :: (store (s8) into unknown-address + 1)
+ ; RISCV64-NEXT: PseudoRET
+ %0:_(p0) = COPY $x10
+ %1:_(s64) = G_CONSTANT i64 7
+ %2:_(s8) = G_TRUNC %1(s64)
+ %3:_(s64) = G_CONSTANT i64 2
+ %4:_(s32) = G_TRUNC %3(s64)
+ G_MEMSET_INLINE %0(p0), %2(s8), %4(s32) :: (store (s8), align 4)
+ PseudoRET
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/memset-inline.ll b/llvm/test/CodeGen/RISCV/GlobalISel/memset-inline.ll
new file mode 100644
index 0000000000000..50f1f6cf9a24d
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/memset-inline.ll
@@ -0,0 +1,69 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel -mtriple=riscv32 < %s \
+; RUN: | FileCheck -check-prefix=RV32I %s
+; RUN: llc -global-isel -mtriple=riscv64 < %s \
+; RUN: | FileCheck -check-prefix=RV64I %s
+
+define void @test_memset_inline_2bytes(ptr %dst, i8 %val) {
+; RV32I-LABEL: test_memset_inline_2bytes:
+; RV32I: # %bb.0:
+; RV32I-NEXT: sb a1, 0(a0)
+; RV32I-NEXT: sb a1, 1(a0)
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: test_memset_inline_2bytes:
+; RV64I: # %bb.0:
+; RV64I-NEXT: sb a1, 0(a0)
+; RV64I-NEXT: sb a1, 1(a0)
+; RV64I-NEXT: ret
+ call void @llvm.memset.inline.p0.i64(ptr %dst, i8 %val, i64 2, i1 false)
+ ret void
+}
+
+define void @test_memset_inline_4bytes(ptr %dst, i8 %val) {
+; RV32I-LABEL: test_memset_inline_4bytes:
+; RV32I: # %bb.0:
+; RV32I-NEXT: sb a1, 0(a0)
+; RV32I-NEXT: sb a1, 1(a0)
+; RV32I-NEXT: sb a1, 2(a0)
+; RV32I-NEXT: sb a1, 3(a0)
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: test_memset_inline_4bytes:
+; RV64I: # %bb.0:
+; RV64I-NEXT: sb a1, 0(a0)
+; RV64I-NEXT: sb a1, 1(a0)
+; RV64I-NEXT: sb a1, 2(a0)
+; RV64I-NEXT: sb a1, 3(a0)
+; RV64I-NEXT: ret
+ call void @llvm.memset.inline.p0.i64(ptr %dst, i8 %val, i64 4, i1 false)
+ ret void
+}
+
+define void @test_memset_inline_8bytes(ptr %dst, i8 %val) {
+; RV32I-LABEL: test_memset_inline_8bytes:
+; RV32I: # %bb.0:
+; RV32I-NEXT: sb a1, 0(a0)
+; RV32I-NEXT: sb a1, 1(a0)
+; RV32I-NEXT: sb a1, 2(a0)
+; RV32I-NEXT: sb a1, 3(a0)
+; RV32I-NEXT: sb a1, 4(a0)
+; RV32I-NEXT: sb a1, 5(a0)
+; RV32I-NEXT: sb a1, 6(a0)
+; RV32I-NEXT: sb a1, 7(a0)
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: test_memset_inline_8bytes:
+; RV64I: # %bb.0:
+; RV64I-NEXT: sb a1, 0(a0)
+; RV64I-NEXT: sb a1, 1(a0)
+; RV64I-NEXT: sb a1, 2(a0)
+; RV64I-NEXT: sb a1, 3(a0)
+; RV64I-NEXT: sb a1, 4(a0)
+; RV64I-NEXT: sb a1, 5(a0)
+; RV64I-NEXT: sb a1, 6(a0)
+; RV64I-NEXT: sb a1, 7(a0)
+; RV64I-NEXT: ret
+ call void @llvm.memset.inline.p0.i64(ptr %dst, i8 %val, i64 8, i1 false)
+ ret void
+}
>From 2a8ac50de0817d535f96d06cc971f216025821b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= <omeragacan at gmail.com>
Date: Wed, 10 Jun 2026 19:34:25 +0100
Subject: [PATCH 06/11] Formatting, minor fixes
---
llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp | 3 +--
llvm/lib/CodeGen/MachineVerifier.cpp | 3 +--
2 files changed, 2 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 4dc2fcb951c8a..7ae9c354440d1 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -11165,7 +11165,7 @@ LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen) {
// FIXME: support dynamically sized G_MEMCPY_INLINE and G_MEMSET_INLINE
assert(Opc != TargetOpcode::G_MEMCPY_INLINE &&
Opc != TargetOpcode::G_MEMSET_INLINE &&
- "inline memcpy with dynamic size is not yet supported");
+ "inline memcpy and memset with dynamic size are not yet supported");
return UnableToLegalize;
}
uint64_t KnownLen = LenVRegAndVal->Value.getZExtValue();
@@ -11182,7 +11182,6 @@ LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen) {
bool IsVolatile = MemOp->isVolatile();
if (Opc == TargetOpcode::G_MEMCPY || Opc == TargetOpcode::G_MEMCPY_INLINE) {
auto &MF = *MI.getParent()->getParent();
- const auto &TLI = *MF.getSubtarget().getTargetLowering();
bool OptSize = shouldLowerMemFuncForSize(MF);
uint64_t Limit = Opc == TargetOpcode::G_MEMCPY_INLINE
? std::numeric_limits<uint64_t>::max()
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index f68ff34654ffe..86187fe3950fb 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -2129,8 +2129,7 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
}
case TargetOpcode::G_BZERO:
case TargetOpcode::G_MEMSET:
- case TargetOpcode::G_MEMSET_INLINE:
- {
+ case TargetOpcode::G_MEMSET_INLINE: {
ArrayRef<MachineMemOperand *> MMOs = MI->memoperands();
std::string Name = Opc == TargetOpcode::G_MEMSET ? "memset"
: Opc == TargetOpcode::G_MEMSET_INLINE ? "memset_inline"
>From 333735ff987095b1117a8b6c23e9242bc6fab413 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= <omeragacan at gmail.com>
Date: Wed, 10 Jun 2026 19:49:22 +0100
Subject: [PATCH 07/11] MOPS support
---
llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp | 4 +++-
llvm/test/CodeGen/AArch64/GlobalISel/inline-memset-forced.mir | 2 ++
.../CodeGen/AArch64/GlobalISel/irtranslator-memset-inline.ll | 1 +
.../CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir | 3 +++
.../CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir | 3 +++
5 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index eca93c51760ac..e6521a211226e 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -1555,6 +1555,7 @@ bool AArch64LegalizerInfo::legalizeCustom(
case TargetOpcode::G_MEMCPY:
case TargetOpcode::G_MEMMOVE:
case TargetOpcode::G_MEMSET:
+ case TargetOpcode::G_MEMSET_INLINE:
return legalizeMemOps(MI, Helper);
case TargetOpcode::G_EXTRACT_VECTOR_ELT:
return legalizeExtractVectorElt(MI, MRI, Helper);
@@ -2554,7 +2555,8 @@ bool AArch64LegalizerInfo::legalizeMemOps(MachineInstr &MI,
MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
// Tagged version MOPSMemorySetTagged is legalised in legalizeIntrinsic
- if (MI.getOpcode() == TargetOpcode::G_MEMSET) {
+ if (MI.getOpcode() == TargetOpcode::G_MEMSET ||
+ MI.getOpcode() == TargetOpcode::G_MEMSET_INLINE) {
// Anyext the value being set to 64 bit (only the bottom 8 bits are read by
// the instruction).
auto &Value = MI.getOperand(1);
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/inline-memset-forced.mir b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memset-forced.mir
index bce7d6fe3c094..816ce08acfb93 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/inline-memset-forced.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memset-forced.mir
@@ -1,5 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
# RUN: llc -mtriple=aarch64 -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=aarch64 -mattr=+mops -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
--- |
target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
target triple = "arm64-apple-darwin"
@@ -65,6 +66,7 @@ body: |
; CHECK-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C8]](s64)
; CHECK-NEXT: G_STORE [[BUILD_VECTOR]](<2 x i64>), [[PTR_ADD7]](p0) :: (store (<2 x i64>) into %ir.dst + 127, align 1, basealign 4)
; CHECK-NEXT: RET_ReallyLR
+ ;
%0:_(p0) = COPY $x0
%1:_(i32) = COPY $w1
%2:_(i8) = G_TRUNC %1(i32)
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-memset-inline.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-memset-inline.ll
index cd67cc8af4367..4e299a2eec47d 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-memset-inline.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-memset-inline.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
; RUN: llc -mtriple=aarch64-unknown-unknown -global-isel -global-isel-abort=1 -verify-machineinstrs -stop-after=irtranslator %s -o - | FileCheck %s
+; RUN: llc -mtriple=aarch64-unknown-unknown -mattr=+mops -global-isel -global-isel-abort=1 -verify-machineinstrs -stop-after=irtranslator %s -o - | FileCheck %s
define void @memset(ptr %dst, i8 %val) {
; CHECK-LABEL: name: memset
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index c82627a1e8c8d..190761c4124e0 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -911,6 +911,9 @@
# DEBUG-NEXT: G_BZERO (opcode {{[0-9]+}}): 2 type indices, 1 imm index
# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: G_MEMSET_INLINE (opcode {{[0-9]+}}): 3 type indices, 0 imm indices
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: G_TRAP (opcode {{[0-9]+}}): 0 type indices, 0 imm indices
# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir
index 2aa2f2efcdc17..df971ec7db334 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir
@@ -900,6 +900,9 @@
# DEBUG-NEXT: G_BZERO (opcode {{[0-9]+}}): 2 type indices, 1 imm index
# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: G_MEMSET_INLINE (opcode {{[0-9]+}}): 3 type indices, 0 imm indices
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_TRAP (opcode {{[0-9]+}}): 0 type indices, 0 imm indices
# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
>From dfb1365d05c4a0e80cd6022a33ce6b3a3d30e82f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= <omeragacan at gmail.com>
Date: Thu, 11 Jun 2026 09:54:44 +0100
Subject: [PATCH 08/11] Post-merge fixups
---
llvm/lib/CodeGen/GlobalISel/Utils.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index bd8c8219e1220..fbb54d8971502 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -2247,8 +2247,8 @@ bool llvm::canLowerMemCpyFamily(const MachineInstr &MI,
}
case TargetOpcode::G_MEMSET:
case TargetOpcode::G_MEMSET_INLINE: {
- uint64_t Limit = Opc == TargetOpcode::G_MEMSET_INLINE
- ? std::numeric_limits<uint64_t>::max()
+ unsigned Limit = Opc == TargetOpcode::G_MEMSET_INLINE
+ ? std::numeric_limits<unsigned>::max()
: TLI.getMaxStoresPerMemset(OptSize);
auto ValVRegAndVal = getIConstantVRegValWithLookThrough(Src, MRI);
bool IsZeroVal = ValVRegAndVal && ValVRegAndVal->Value == 0;
>From 7be6391749846972bf01453237a6cf800dd19a4e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= <omeragacan at gmail.com>
Date: Thu, 11 Jun 2026 10:09:34 +0100
Subject: [PATCH 09/11] Remove dupd fun tryEmitMemsetInline, refactor
tryEmitMemcpyInline
---
llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h | 10 ++--------
llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp | 10 +---------
.../AArch64/GISel/AArch64O0PreLegalizerCombiner.cpp | 3 +--
.../AArch64/GISel/AArch64PreLegalizerCombiner.cpp | 3 +--
llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp | 3 +--
5 files changed, 6 insertions(+), 23 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index da127baeff08c..a04ff991b2cf8 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -861,16 +861,10 @@ class CombinerHelper {
LLVM_ABI bool tryCombine(MachineInstr &MI) const;
/// Emit loads and stores that perform the given memcpy.
- /// Assumes \p MI is a G_MEMCPY_INLINE
+ /// Assumes \p MI is a G_MEMCPY_INLINE or a G_MEMSET_INLINE
/// TODO: implement dynamically sized inline memcpy,
/// and rename: s/bool tryEmit/void emit/
- LLVM_ABI bool tryEmitMemcpyInline(MachineInstr &MI) const;
-
- /// Emit stores that perform the given memset.
- /// Assumes \p MI is a G_MEMSET_INLINE
- /// TODO: implement dynamically sized inline memset,
- /// and rename: s/bool tryEmit/void emit/
- LLVM_ABI bool tryEmitMemsetInline(MachineInstr &MI) const;
+ LLVM_ABI bool tryEmitMemcpyInlineFamily(MachineInstr &MI) const;
/// Match:
/// (G_UMULO x, 2) -> (G_UADDO x, x)
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index f7159ad0617b6..3cc00d816c0e6 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -1721,15 +1721,7 @@ void CombinerHelper::applyOptBrCondByInvertingCond(
Observer.changedInstr(*BrCond);
}
-bool CombinerHelper::tryEmitMemcpyInline(MachineInstr &MI) const {
- MachineIRBuilder HelperBuilder(MI);
- GISelObserverWrapper DummyObserver;
- LegalizerHelper Helper(HelperBuilder.getMF(), DummyObserver, HelperBuilder);
- return Helper.lowerMemCpyFamily(MI) ==
- LegalizerHelper::LegalizeResult::Legalized;
-}
-
-bool CombinerHelper::tryEmitMemsetInline(MachineInstr &MI) const {
+bool CombinerHelper::tryEmitMemcpyInlineFamily(MachineInstr &MI) const {
MachineIRBuilder HelperBuilder(MI);
GISelObserverWrapper DummyObserver;
LegalizerHelper Helper(HelperBuilder.getMF(), DummyObserver, HelperBuilder);
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64O0PreLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64O0PreLegalizerCombiner.cpp
index 3bca9053ca429..0846bd7994a31 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64O0PreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64O0PreLegalizerCombiner.cpp
@@ -95,9 +95,8 @@ bool AArch64O0PreLegalizerCombinerImpl::tryCombineAll(MachineInstr &MI) const {
unsigned Opc = MI.getOpcode();
switch (Opc) {
case TargetOpcode::G_MEMCPY_INLINE:
- return Helper.tryEmitMemcpyInline(MI);
case TargetOpcode::G_MEMSET_INLINE:
- return Helper.tryEmitMemsetInline(MI);
+ return Helper.tryEmitMemcpyInlineFamily(MI);
case TargetOpcode::G_MEMCPY:
case TargetOpcode::G_MEMMOVE:
case TargetOpcode::G_MEMSET: {
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
index cf4c1546efdfd..9834ea8ce5df9 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
@@ -792,9 +792,8 @@ bool AArch64PreLegalizerCombinerImpl::tryCombineAll(MachineInstr &MI) const {
unsigned Opc = MI.getOpcode();
switch (Opc) {
case TargetOpcode::G_MEMCPY_INLINE:
- return Helper.tryEmitMemcpyInline(MI);
case TargetOpcode::G_MEMSET_INLINE:
- return Helper.tryEmitMemsetInline(MI);
+ return Helper.tryEmitMemcpyInlineFamily(MI);
case TargetOpcode::G_MEMCPY:
case TargetOpcode::G_MEMMOVE:
case TargetOpcode::G_MEMSET: {
diff --git a/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp b/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
index 43cf18517026b..dd9b76d9c9ecc 100644
--- a/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
@@ -58,9 +58,8 @@ class MipsPreLegalizerCombinerImpl : public Combiner {
default:
return false;
case TargetOpcode::G_MEMCPY_INLINE:
- return Helper.tryEmitMemcpyInline(MI);
case TargetOpcode::G_MEMSET_INLINE:
- return Helper.tryEmitMemsetInline(MI);
+ return Helper.tryEmitMemcpyInlineFamily(MI);
case TargetOpcode::G_LOAD:
case TargetOpcode::G_SEXTLOAD:
case TargetOpcode::G_ZEXTLOAD: {
>From a7c38b926b071ec17172cf548b3e9acc915ddef8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= <omeragacan at gmail.com>
Date: Fri, 12 Jun 2026 12:55:54 +0100
Subject: [PATCH 10/11] Add inline-small-memset test
---
.../GlobalISel/inline-small-memset.mir | 72 +++++++++++++++++++
1 file changed, 72 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/inline-small-memset.mir
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/inline-small-memset.mir b/llvm/test/CodeGen/AArch64/GlobalISel/inline-small-memset.mir
new file mode 100644
index 0000000000000..4c1b28d32fcdd
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/inline-small-memset.mir
@@ -0,0 +1,72 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=aarch64 -run-pass=aarch64-prelegalizer-combiner -O0 -verify-machineinstrs %s -o - | FileCheck %s
+--- |
+ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+ target triple = "arm64-apple-darwin"
+
+ declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg) #0
+
+ define void @test_small_memset(ptr nocapture %dst) {
+ entry:
+ tail call void @llvm.memset.p0.i64(ptr align 4 %dst, i8 12, i64 32, i1 false)
+ ret void
+ }
+
+ define void @test_large_memset(ptr nocapture %dst) {
+ entry:
+ tail call void @llvm.memset.p0.i64(ptr align 4 %dst, i8 12, i64 36, i1 false)
+ ret void
+ }
+
+ attributes #0 = { argmemonly nounwind }
+
+...
+---
+name: test_small_memset
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ liveins: $x0
+
+ ; CHECK-LABEL: name: test_small_memset
+ ; CHECK: liveins: $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 868082074056920076
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x i64>) = G_BUILD_VECTOR [[C]](i64), [[C]](i64)
+ ; CHECK-NEXT: G_STORE [[BUILD_VECTOR]](<2 x i64>), [[COPY]](p0) :: (store (<2 x i64>) into %ir.dst, align 4)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+ ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](s64)
+ ; CHECK-NEXT: G_STORE [[BUILD_VECTOR]](<2 x i64>), [[PTR_ADD]](p0) :: (store (<2 x i64>) into %ir.dst + 16, align 4)
+ ; CHECK-NEXT: RET_ReallyLR
+ %0:_(p0) = COPY $x0
+ %1:_(i8) = G_CONSTANT i8 12
+ %2:_(i64) = G_CONSTANT i64 32
+ G_MEMSET %0(p0), %1(i8), %2(i64), 1 :: (store (s8) into %ir.dst, align 4)
+ RET_ReallyLR
+
+...
+---
+name: test_large_memset
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ liveins: $x0
+
+ ; CHECK-LABEL: name: test_large_memset
+ ; CHECK: liveins: $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i8) = G_CONSTANT i8 12
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 36
+ ; CHECK-NEXT: G_MEMSET [[COPY]](p0), [[C]](i8), [[C1]](i64), 1 :: (store (s8) into %ir.dst, align 4)
+ ; CHECK-NEXT: RET_ReallyLR
+ %0:_(p0) = COPY $x0
+ %1:_(i8) = G_CONSTANT i8 12
+ %2:_(i64) = G_CONSTANT i64 36
+ G_MEMSET %0(p0), %1(i8), %2(i64), 1 :: (store (s8) into %ir.dst, align 4)
+ RET_ReallyLR
+
+...
>From 5a99d4b8d2c10ea13e609ec16de5730074bb7062 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= <omeragacan at gmail.com>
Date: Fri, 12 Jun 2026 13:00:24 +0100
Subject: [PATCH 11/11] Fold getActionDefinitionsBuilder call
---
llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index 305f977d14914..fdd5e71c32830 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -527,9 +527,7 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
getActionDefinitionsBuilder({G_MEMCPY, G_MEMMOVE, G_MEMSET}).libcall();
- getActionDefinitionsBuilder(G_MEMCPY_INLINE).lower();
-
- getActionDefinitionsBuilder(G_MEMSET_INLINE).lower();
+ getActionDefinitionsBuilder({G_MEMCPY_INLINE, G_MEMSET_INLINE}).lower();
getActionDefinitionsBuilder({G_DYN_STACKALLOC, G_STACKSAVE, G_STACKRESTORE})
.lower();
More information about the llvm-commits
mailing list