[llvm] [GlobalISel] Implement `llvm.memset.inline` (PR #203198)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 12 02:10:55 PDT 2026
=?utf-8?q?Ömer_Sinan_Ağacan?= <omeragacan at gmail.com>,
=?utf-8?q?Ömer_Sinan_Ağacan?= <omeragacan at gmail.com>,
=?utf-8?q?Ömer_Sinan_Ağacan?= <omeragacan at gmail.com>,
=?utf-8?q?Ömer_Sinan_Ağacan?= <omeragacan at gmail.com>,
=?utf-8?q?Ömer_Sinan_Ağacan?= <omeragacan at gmail.com>,
=?utf-8?q?Ömer_Sinan_Ağacan?= <omeragacan at gmail.com>,
=?utf-8?q?Ömer_Sinan_Ağacan?= <omeragacan at gmail.com>,
=?utf-8?q?Ömer_Sinan_Ağacan?= <omeragacan at gmail.com>
Message-ID:
In-Reply-To: <llvm.org/llvm/llvm-project/pull/203198 at github.com>
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Ömer Sinan Ağacan (osa1)
<details>
<summary>Changes</summary>
Fixes the crash when compiling IR with `llvm.memset.inline`:
LLVM ERROR: cannot select: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@<!-- -->llvm.memset.inline), %0:gpr(p0), %1:gpr(i8), %2:gpr(i64), 0 (in function: zero_many_small)
Both the changes in code and tests follow the existing code and tests
for `llvm.memcpy.inline`. E.g. the test
`CodeGen/AMDGPU/GlobalISel/llvm.memset.inline.ll` is the same test as
`llvm.memcpy.inline.ll` in the same dir, just with `memset.inline`
instead of `memcpy.inline`.
Related issue: #<!-- -->63577
---
Patch is 41.72 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/203198.diff
26 Files Affected:
- (modified) llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h (+2-2)
- (modified) llvm/include/llvm/Support/TargetOpcodes.def (+3)
- (modified) llvm/include/llvm/Target/GenericOpcodes.td (+7)
- (modified) llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp (+1-1)
- (modified) llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp (+6-2)
- (modified) llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp (+4-2)
- (modified) llvm/lib/CodeGen/GlobalISel/Utils.cpp (+13-7)
- (modified) llvm/lib/CodeGen/MachineVerifier.cpp (+9-5)
- (modified) llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp (+2)
- (modified) llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp (+6-1)
- (modified) llvm/lib/Target/AArch64/GISel/AArch64O0PreLegalizerCombiner.cpp (+2-1)
- (modified) llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp (+2-1)
- (modified) llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp (+2-1)
- (modified) llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp (+2-1)
- (modified) llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp (+2)
- (modified) llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp (+3-1)
- (modified) llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp (+1-1)
- (added) llvm/test/CodeGen/AArch64/GlobalISel/inline-memset-forced.mir (+77)
- (added) llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-memset-inline.ll (+142)
- (modified) llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir (+3)
- (added) llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-memsetinline.mir (+59)
- (added) llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memset.inline.ll (+20)
- (added) llvm/test/CodeGen/Mips/GlobalISel/mips-prelegalizer-combiner/inline-memset.mir (+57)
- (modified) llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir (+3)
- (added) llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-memset-inline.mir (+44)
- (added) llvm/test/CodeGen/RISCV/GlobalISel/memset-inline.ll (+69)
``````````diff
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index aa61310994a67..a04ff991b2cf8 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -861,10 +861,10 @@ class CombinerHelper {
LLVM_ABI bool tryCombine(MachineInstr &MI) const;
/// Emit loads and stores that perform the given memcpy.
- /// Assumes \p MI is a G_MEMCPY_INLINE
+ /// Assumes \p MI is a G_MEMCPY_INLINE or a G_MEMSET_INLINE
/// TODO: implement dynamically sized inline memcpy,
/// and rename: s/bool tryEmit/void emit/
- LLVM_ABI bool tryEmitMemcpyInline(MachineInstr &MI) const;
+ LLVM_ABI bool tryEmitMemcpyInlineFamily(MachineInstr &MI) const;
/// Match:
/// (G_UMULO x, 2) -> (G_UADDO x, x)
diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def
index 19b48cb9765ff..051cea764e5a0 100644
--- a/llvm/include/llvm/Support/TargetOpcodes.def
+++ b/llvm/include/llvm/Support/TargetOpcodes.def
@@ -959,6 +959,9 @@ HANDLE_TARGET_OPCODE(G_MEMMOVE)
HANDLE_TARGET_OPCODE(G_MEMSET)
HANDLE_TARGET_OPCODE(G_BZERO)
+/// llvm.memset.inline intrinsic
+HANDLE_TARGET_OPCODE(G_MEMSET_INLINE)
+
/// llvm.trap, llvm.debugtrap and llvm.ubsantrap intrinsics
HANDLE_TARGET_OPCODE(G_TRAP)
HANDLE_TARGET_OPCODE(G_DEBUGTRAP)
diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index b0edb462f0f9d..8270ae7690953 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -1863,6 +1863,13 @@ def G_BZERO : GenericInstruction {
let mayStore = true;
}
+def G_MEMSET_INLINE : GenericInstruction {
+ let OutOperandList = (outs);
+ let InOperandList = (ins ptype0:$dst_addr, type1:$value, type2:$size);
+ let hasSideEffects = false;
+ let mayStore = true;
+}
+
//------------------------------------------------------------------------------
// Trap intrinsics
//------------------------------------------------------------------------------
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 88b68d7685c63..3cc00d816c0e6 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -1721,7 +1721,7 @@ void CombinerHelper::applyOptBrCondByInvertingCond(
Observer.changedInstr(*BrCond);
}
-bool CombinerHelper::tryEmitMemcpyInline(MachineInstr &MI) const {
+bool CombinerHelper::tryEmitMemcpyInlineFamily(MachineInstr &MI) const {
MachineIRBuilder HelperBuilder(MI);
GISelObserverWrapper DummyObserver;
LegalizerHelper Helper(HelperBuilder.getMF(), DummyObserver, HelperBuilder);
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 28755da8327aa..4547b0c41cb96 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1805,7 +1805,8 @@ bool IRTranslator::translateMemFunc(const CallInst &CI,
DstAlign = MSI->getDestAlign().valueOrOne();
}
- if (Opcode != TargetOpcode::G_MEMCPY_INLINE) {
+ if (Opcode != TargetOpcode::G_MEMCPY_INLINE &&
+ Opcode != TargetOpcode::G_MEMSET_INLINE) {
// We need to propagate the tail call flag from the IR inst as an argument.
// Otherwise, we have to pessimize and assume later that we cannot tail call
// any memory intrinsics.
@@ -1835,7 +1836,8 @@ bool IRTranslator::translateMemFunc(const CallInst &CI,
ICall.addMemOperand(
MF->getMachineMemOperand(MachinePointerInfo(CI.getArgOperand(0)),
StoreFlags, 1, DstAlign, AAInfo));
- if (Opcode != TargetOpcode::G_MEMSET)
+ if (Opcode != TargetOpcode::G_MEMSET &&
+ Opcode != TargetOpcode::G_MEMSET_INLINE)
ICall.addMemOperand(MF->getMachineMemOperand(
MachinePointerInfo(SrcPtr), LoadFlags, 1, SrcAlign, AAInfo));
@@ -2445,6 +2447,8 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMMOVE);
case Intrinsic::memset:
return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMSET);
+ case Intrinsic::memset_inline:
+ return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMSET_INLINE);
case Intrinsic::eh_typeid_for: {
GlobalValue *GV = ExtractTypeInfo(CI.getArgOperand(0));
Register Reg = getOrCreateVReg(CI);
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 7c1333e127deb..d7400e81a0141 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -4947,6 +4947,7 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) {
case G_MEMCPY:
case G_MEMMOVE:
case G_MEMCPY_INLINE:
+ case G_MEMSET_INLINE:
return lowerMemCpyFamily(MI);
case G_ZEXT:
case G_SEXT:
@@ -11005,7 +11006,8 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerMemCpyFamily(
const unsigned Opc = MI.getOpcode();
assert((Opc == TargetOpcode::G_MEMCPY ||
Opc == TargetOpcode::G_MEMCPY_INLINE ||
- Opc == TargetOpcode::G_MEMMOVE || Opc == TargetOpcode::G_MEMSET) &&
+ Opc == TargetOpcode::G_MEMMOVE || Opc == TargetOpcode::G_MEMSET ||
+ Opc == TargetOpcode::G_MEMSET_INLINE) &&
"Expected memcpy like instruction");
if (KnownLen == 0) {
@@ -11020,7 +11022,7 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerMemCpyFamily(
if (Opc == TargetOpcode::G_MEMMOVE)
return lowerMemmove(MI, Dst, Src, KnownLen, Alignment, DstAlignCanChange,
MemOps);
- if (Opc == TargetOpcode::G_MEMSET)
+ if (Opc == TargetOpcode::G_MEMSET || Opc == TargetOpcode::G_MEMSET_INLINE)
return lowerMemset(MI, Dst, Src, KnownLen, Alignment, DstAlignCanChange,
MemOps);
return UnableToLegalize;
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 4000f0027ab85..fbb54d8971502 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -2159,7 +2159,8 @@ bool llvm::canLowerMemCpyFamily(const MachineInstr &MI,
const unsigned Opc = MI.getOpcode();
assert((Opc == TargetOpcode::G_MEMCPY ||
Opc == TargetOpcode::G_MEMCPY_INLINE ||
- Opc == TargetOpcode::G_MEMMOVE || Opc == TargetOpcode::G_MEMSET) &&
+ Opc == TargetOpcode::G_MEMMOVE || Opc == TargetOpcode::G_MEMSET ||
+ Opc == TargetOpcode::G_MEMSET_INLINE) &&
"Expected memcpy like instruction");
auto MMOIt = MI.memoperands_begin();
@@ -2171,7 +2172,7 @@ bool llvm::canLowerMemCpyFamily(const MachineInstr &MI,
Register Len;
std::tie(Dst, Src, Len) = MI.getFirst3Regs();
- if (Opc != TargetOpcode::G_MEMSET) {
+ if (Opc != TargetOpcode::G_MEMSET && Opc != TargetOpcode::G_MEMSET_INLINE) {
assert(MMOIt != MI.memoperands_end() && "Expected a second MMO on MI");
MemOp = *(++MMOIt);
SrcAlign = MemOp->getBaseAlign();
@@ -2181,9 +2182,10 @@ bool llvm::canLowerMemCpyFamily(const MachineInstr &MI,
// See if this is a constant length copy.
auto LenVRegAndVal = getIConstantVRegValWithLookThrough(Len, MRI);
if (!LenVRegAndVal) {
- // FIXME: support dynamically sized G_MEMCPY_INLINE
+ // FIXME: support dynamically sized G_MEMCPY_INLINE and G_MEMSET_INLINE
assert(Opc != TargetOpcode::G_MEMCPY_INLINE &&
- "inline memcpy with dynamic size is not yet supported");
+ Opc != TargetOpcode::G_MEMSET_INLINE &&
+ "inline memcpy and memset with dynamic size are not yet supported");
return false;
}
@@ -2193,7 +2195,8 @@ bool llvm::canLowerMemCpyFamily(const MachineInstr &MI,
if (KnownLen == 0)
return true;
- if (Opc != TargetOpcode::G_MEMCPY_INLINE && MaxLen && KnownLen > MaxLen)
+ if (Opc != TargetOpcode::G_MEMCPY_INLINE &&
+ Opc != TargetOpcode::G_MEMSET_INLINE && MaxLen && KnownLen > MaxLen)
return false;
bool IsVolatile = MemOp->isVolatile();
@@ -2242,8 +2245,11 @@ bool llvm::canLowerMemCpyFamily(const MachineInstr &MI,
DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
MF.getFunction().getAttributes(), TLI);
}
- case TargetOpcode::G_MEMSET: {
- unsigned Limit = TLI.getMaxStoresPerMemset(OptSize);
+ case TargetOpcode::G_MEMSET:
+ case TargetOpcode::G_MEMSET_INLINE: {
+ unsigned Limit = Opc == TargetOpcode::G_MEMSET_INLINE
+ ? std::numeric_limits<unsigned>::max()
+ : TLI.getMaxStoresPerMemset(OptSize);
auto ValVRegAndVal = getIConstantVRegValWithLookThrough(Src, MRI);
bool IsZeroVal = ValVRegAndVal && ValVRegAndVal->Value == 0;
return findGISelOptimalMemOpLowering(
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index c611b837a86b4..86187fe3950fb 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -2128,9 +2128,12 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
break;
}
case TargetOpcode::G_BZERO:
- case TargetOpcode::G_MEMSET: {
+ case TargetOpcode::G_MEMSET:
+ case TargetOpcode::G_MEMSET_INLINE: {
ArrayRef<MachineMemOperand *> MMOs = MI->memoperands();
- std::string Name = Opc == TargetOpcode::G_MEMSET ? "memset" : "bzero";
+ std::string Name = Opc == TargetOpcode::G_MEMSET ? "memset"
+ : Opc == TargetOpcode::G_MEMSET_INLINE ? "memset_inline"
+ : "bzero";
if (MMOs.size() != 1) {
report(Twine(Name, " must have 1 memory operand"), MI);
break;
@@ -2150,9 +2153,10 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
if (DstPtrTy.getAddressSpace() != MMOs[0]->getAddrSpace())
report("inconsistent " + Twine(Name, " address space"), MI);
- if (!MI->getOperand(MI->getNumOperands() - 1).isImm() ||
- (MI->getOperand(MI->getNumOperands() - 1).getImm() & ~1LL))
- report("'tail' flag (last operand) must be an immediate 0 or 1", MI);
+ if (Opc != TargetOpcode::G_MEMSET_INLINE)
+ if (!MI->getOperand(MI->getNumOperands() - 1).isImm() ||
+ (MI->getOperand(MI->getNumOperands() - 1).getImm() & ~1LL))
+ report("'tail' flag (last operand) must be an immediate 0 or 1", MI);
break;
}
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index cf650fd5c4e72..4f4c999ab244d 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -3603,6 +3603,7 @@ bool AArch64InstructionSelector::select(MachineInstr &I) {
case TargetOpcode::G_MEMCPY_INLINE:
case TargetOpcode::G_MEMMOVE:
case TargetOpcode::G_MEMSET:
+ case TargetOpcode::G_MEMSET_INLINE:
assert(STI.hasMOPS() && "Shouldn't get here without +mops feature");
return selectMOPS(I, MRI);
}
@@ -3629,6 +3630,7 @@ bool AArch64InstructionSelector::selectMOPS(MachineInstr &GI,
Mopcode = AArch64::MOPSMemoryMovePseudo;
break;
case TargetOpcode::G_MEMSET:
+ case TargetOpcode::G_MEMSET_INLINE:
// For tagged memset see llvm.aarch64.mops.memset.tag
Mopcode = AArch64::MOPSMemorySetPseudo;
break;
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index c0756ab66cb49..e6521a211226e 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -1376,6 +1376,9 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
getActionDefinitionsBuilder(G_MEMCPY_INLINE)
.legalForCartesianProduct({p0}, {p0}, {s64});
+ getActionDefinitionsBuilder(G_MEMSET_INLINE)
+ .legalForCartesianProduct({p0}, {s64}, {s64})
+ .customForCartesianProduct({p0}, {s8}, {s64});
} else {
getActionDefinitionsBuilder({G_BZERO, G_MEMCPY, G_MEMMOVE, G_MEMSET})
.libcall();
@@ -1552,6 +1555,7 @@ bool AArch64LegalizerInfo::legalizeCustom(
case TargetOpcode::G_MEMCPY:
case TargetOpcode::G_MEMMOVE:
case TargetOpcode::G_MEMSET:
+ case TargetOpcode::G_MEMSET_INLINE:
return legalizeMemOps(MI, Helper);
case TargetOpcode::G_EXTRACT_VECTOR_ELT:
return legalizeExtractVectorElt(MI, MRI, Helper);
@@ -2551,7 +2555,8 @@ bool AArch64LegalizerInfo::legalizeMemOps(MachineInstr &MI,
MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
// Tagged version MOPSMemorySetTagged is legalised in legalizeIntrinsic
- if (MI.getOpcode() == TargetOpcode::G_MEMSET) {
+ if (MI.getOpcode() == TargetOpcode::G_MEMSET ||
+ MI.getOpcode() == TargetOpcode::G_MEMSET_INLINE) {
// Anyext the value being set to 64 bit (only the bottom 8 bits are read by
// the instruction).
auto &Value = MI.getOperand(1);
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64O0PreLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64O0PreLegalizerCombiner.cpp
index 613cf24ed0aa6..0846bd7994a31 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64O0PreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64O0PreLegalizerCombiner.cpp
@@ -95,7 +95,8 @@ bool AArch64O0PreLegalizerCombinerImpl::tryCombineAll(MachineInstr &MI) const {
unsigned Opc = MI.getOpcode();
switch (Opc) {
case TargetOpcode::G_MEMCPY_INLINE:
- return Helper.tryEmitMemcpyInline(MI);
+ case TargetOpcode::G_MEMSET_INLINE:
+ return Helper.tryEmitMemcpyInlineFamily(MI);
case TargetOpcode::G_MEMCPY:
case TargetOpcode::G_MEMMOVE:
case TargetOpcode::G_MEMSET: {
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
index d8aae8bf420aa..9834ea8ce5df9 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
@@ -792,7 +792,8 @@ bool AArch64PreLegalizerCombinerImpl::tryCombineAll(MachineInstr &MI) const {
unsigned Opc = MI.getOpcode();
switch (Opc) {
case TargetOpcode::G_MEMCPY_INLINE:
- return Helper.tryEmitMemcpyInline(MI);
+ case TargetOpcode::G_MEMSET_INLINE:
+ return Helper.tryEmitMemcpyInlineFamily(MI);
case TargetOpcode::G_MEMCPY:
case TargetOpcode::G_MEMMOVE:
case TargetOpcode::G_MEMSET: {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index e341392ae068a..84ed533863fd9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -2238,7 +2238,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
.lower();
}
- getActionDefinitionsBuilder({G_MEMCPY, G_MEMCPY_INLINE, G_MEMMOVE, G_MEMSET})
+ getActionDefinitionsBuilder(
+ {G_MEMCPY, G_MEMCPY_INLINE, G_MEMMOVE, G_MEMSET, G_MEMSET_INLINE})
.lower();
getActionDefinitionsBuilder({G_TRAP, G_DEBUGTRAP}).custom();
diff --git a/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp b/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
index 5ada356de7e44..dd9b76d9c9ecc 100644
--- a/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
@@ -58,7 +58,8 @@ class MipsPreLegalizerCombinerImpl : public Combiner {
default:
return false;
case TargetOpcode::G_MEMCPY_INLINE:
- return Helper.tryEmitMemcpyInline(MI);
+ case TargetOpcode::G_MEMSET_INLINE:
+ return Helper.tryEmitMemcpyInlineFamily(MI);
case TargetOpcode::G_LOAD:
case TargetOpcode::G_SEXTLOAD:
case TargetOpcode::G_ZEXTLOAD: {
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index 716fab95eb943..305f977d14914 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -529,6 +529,8 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
getActionDefinitionsBuilder(G_MEMCPY_INLINE).lower();
+ getActionDefinitionsBuilder(G_MEMSET_INLINE).lower();
+
getActionDefinitionsBuilder({G_DYN_STACKALLOC, G_STACKSAVE, G_STACKRESTORE})
.lower();
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 79577505157fd..2841fd38f5b04 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -1042,6 +1042,7 @@ bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
case TargetOpcode::G_MEMCPY:
case TargetOpcode::G_MEMCPY_INLINE:
case TargetOpcode::G_MEMSET:
+ case TargetOpcode::G_MEMSET_INLINE:
return selectMemOperation(ResVReg, I);
case TargetOpcode::G_ICMP:
@@ -2329,7 +2330,8 @@ bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg,
return true;
Register SrcReg = I.getOperand(1).getReg();
- if (I.getOpcode() == TargetOpcode::G_MEMSET) {
+ if (I.getOpcode() == TargetOpcode::G_MEMSET ||
+ I.getOpcode() == TargetOpcode::G_MEMSET_INLINE) {
Register VarReg = getOrCreateMemSetGlobal(I);
if (!VarReg.isValid())
return false;
diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
index 6c9b7eb2ef37e..512d1494a7b36 100644
--- a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
@@ -307,7 +307,7 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
.unsupportedIf(LegalityPredicates::any(typeIs(0, p9), typeIs(1, p9)))
.legalIf(all(typeInSet(0, allPtrs), typeInSet(1, allPtrs)));
- getActionDefinitionsBuilder(G_MEMSET)
+ getActionDefinitionsBuilder({G_MEMSET, G_MEMSET_INLINE})
.unsupportedIf(typeIs(0, p9))
.legalIf(all(typeInSet(0, allPtrs), typeInSet(1, allIntScalars)));
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/inline-memset-forced.mir b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memset-forced.mir
new file mode 100644
index 0000000000000..816ce08acfb93
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memset-forced.mir
@@ -0,0 +1,77 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=aarch64 -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=aarch64 -mattr=+mops -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+--- |
+ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+ target triple = "arm64-apple-darwin"
+
+ declare void @llvm.memset.inline.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg) #0
+
+ define void @test_memset_inline(ptr nocapture %dst, i8 %val) local_unnamed_addr {
+ entry:
+ tail call void @llvm.memset.inline.p0.i64(ptr align 4 %dst, i8 %val, i64 143, i1 false)
+ ret void
+ }
+
+ attributes #0 = { argmemonly nounwind }
+
+...
+---
+name: test_memset_inline
+alignment: 4
+tracksRegLiveness: true
+registers:
+ - { id: 0, class: _ }
+ - { id: 1, class: _ }
+ - { id: 2, class: _ }
+ - { id: 3, class: _ }
+machineFunctionInfo: {}
+body: |
+ bb.1.entry:
+ liveins: $w1, $x0
+
+ ; CHECK-LABEL: name: test_memset_inline
+ ; CHECK: liveins: $w1, $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $w1
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i8) = G_TRUNC [[COPY1]](i32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i64) = G_ZEXT [[TRUNC]](i8)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 72340172838076673
+ ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(i64) = G_MUL [[ZEXT]], [[C]]
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x i64>) = G_BUILD_VECTOR [[MUL]](i64), [[MUL]](i64)
+ ; CHECK-NEXT: G_STORE [[BUILD_VECTOR]](<2 x i64>), [[COPY]](p0) :: (store (<2 x i64>) into %ir.dst, align 4)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+ ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](s64)
+...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/203198
More information about the llvm-commits
mailing list