[llvm] [SelectionDAG] Update MemOp to distinguish memmove from memcpy (PR #206005)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 26 04:34:23 PDT 2026
=?utf-8?q?Ömer_Sinan_Ağacan?= <omeragacan at gmail.com>
Message-ID:
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206005 at github.com>
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-arm
Author: Ömer Sinan Ağacan (osa1)
<details>
<summary>Changes</summary>
---
Full diff: https://github.com/llvm/llvm-project/pull/206005.diff
8 Files Affected:
- (modified) llvm/include/llvm/CodeGen/TargetLowering.h (+46-26)
- (modified) llvm/lib/CodeGen/GlobalISel/Utils.cpp (+4-3)
- (modified) llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp (+1-1)
- (modified) llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp (+1-1)
- (modified) llvm/lib/Target/ARM/ARMISelLowering.cpp (+1-1)
- (modified) llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp (+2-1)
- (modified) llvm/lib/Target/RISCV/RISCVISelLowering.cpp (+1-1)
- (modified) llvm/lib/Target/X86/X86ISelLoweringCall.cpp (+2-1)
``````````diff
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 7a10a693934fd..e6b98195ce54c 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -118,21 +118,27 @@ enum Preference : uint8_t {
// MemOp models a memory operation, either memset or memcpy/memmove.
struct MemOp {
private:
+ enum class MemOpKind {
+ Memset,
+ MemsetWithZero, // memset the memory with zeros
+ Memcpy, // copy memory from source to destination, source and destination do
+ // not overlap
+ MemcpyStrSrc, // memcpy source is an in-register constant, so it does not
+ // need to be loaded
+ Memmove, // memmove: like memcpy for source and destination regions may
+ // overlap
+ };
+
// Shared
uint64_t Size;
bool DstAlignCanChange; // true if destination alignment can satisfy any
// constraint.
Align DstAlign; // Specified alignment of the memory operation.
- bool AllowOverlap;
- // memset only
- bool IsMemset; // If setthis memory operation is a memset.
- bool ZeroMemset; // If set clears out memory with zeros.
- // memcpy only
- bool MemcpyStrSrc; // Indicates whether the memcpy source is an in-register
- // constant so it does not need to be loaded.
- Align SrcAlign; // Inferred alignment of the source or default value if the
- // memory operation does not need to load the value.
+ bool IsVolatile;
+ MemOpKind Kind;
+ Align SrcAlign; // Inferred alignment of the source or default value if the
+ // memory operation does not need to load the value.
public:
static MemOp Copy(uint64_t Size, bool DstAlignCanChange, Align DstAlign,
Align SrcAlign, bool IsVolatile,
@@ -141,10 +147,20 @@ struct MemOp {
Op.Size = Size;
Op.DstAlignCanChange = DstAlignCanChange;
Op.DstAlign = DstAlign;
- Op.AllowOverlap = !IsVolatile;
- Op.IsMemset = false;
- Op.ZeroMemset = false;
- Op.MemcpyStrSrc = MemcpyStrSrc;
+ Op.IsVolatile = IsVolatile;
+ Op.Kind = MemcpyStrSrc ? MemOpKind::MemcpyStrSrc : MemOpKind::Memcpy;
+ Op.SrcAlign = SrcAlign;
+ return Op;
+ }
+
+ static MemOp Move(uint64_t Size, bool DstAlignCanChange, Align DstAlign,
+ Align SrcAlign, bool IsVolatile) {
+ MemOp Op;
+ Op.Size = Size;
+ Op.DstAlignCanChange = DstAlignCanChange;
+ Op.DstAlign = DstAlign;
+ Op.IsVolatile = IsVolatile;
+ Op.Kind = MemOpKind::Memmove;
Op.SrcAlign = SrcAlign;
return Op;
}
@@ -155,10 +171,8 @@ struct MemOp {
Op.Size = Size;
Op.DstAlignCanChange = DstAlignCanChange;
Op.DstAlign = DstAlign;
- Op.AllowOverlap = !IsVolatile;
- Op.IsMemset = true;
- Op.ZeroMemset = IsZeroMemset;
- Op.MemcpyStrSrc = false;
+ Op.IsVolatile = IsVolatile;
+ Op.Kind = IsZeroMemset ? MemOpKind::MemsetWithZero : MemOpKind::Memset;
return Op;
}
@@ -168,19 +182,25 @@ struct MemOp {
return DstAlign;
}
bool isFixedDstAlign() const { return !DstAlignCanChange; }
- bool allowOverlap() const { return AllowOverlap; }
- bool isMemset() const { return IsMemset; }
- bool isMemcpy() const { return !IsMemset; }
- bool isMemcpyWithFixedDstAlign() const {
- return isMemcpy() && !DstAlignCanChange;
+ bool allowOverlap() const { return !IsVolatile; }
+ bool isMemset() const {
+ return Kind == MemOpKind::Memset || Kind == MemOpKind::MemsetWithZero;
+ }
+ bool isMemcpy() const {
+ return Kind == MemOpKind::Memcpy || Kind == MemOpKind::MemcpyStrSrc;
+ }
+ bool isMemmove() const { return Kind == MemOpKind::Memmove; }
+ bool isMemcpyOrMemmove() const { return isMemcpy() || isMemmove(); }
+ bool isMemcpyOrMemmoveWithFixedDstAlign() const {
+ return isMemcpyOrMemmove() && !DstAlignCanChange;
}
- bool isZeroMemset() const { return isMemset() && ZeroMemset; }
+ bool isZeroMemset() const { return Kind == MemOpKind::MemsetWithZero; }
bool isMemcpyStrSrc() const {
- assert(isMemcpy() && "Must be a memcpy");
- return MemcpyStrSrc;
+ assert(isMemcpyOrMemmove() && "Must be a memcpy or memmove");
+ return Kind == MemOpKind::MemcpyStrSrc;
}
Align getSrcAlign() const {
- assert(isMemcpy() && "Must be a memcpy");
+ assert(isMemcpyOrMemmove() && "Must be a memcpy or memmove");
return SrcAlign;
}
bool isSrcAligned(Align AlignCheck) const {
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index fbb54d8971502..7ef788614ae9d 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -2089,7 +2089,8 @@ static bool findGISelOptimalMemOpLowering(std::vector<LLT> &MemOps,
unsigned DstAS, unsigned SrcAS,
const AttributeList &FuncAttributes,
const TargetLowering &TLI) {
- if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign())
+ if (Op.isMemcpyOrMemmoveWithFixedDstAlign() &&
+ Op.getSrcAlign() < Op.getDstAlign())
return false;
LLT Ty = TLI.getOptimalMemOpLLT(Op, FuncAttributes);
@@ -2235,12 +2236,12 @@ bool llvm::canLowerMemCpyFamily(const MachineInstr &MI,
const auto &SrcMMO = **std::next(MI.memoperands_begin());
MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo();
unsigned Limit = TLI.getMaxStoresPerMemmove(OptSize);
- // FIXME: SelectionDAG always passes false for 'AllowOverlap', apparently
+ // FIXME: SelectionDAG always passes true for 'IsVolatile', apparently
// due to a bug in it's findOptimalMemOpLowering implementation. For now do
// the same thing here.
return findGISelOptimalMemOpLowering(
MemOps, Limit,
- MemOp::Copy(KnownLen, DstAlignCanChange, std::min(DstAlign, SrcAlign),
+ MemOp::Move(KnownLen, DstAlignCanChange, std::min(DstAlign, SrcAlign),
SrcAlign, /*IsVolatile=*/true),
DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
MF.getFunction().getAttributes(), TLI);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index c542f8e7cc20b..5329f240539f5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -9562,7 +9562,7 @@ static SDValue getMemmoveLoadsAndStores(
unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
if (!TLI.findOptimalMemOpLowering(
C, MemOps, Limit,
- MemOp::Copy(Size, DstAlignCanChange, DstAlign, SrcAlign, isVol),
+ MemOp::Move(Size, DstAlignCanChange, DstAlign, SrcAlign, isVol),
DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
MF.getFunction().getAttributes(), nullptr))
return SDValue();
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index cc3deaa83f63b..91e72ffd89303 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -224,7 +224,7 @@ bool TargetLowering::findOptimalMemOpLowering(
// Use the largest integer type whose alignment constraints are satisfied.
VT = MVT::LAST_INTEGER_VALUETYPE;
if (Op.isFixedDstAlign()) {
- bool LoadsFromSrc = Op.isMemcpy() && !Op.isMemcpyStrSrc();
+ bool LoadsFromSrc = Op.isMemcpyOrMemmove() && !Op.isMemcpyStrSrc();
while (VT != MVT::i8) {
unsigned VTSize = VT.getSizeInBits() / 8;
bool DstOk =
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 042ae5cc6b74c..db4d26db155cf 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -19370,7 +19370,7 @@ EVT ARMTargetLowering::getOptimalMemOpType(
LLVMContext &Context, const MemOp &Op,
const AttributeList &FuncAttributes) const {
// See if we can use NEON instructions for this...
- if ((Op.isMemcpy() || Op.isZeroMemset()) && Subtarget->hasNEON() &&
+ if ((Op.isMemcpyOrMemmove() || Op.isZeroMemset()) && Subtarget->hasNEON() &&
!FuncAttributes.hasFnAttr(Attribute::NoImplicitFloat)) {
unsigned Fast;
if (Op.size() >= 16 &&
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index ab48a0c4ba39c..76adf61692884 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -1229,7 +1229,8 @@ int ARMTTIImpl::getNumMemOps(const IntrinsicInst *I) const {
const Align DstAlign = MC->getDestAlign().valueOrOne();
const Align SrcAlign = MC->getSourceAlign().valueOrOne();
- MOp = MemOp::Copy(Size, /*DstAlignCanChange*/ false, DstAlign, SrcAlign,
+ // Use the most restrictive of memset, memcpy, memmove.
+ MOp = MemOp::Move(Size, /*DstAlignCanChange*/ false, DstAlign, SrcAlign,
/*IsVolatile*/ false);
DstAddrSpace = MC->getDestAddressSpace();
SrcAddrSpace = MC->getSourceAddressSpace();
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index e5ce36afb7b2f..2b4b95152481c 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -26777,7 +26777,7 @@ EVT RISCVTargetLowering::getOptimalMemOpType(
Align RequiredAlign(PreferredVT.getStoreSize());
if (Op.isFixedDstAlign())
RequiredAlign = std::min(RequiredAlign, Op.getDstAlign());
- if (Op.isMemcpy())
+ if (Op.isMemcpyOrMemmove())
RequiredAlign = std::min(RequiredAlign, Op.getSrcAlign());
PreferredVT = MVT::getIntegerVT(RequiredAlign.value() * 8);
}
diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
index 7c068115df481..fdd78a1dfbc23 100644
--- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
+++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
@@ -321,7 +321,8 @@ EVT X86TargetLowering::getOptimalMemOpType(
if (Subtarget.hasSSE1() && (Subtarget.is64Bit() || Subtarget.hasX87()) &&
(Subtarget.getPreferVectorWidth() >= 128))
return MVT::v4f32;
- } else if (((Op.isMemcpy() && !Op.isMemcpyStrSrc()) || Op.isZeroMemset()) &&
+ } else if (((Op.isMemcpyOrMemmove() && !Op.isMemcpyStrSrc()) ||
+ Op.isZeroMemset()) &&
Op.size() >= 8 && !Subtarget.is64Bit() && Subtarget.hasSSE2()) {
// Do not use f64 to lower memcpy if source is string constant. It's
// better to use i32 to avoid the loads.
``````````
</details>
https://github.com/llvm/llvm-project/pull/206005
More information about the llvm-commits
mailing list