[llvm] 42d52df - [SelectionDAG] Update MemOp to distinguish memmove from memcpy (#206005)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 14 04:29:32 PDT 2026
Author: Ömer Sinan Ağacan
Date: 2026-07-14T11:29:28Z
New Revision: 42d52dfb9ceae5cb261b6032e50fba512479c7ce
URL: https://github.com/llvm/llvm-project/commit/42d52dfb9ceae5cb261b6032e50fba512479c7ce
DIFF: https://github.com/llvm/llvm-project/commit/42d52dfb9ceae5cb261b6032e50fba512479c7ce.diff
LOG: [SelectionDAG] Update MemOp to distinguish memmove from memcpy (#206005)
Added:
Modified:
llvm/include/llvm/CodeGen/TargetLowering.h
llvm/lib/CodeGen/GlobalISel/Utils.cpp
llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
llvm/lib/Target/ARM/ARMISelLowering.cpp
llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
llvm/lib/Target/RISCV/RISCVISelLowering.cpp
llvm/lib/Target/X86/X86ISelLoweringCall.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index b8b073962cb29..9a6dd7735421e 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -119,21 +119,27 @@ enum Preference : uint8_t {
// MemOp models a memory operation, either memset or memcpy/memmove.
struct MemOp {
private:
+ enum class MemOpKind {
+ Memset,
+ MemsetWithZero, // memset the memory with zeros
+ Memcpy, // copy memory from source to destination, source and destination do
+ // not overlap
+ MemcpyStrSrc, // memcpy source is an in-register constant, so it does not
+ // need to be loaded
+ Memmove, // memmove: like memcpy, but source and destination regions may
+ // overlap
+ };
+
// Shared
uint64_t Size;
bool DstAlignCanChange; // true if destination alignment can satisfy any
// constraint.
Align DstAlign; // Specified alignment of the memory operation.
- bool AllowOverlap;
- // memset only
- bool IsMemset; // If setthis memory operation is a memset.
- bool ZeroMemset; // If set clears out memory with zeros.
- // memcpy only
- bool MemcpyStrSrc; // Indicates whether the memcpy source is an in-register
- // constant so it does not need to be loaded.
- Align SrcAlign; // Inferred alignment of the source or default value if the
- // memory operation does not need to load the value.
+ bool IsVolatile;
+ MemOpKind Kind;
+ Align SrcAlign; // Inferred alignment of the source or default value if the
+ // memory operation does not need to load the value.
public:
static MemOp Copy(uint64_t Size, bool DstAlignCanChange, Align DstAlign,
Align SrcAlign, bool IsVolatile,
@@ -142,10 +148,20 @@ struct MemOp {
Op.Size = Size;
Op.DstAlignCanChange = DstAlignCanChange;
Op.DstAlign = DstAlign;
- Op.AllowOverlap = !IsVolatile;
- Op.IsMemset = false;
- Op.ZeroMemset = false;
- Op.MemcpyStrSrc = MemcpyStrSrc;
+ Op.IsVolatile = IsVolatile;
+ Op.Kind = MemcpyStrSrc ? MemOpKind::MemcpyStrSrc : MemOpKind::Memcpy;
+ Op.SrcAlign = SrcAlign;
+ return Op;
+ }
+
+ static MemOp Move(uint64_t Size, bool DstAlignCanChange, Align DstAlign,
+ Align SrcAlign, bool IsVolatile) {
+ MemOp Op;
+ Op.Size = Size;
+ Op.DstAlignCanChange = DstAlignCanChange;
+ Op.DstAlign = DstAlign;
+ Op.IsVolatile = IsVolatile;
+ Op.Kind = MemOpKind::Memmove;
Op.SrcAlign = SrcAlign;
return Op;
}
@@ -156,10 +172,8 @@ struct MemOp {
Op.Size = Size;
Op.DstAlignCanChange = DstAlignCanChange;
Op.DstAlign = DstAlign;
- Op.AllowOverlap = !IsVolatile;
- Op.IsMemset = true;
- Op.ZeroMemset = IsZeroMemset;
- Op.MemcpyStrSrc = false;
+ Op.IsVolatile = IsVolatile;
+ Op.Kind = IsZeroMemset ? MemOpKind::MemsetWithZero : MemOpKind::Memset;
return Op;
}
@@ -169,19 +183,22 @@ struct MemOp {
return DstAlign;
}
bool isFixedDstAlign() const { return !DstAlignCanChange; }
- bool allowOverlap() const { return AllowOverlap; }
- bool isMemset() const { return IsMemset; }
- bool isMemcpy() const { return !IsMemset; }
- bool isMemcpyWithFixedDstAlign() const {
- return isMemcpy() && !DstAlignCanChange;
+ bool isVolatile() const { return IsVolatile; }
+ bool isMemset() const {
+ return Kind == MemOpKind::Memset || Kind == MemOpKind::MemsetWithZero;
+ }
+ bool isMemcpy() const {
+ return Kind == MemOpKind::Memcpy || Kind == MemOpKind::MemcpyStrSrc;
}
- bool isZeroMemset() const { return isMemset() && ZeroMemset; }
- bool isMemcpyStrSrc() const {
- assert(isMemcpy() && "Must be a memcpy");
- return MemcpyStrSrc;
+ bool isMemmove() const { return Kind == MemOpKind::Memmove; }
+ bool isMemcpyOrMemmove() const { return isMemcpy() || isMemmove(); }
+ bool isMemcpyOrMemmoveWithFixedDstAlign() const {
+ return isMemcpyOrMemmove() && !DstAlignCanChange;
}
+ bool isZeroMemset() const { return Kind == MemOpKind::MemsetWithZero; }
+ bool isMemcpyStrSrc() const { return Kind == MemOpKind::MemcpyStrSrc; }
Align getSrcAlign() const {
- assert(isMemcpy() && "Must be a memcpy");
+ assert(isMemcpyOrMemmove() && "Must be a memcpy or memmove");
return SrcAlign;
}
bool isSrcAligned(Align AlignCheck) const {
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index fbb54d8971502..7c79821f9651d 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -2089,7 +2089,8 @@ static bool findGISelOptimalMemOpLowering(std::vector<LLT> &MemOps,
unsigned DstAS, unsigned SrcAS,
const AttributeList &FuncAttributes,
const TargetLowering &TLI) {
- if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign())
+ if (Op.isMemcpyOrMemmoveWithFixedDstAlign() &&
+ Op.getSrcAlign() < Op.getDstAlign())
return false;
LLT Ty = TLI.getOptimalMemOpLLT(Op, FuncAttributes);
@@ -2128,7 +2129,7 @@ static bool findGISelOptimalMemOpLowering(std::vector<LLT> &MemOps,
unsigned Fast;
// Need to get a VT equivalent for allowMisalignedMemoryAccesses().
MVT VT = getMVTForLLT(Ty);
- if (NumMemOps && Op.allowOverlap() && NewTySize < Size &&
+ if (NumMemOps && !Op.isVolatile() && NewTySize < Size &&
TLI.allowsMisalignedMemoryAccesses(
VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
MachineMemOperand::MONone, &Fast) &&
@@ -2235,12 +2236,12 @@ bool llvm::canLowerMemCpyFamily(const MachineInstr &MI,
const auto &SrcMMO = **std::next(MI.memoperands_begin());
MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo();
unsigned Limit = TLI.getMaxStoresPerMemmove(OptSize);
- // FIXME: SelectionDAG always passes false for 'AllowOverlap', apparently
+ // FIXME: SelectionDAG always passes true for 'IsVolatile', apparently
// due to a bug in it's findOptimalMemOpLowering implementation. For now do
// the same thing here.
return findGISelOptimalMemOpLowering(
MemOps, Limit,
- MemOp::Copy(KnownLen, DstAlignCanChange, std::min(DstAlign, SrcAlign),
+ MemOp::Move(KnownLen, DstAlignCanChange, std::min(DstAlign, SrcAlign),
SrcAlign, /*IsVolatile=*/true),
DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
MF.getFunction().getAttributes(), TLI);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 626803ed92a40..9719d603e1869 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -9593,7 +9593,7 @@ static SDValue getMemmoveLoadsAndStores(
unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
if (!TLI.findOptimalMemOpLowering(
C, MemOps, Limit,
- MemOp::Copy(Size, DstAlignCanChange, DstAlign, SrcAlign, isVol),
+ MemOp::Move(Size, DstAlignCanChange, DstAlign, SrcAlign, isVol),
DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
MF.getFunction().getAttributes(), nullptr))
return SDValue();
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index bca34c5c347ee..2ecadac62ee80 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -224,7 +224,7 @@ bool TargetLowering::findOptimalMemOpLowering(
// Use the largest integer type whose alignment constraints are satisfied.
VT = MVT::LAST_INTEGER_VALUETYPE;
if (Op.isFixedDstAlign()) {
- bool LoadsFromSrc = Op.isMemcpy() && !Op.isMemcpyStrSrc();
+ bool LoadsFromSrc = Op.isMemcpyOrMemmove() && !Op.isMemcpyStrSrc();
while (VT != MVT::i8) {
unsigned VTSize = VT.getSizeInBits() / 8;
bool DstOk =
@@ -288,7 +288,7 @@ bool TargetLowering::findOptimalMemOpLowering(
// If the new VT cannot cover all of the remaining bits, then consider
// issuing a (or a pair of) unaligned and overlapping load / store.
unsigned Fast;
- if (NumMemOps && Op.allowOverlap() && NewVTSize < Size &&
+ if (NumMemOps && !Op.isVolatile() && NewVTSize < Size &&
allowsMisalignedMemoryAccesses(
VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
MachineMemOperand::MONone, &Fast) &&
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 35c46c3f0a75a..81cca300389e7 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -19405,7 +19405,7 @@ EVT ARMTargetLowering::getOptimalMemOpType(
LLVMContext &Context, const MemOp &Op,
const AttributeList &FuncAttributes) const {
// See if we can use NEON instructions for this...
- if ((Op.isMemcpy() || Op.isZeroMemset()) && Subtarget->hasNEON() &&
+ if ((Op.isMemcpyOrMemmove() || Op.isZeroMemset()) && Subtarget->hasNEON() &&
!FuncAttributes.hasFnAttr(Attribute::NoImplicitFloat)) {
unsigned Fast;
if (Op.size() >= 16 &&
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index 2b71426d861d0..7d4f9ae5b023b 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -1162,7 +1162,8 @@ int ARMTTIImpl::getNumMemOps(const IntrinsicInst *I) const {
const Align DstAlign = MC->getDestAlign().valueOrOne();
const Align SrcAlign = MC->getSourceAlign().valueOrOne();
- MOp = MemOp::Copy(Size, /*DstAlignCanChange*/ false, DstAlign, SrcAlign,
+ // Use the most restrictive of memset, memcpy, memmove.
+ MOp = MemOp::Move(Size, /*DstAlignCanChange*/ false, DstAlign, SrcAlign,
/*IsVolatile*/ false);
DstAddrSpace = MC->getDestAddressSpace();
SrcAddrSpace = MC->getSourceAddressSpace();
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 61a954099d9ea..738aa6dcf780a 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -27319,7 +27319,7 @@ EVT RISCVTargetLowering::getOptimalMemOpType(
Align RequiredAlign(PreferredVT.getStoreSize());
if (Op.isFixedDstAlign())
RequiredAlign = std::min(RequiredAlign, Op.getDstAlign());
- if (Op.isMemcpy())
+ if (Op.isMemcpyOrMemmove())
RequiredAlign = std::min(RequiredAlign, Op.getSrcAlign());
PreferredVT = MVT::getIntegerVT(RequiredAlign.value() * 8);
}
diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
index 819e0a023c1c5..8f14198cf268b 100644
--- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
+++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
@@ -321,7 +321,8 @@ EVT X86TargetLowering::getOptimalMemOpType(
if (Subtarget.hasSSE1() && (Subtarget.is64Bit() || Subtarget.hasX87()) &&
(Subtarget.getPreferVectorWidth() >= 128))
return MVT::v4f32;
- } else if (((Op.isMemcpy() && !Op.isMemcpyStrSrc()) || Op.isZeroMemset()) &&
+ } else if (((Op.isMemcpyOrMemmove() && !Op.isMemcpyStrSrc()) ||
+ Op.isZeroMemset()) &&
Op.size() >= 8 && !Subtarget.is64Bit() && Subtarget.hasSSE2()) {
// Do not use f64 to lower memcpy if source is string constant. It's
// better to use i32 to avoid the loads.
More information about the llvm-commits
mailing list