[llvm] [SelectionDAG] Update MemOp to distinguish memmove from memcpy (PR #206005)
Ömer Sinan Ağacan via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 14 03:47:53 PDT 2026
https://github.com/osa1 updated https://github.com/llvm/llvm-project/pull/206005
>From 05cea3ce46324796b4bc43466a43bba8c5c6c067 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= <omeragacan at gmail.com>
Date: Fri, 26 Jun 2026 09:13:41 +0100
Subject: [PATCH 1/6] [SelectionDAG] Update MemOp to distinguish memmove from
memcpy
---
llvm/include/llvm/CodeGen/TargetLowering.h | 72 ++++++++++++-------
llvm/lib/CodeGen/GlobalISel/Utils.cpp | 7 +-
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 2 +-
.../CodeGen/SelectionDAG/TargetLowering.cpp | 2 +-
llvm/lib/Target/ARM/ARMISelLowering.cpp | 2 +-
.../lib/Target/ARM/ARMTargetTransformInfo.cpp | 3 +-
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 2 +-
llvm/lib/Target/X86/X86ISelLoweringCall.cpp | 3 +-
8 files changed, 60 insertions(+), 33 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 7a10a693934fd..5877082823e9b 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -118,19 +118,25 @@ enum Preference : uint8_t {
// MemOp models a memory operation, either memset or memcpy/memmove.
struct MemOp {
private:
+ enum class MemOpKind {
+ Memset,
+ MemsetWithZero, // memset the memory with zeros
+ Memcpy, // copy memory from source to destination, source and destination do
+ // not overlap
+ MemcpyStrSrc, // memcpy source is an in-register constant, so it does not
+ // need to be loaded
+ Memmove, // memmove: like memcpy for source and destination regions may
+ // overlap
+ };
+
// Shared
uint64_t Size;
bool DstAlignCanChange; // true if destination alignment can satisfy any
// constraint.
Align DstAlign; // Specified alignment of the memory operation.
- bool AllowOverlap;
- // memset only
- bool IsMemset; // If setthis memory operation is a memset.
- bool ZeroMemset; // If set clears out memory with zeros.
- // memcpy only
- bool MemcpyStrSrc; // Indicates whether the memcpy source is an in-register
- // constant so it does not need to be loaded.
+ bool IsVolatile;
+ MemOpKind Kind;
Align SrcAlign; // Inferred alignment of the source or default value if the
// memory operation does not need to load the value.
public:
@@ -141,10 +147,20 @@ struct MemOp {
Op.Size = Size;
Op.DstAlignCanChange = DstAlignCanChange;
Op.DstAlign = DstAlign;
- Op.AllowOverlap = !IsVolatile;
- Op.IsMemset = false;
- Op.ZeroMemset = false;
- Op.MemcpyStrSrc = MemcpyStrSrc;
+ Op.IsVolatile = IsVolatile;
+ Op.Kind = MemcpyStrSrc ? MemOpKind::MemcpyStrSrc : MemOpKind::Memcpy;
+ Op.SrcAlign = SrcAlign;
+ return Op;
+ }
+
+ static MemOp Move(uint64_t Size, bool DstAlignCanChange, Align DstAlign,
+ Align SrcAlign, bool IsVolatile) {
+ MemOp Op;
+ Op.Size = Size;
+ Op.DstAlignCanChange = DstAlignCanChange;
+ Op.DstAlign = DstAlign;
+ Op.IsVolatile = IsVolatile;
+ Op.Kind = MemOpKind::Memmove;
Op.SrcAlign = SrcAlign;
return Op;
}
@@ -155,10 +171,8 @@ struct MemOp {
Op.Size = Size;
Op.DstAlignCanChange = DstAlignCanChange;
Op.DstAlign = DstAlign;
- Op.AllowOverlap = !IsVolatile;
- Op.IsMemset = true;
- Op.ZeroMemset = IsZeroMemset;
- Op.MemcpyStrSrc = false;
+ Op.IsVolatile = IsVolatile;
+ Op.Kind = IsZeroMemset ? MemOpKind::MemsetWithZero : MemOpKind::Memset;
return Op;
}
@@ -168,19 +182,29 @@ struct MemOp {
return DstAlign;
}
bool isFixedDstAlign() const { return !DstAlignCanChange; }
- bool allowOverlap() const { return AllowOverlap; }
- bool isMemset() const { return IsMemset; }
- bool isMemcpy() const { return !IsMemset; }
- bool isMemcpyWithFixedDstAlign() const {
- return isMemcpy() && !DstAlignCanChange;
+ bool allowOverlap() const { return !IsVolatile; }
+ bool isMemset() const {
+ return Kind == MemOpKind::Memset || Kind == MemOpKind::MemsetWithZero;
+ }
+ bool isMemcpy() const {
+ return Kind == MemOpKind::Memcpy || Kind == MemOpKind::MemcpyStrSrc;
+ }
+ bool isMemmove() const {
+ return Kind == MemOpKind::Memmove;
+ }
+ bool isMemcpyOrMemmove() const {
+ return isMemcpy() || isMemmove();
+ }
+ bool isMemcpyOrMemmoveWithFixedDstAlign() const {
+ return isMemcpyOrMemmove() && !DstAlignCanChange;
}
- bool isZeroMemset() const { return isMemset() && ZeroMemset; }
+ bool isZeroMemset() const { return Kind == MemOpKind::MemsetWithZero; }
bool isMemcpyStrSrc() const {
- assert(isMemcpy() && "Must be a memcpy");
- return MemcpyStrSrc;
+ assert(isMemcpyOrMemmove() && "Must be a memcpy or memmove");
+ return Kind == MemOpKind::MemcpyStrSrc;
}
Align getSrcAlign() const {
- assert(isMemcpy() && "Must be a memcpy");
+ assert(isMemcpyOrMemmove() && "Must be a memcpy or memmove");
return SrcAlign;
}
bool isSrcAligned(Align AlignCheck) const {
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index fbb54d8971502..7ef788614ae9d 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -2089,7 +2089,8 @@ static bool findGISelOptimalMemOpLowering(std::vector<LLT> &MemOps,
unsigned DstAS, unsigned SrcAS,
const AttributeList &FuncAttributes,
const TargetLowering &TLI) {
- if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign())
+ if (Op.isMemcpyOrMemmoveWithFixedDstAlign() &&
+ Op.getSrcAlign() < Op.getDstAlign())
return false;
LLT Ty = TLI.getOptimalMemOpLLT(Op, FuncAttributes);
@@ -2235,12 +2236,12 @@ bool llvm::canLowerMemCpyFamily(const MachineInstr &MI,
const auto &SrcMMO = **std::next(MI.memoperands_begin());
MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo();
unsigned Limit = TLI.getMaxStoresPerMemmove(OptSize);
- // FIXME: SelectionDAG always passes false for 'AllowOverlap', apparently
+ // FIXME: SelectionDAG always passes true for 'IsVolatile', apparently
// due to a bug in it's findOptimalMemOpLowering implementation. For now do
// the same thing here.
return findGISelOptimalMemOpLowering(
MemOps, Limit,
- MemOp::Copy(KnownLen, DstAlignCanChange, std::min(DstAlign, SrcAlign),
+ MemOp::Move(KnownLen, DstAlignCanChange, std::min(DstAlign, SrcAlign),
SrcAlign, /*IsVolatile=*/true),
DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
MF.getFunction().getAttributes(), TLI);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index c542f8e7cc20b..5329f240539f5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -9562,7 +9562,7 @@ static SDValue getMemmoveLoadsAndStores(
unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
if (!TLI.findOptimalMemOpLowering(
C, MemOps, Limit,
- MemOp::Copy(Size, DstAlignCanChange, DstAlign, SrcAlign, isVol),
+ MemOp::Move(Size, DstAlignCanChange, DstAlign, SrcAlign, isVol),
DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
MF.getFunction().getAttributes(), nullptr))
return SDValue();
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index cc3deaa83f63b..91e72ffd89303 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -224,7 +224,7 @@ bool TargetLowering::findOptimalMemOpLowering(
// Use the largest integer type whose alignment constraints are satisfied.
VT = MVT::LAST_INTEGER_VALUETYPE;
if (Op.isFixedDstAlign()) {
- bool LoadsFromSrc = Op.isMemcpy() && !Op.isMemcpyStrSrc();
+ bool LoadsFromSrc = Op.isMemcpyOrMemmove() && !Op.isMemcpyStrSrc();
while (VT != MVT::i8) {
unsigned VTSize = VT.getSizeInBits() / 8;
bool DstOk =
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 042ae5cc6b74c..db4d26db155cf 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -19370,7 +19370,7 @@ EVT ARMTargetLowering::getOptimalMemOpType(
LLVMContext &Context, const MemOp &Op,
const AttributeList &FuncAttributes) const {
// See if we can use NEON instructions for this...
- if ((Op.isMemcpy() || Op.isZeroMemset()) && Subtarget->hasNEON() &&
+ if ((Op.isMemcpyOrMemmove() || Op.isZeroMemset()) && Subtarget->hasNEON() &&
!FuncAttributes.hasFnAttr(Attribute::NoImplicitFloat)) {
unsigned Fast;
if (Op.size() >= 16 &&
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index ab48a0c4ba39c..76adf61692884 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -1229,7 +1229,8 @@ int ARMTTIImpl::getNumMemOps(const IntrinsicInst *I) const {
const Align DstAlign = MC->getDestAlign().valueOrOne();
const Align SrcAlign = MC->getSourceAlign().valueOrOne();
- MOp = MemOp::Copy(Size, /*DstAlignCanChange*/ false, DstAlign, SrcAlign,
+ // Use the most restrictive of memset, memcpy, memmove.
+ MOp = MemOp::Move(Size, /*DstAlignCanChange*/ false, DstAlign, SrcAlign,
/*IsVolatile*/ false);
DstAddrSpace = MC->getDestAddressSpace();
SrcAddrSpace = MC->getSourceAddressSpace();
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index e5ce36afb7b2f..2b4b95152481c 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -26777,7 +26777,7 @@ EVT RISCVTargetLowering::getOptimalMemOpType(
Align RequiredAlign(PreferredVT.getStoreSize());
if (Op.isFixedDstAlign())
RequiredAlign = std::min(RequiredAlign, Op.getDstAlign());
- if (Op.isMemcpy())
+ if (Op.isMemcpyOrMemmove())
RequiredAlign = std::min(RequiredAlign, Op.getSrcAlign());
PreferredVT = MVT::getIntegerVT(RequiredAlign.value() * 8);
}
diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
index 7c068115df481..fdd78a1dfbc23 100644
--- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
+++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
@@ -321,7 +321,8 @@ EVT X86TargetLowering::getOptimalMemOpType(
if (Subtarget.hasSSE1() && (Subtarget.is64Bit() || Subtarget.hasX87()) &&
(Subtarget.getPreferVectorWidth() >= 128))
return MVT::v4f32;
- } else if (((Op.isMemcpy() && !Op.isMemcpyStrSrc()) || Op.isZeroMemset()) &&
+ } else if (((Op.isMemcpyOrMemmove() && !Op.isMemcpyStrSrc()) ||
+ Op.isZeroMemset()) &&
Op.size() >= 8 && !Subtarget.is64Bit() && Subtarget.hasSSE2()) {
// Do not use f64 to lower memcpy if source is string constant. It's
// better to use i32 to avoid the loads.
>From a1756139ca3b37ff7dae31b2c61f114cde1c5505 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= <omeragacan at gmail.com>
Date: Fri, 26 Jun 2026 09:23:39 +0100
Subject: [PATCH 2/6] Formatting
---
llvm/include/llvm/CodeGen/TargetLowering.h | 16 ++++++----------
1 file changed, 6 insertions(+), 10 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 5877082823e9b..e6b98195ce54c 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -125,8 +125,8 @@ struct MemOp {
// not overlap
MemcpyStrSrc, // memcpy source is an in-register constant, so it does not
// need to be loaded
- Memmove, // memmove: like memcpy for source and destination regions may
- // overlap
+ Memmove, // memmove: like memcpy for source and destination regions may
+ // overlap
};
// Shared
@@ -137,8 +137,8 @@ struct MemOp {
bool IsVolatile;
MemOpKind Kind;
- Align SrcAlign; // Inferred alignment of the source or default value if the
- // memory operation does not need to load the value.
+ Align SrcAlign; // Inferred alignment of the source or default value if the
+ // memory operation does not need to load the value.
public:
static MemOp Copy(uint64_t Size, bool DstAlignCanChange, Align DstAlign,
Align SrcAlign, bool IsVolatile,
@@ -189,12 +189,8 @@ struct MemOp {
bool isMemcpy() const {
return Kind == MemOpKind::Memcpy || Kind == MemOpKind::MemcpyStrSrc;
}
- bool isMemmove() const {
- return Kind == MemOpKind::Memmove;
- }
- bool isMemcpyOrMemmove() const {
- return isMemcpy() || isMemmove();
- }
+ bool isMemmove() const { return Kind == MemOpKind::Memmove; }
+ bool isMemcpyOrMemmove() const { return isMemcpy() || isMemmove(); }
bool isMemcpyOrMemmoveWithFixedDstAlign() const {
return isMemcpyOrMemmove() && !DstAlignCanChange;
}
>From 4d19f99bf156e78a9c54492dc65785bef92868f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= <omeragacan at gmail.com>
Date: Fri, 26 Jun 2026 13:16:52 +0100
Subject: [PATCH 3/6] Rename allowOverlap --> isVolatile with negated return
value
---
llvm/include/llvm/CodeGen/TargetLowering.h | 2 +-
llvm/lib/CodeGen/GlobalISel/Utils.cpp | 2 +-
llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index e6b98195ce54c..2aaed893027af 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -182,7 +182,7 @@ struct MemOp {
return DstAlign;
}
bool isFixedDstAlign() const { return !DstAlignCanChange; }
- bool allowOverlap() const { return !IsVolatile; }
+ bool isVolatile() const { return IsVolatile; }
bool isMemset() const {
return Kind == MemOpKind::Memset || Kind == MemOpKind::MemsetWithZero;
}
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 7ef788614ae9d..7c79821f9651d 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -2129,7 +2129,7 @@ static bool findGISelOptimalMemOpLowering(std::vector<LLT> &MemOps,
unsigned Fast;
// Need to get a VT equivalent for allowMisalignedMemoryAccesses().
MVT VT = getMVTForLLT(Ty);
- if (NumMemOps && Op.allowOverlap() && NewTySize < Size &&
+ if (NumMemOps && !Op.isVolatile() && NewTySize < Size &&
TLI.allowsMisalignedMemoryAccesses(
VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
MachineMemOperand::MONone, &Fast) &&
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 91e72ffd89303..19fa487bbcfb8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -288,7 +288,7 @@ bool TargetLowering::findOptimalMemOpLowering(
// If the new VT cannot cover all of the remaining bits, then consider
// issuing a (or a pair of) unaligned and overlapping load / store.
unsigned Fast;
- if (NumMemOps && Op.allowOverlap() && NewVTSize < Size &&
+ if (NumMemOps && !Op.isVolatile() && NewVTSize < Size &&
allowsMisalignedMemoryAccesses(
VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
MachineMemOperand::MONone, &Fast) &&
>From 592a19c7854501f643583411d1390841feb11215 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= <omeragacan at gmail.com>
Date: Tue, 30 Jun 2026 09:53:40 +0100
Subject: [PATCH 4/6] Fix typo
---
llvm/include/llvm/CodeGen/TargetLowering.h | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index ebc1b17a2872b..129d5b6e59e37 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -125,8 +125,8 @@ struct MemOp {
// not overlap
MemcpyStrSrc, // memcpy source is an in-register constant, so it does not
// need to be loaded
- Memmove, // memmove: like memcpy for source and destination regions may
- // overlap
+ Memmove, // memmove: like memcpy, but source and destination regions may
+ // overlap
};
// Shared
>From 95dd9c70c8b6820b990afc916656b97ac81f42ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= <omeragacan at gmail.com>
Date: Tue, 14 Jul 2026 11:37:26 +0100
Subject: [PATCH 5/6] Address review
---
llvm/include/llvm/CodeGen/TargetLowering.h | 1 -
1 file changed, 1 deletion(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index b139ef42ab322..7ae62df1d4d9a 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -197,7 +197,6 @@ struct MemOp {
}
bool isZeroMemset() const { return Kind == MemOpKind::MemsetWithZero; }
bool isMemcpyStrSrc() const {
- assert(isMemcpyOrMemmove() && "Must be a memcpy or memmove");
return Kind == MemOpKind::MemcpyStrSrc;
}
Align getSrcAlign() const {
>From eaf1a007ce40d9114254f125403b0a87a7bd5b86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= <omeragacan at gmail.com>
Date: Tue, 14 Jul 2026 11:47:37 +0100
Subject: [PATCH 6/6] Fix formatting
---
llvm/include/llvm/CodeGen/TargetLowering.h | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 7ae62df1d4d9a..9a6dd7735421e 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -196,9 +196,7 @@ struct MemOp {
return isMemcpyOrMemmove() && !DstAlignCanChange;
}
bool isZeroMemset() const { return Kind == MemOpKind::MemsetWithZero; }
- bool isMemcpyStrSrc() const {
- return Kind == MemOpKind::MemcpyStrSrc;
- }
+ bool isMemcpyStrSrc() const { return Kind == MemOpKind::MemcpyStrSrc; }
Align getSrcAlign() const {
assert(isMemcpyOrMemmove() && "Must be a memcpy or memmove");
return SrcAlign;
More information about the llvm-commits
mailing list