[llvm] Reapply "[GlobalISel] Add a shared matcher for memcpy-family instructions (NFC)" (#202275) (PR #202298)
Cullen Rhodes via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 8 02:14:55 PDT 2026
https://github.com/c-rhodes created https://github.com/llvm/llvm-project/pull/202298
sanitizer-aarch64-linux-bootstrap-ubsan broke after #201766:
lab.llvm.org/buildbot/#/builders/85/builds/22356
failed tests:
LLVM :: CodeGen/AArch64/aarch64-mops.ll
LLVM :: CodeGen/AArch64/memsize-remarks.ll
The culprit is canLowerMemCpyFamily returning true for zero-length ops
before initializing IsVolatile. The memcpy-family lowering helpers don't
use IsVolatile, it's only needed while building the lowering plan with
findGISelOptimalMemOpLowering and shouldn't have been forwarded.
I've also check the other arguments and simplified alignment too.
This reverts commit 2de2edb943fe1b83d79bdffa03606eb8c5452e9b.
>From 87f73e254178a4ca8b936f4ee6408e5d44f6c6cc Mon Sep 17 00:00:00 2001
From: Cullen Rhodes <cullen.rhodes at arm.com>
Date: Mon, 8 Jun 2026 08:52:13 +0000
Subject: [PATCH 1/3] Reapply "[GlobalISel] Add a shared matcher for
memcpy-family instructions (NFC)" (#202275)
This reverts commit 2de2edb943fe1b83d79bdffa03606eb8c5452e9b.
---
.../llvm/CodeGen/GlobalISel/LegalizerHelper.h | 17 +-
llvm/include/llvm/CodeGen/GlobalISel/Utils.h | 15 ++
.../CodeGen/GlobalISel/LegalizerHelper.cpp | 217 +++---------------
llvm/lib/CodeGen/GlobalISel/Utils.cpp | 177 ++++++++++++++
4 files changed, 239 insertions(+), 187 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index 47d7da7a0eb52..c9cd1099d3e9e 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -305,13 +305,16 @@ class LegalizerHelper {
// Memcpy family legalization helpers.
LegalizeResult lowerMemset(MachineInstr &MI, Register Dst, Register Val,
uint64_t KnownLen, Align Alignment,
- bool IsVolatile);
+ bool IsVolatile, bool DstAlignCanChange,
+ ArrayRef<LLT> MemOps);
LegalizeResult lowerMemcpy(MachineInstr &MI, Register Dst, Register Src,
- uint64_t KnownLen, uint64_t Limit, Align DstAlign,
- Align SrcAlign, bool IsVolatile);
+ uint64_t KnownLen, Align DstAlign, Align SrcAlign,
+ bool IsVolatile, bool DstAlignCanChange,
+ ArrayRef<LLT> MemOps);
LegalizeResult lowerMemmove(MachineInstr &MI, Register Dst, Register Src,
uint64_t KnownLen, Align DstAlign, Align SrcAlign,
- bool IsVolatile);
+ bool IsVolatile, bool DstAlignCanChange,
+ ArrayRef<LLT> MemOps);
// Implements floating-point environment read/write via library function call.
LegalizeResult createGetStateLibcall(MachineInstr &MI,
@@ -570,6 +573,12 @@ class LegalizerHelper {
LLVM_ABI LegalizeResult lowerAbsDiffToMinMax(MachineInstr &MI);
LLVM_ABI LegalizeResult lowerFAbs(MachineInstr &MI);
LLVM_ABI LegalizeResult lowerVectorReduction(MachineInstr &MI);
+ LLVM_ABI LegalizeResult lowerMemCpyFamily(MachineInstr &MI, Register Dst,
+ Register Src, uint64_t KnownLen,
+ Align DstAlign, Align SrcAlign,
+ bool IsVolatile,
+ bool DstAlignCanChange,
+ ArrayRef<LLT> MemOps);
LLVM_ABI LegalizeResult lowerMemCpyFamily(MachineInstr &MI,
unsigned MaxLen = 0);
LLVM_ABI LegalizeResult lowerVAArg(MachineInstr &MI);
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
index 4689cc4df78b3..cfb7f376ff8e8 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
@@ -25,6 +25,7 @@
#include "llvm/Support/Compiler.h"
#include <cstdint>
+#include <vector>
namespace llvm {
@@ -203,6 +204,20 @@ LLVM_ABI std::optional<ValueAndVReg> getAnyConstantVRegValWithLookThrough(
Register VReg, const MachineRegisterInfo &MRI,
bool LookThroughInstrs = true, bool LookThroughAnyExt = false);
+using MemCpyFamilyLoweringInfo =
+ std::tuple<Register, Register, uint64_t, Align, Align, bool, bool,
+ std::vector<LLT>>;
+
+/// Matcher for memcpy-like instructions. For non-zero lengths, \p MemOps
+/// contains the load/store types to emit.
+LLVM_ABI bool canLowerMemCpyFamily(const MachineInstr &MI,
+ const MachineRegisterInfo &MRI,
+ unsigned MaxLen, Register &Dst,
+ Register &Src, uint64_t &KnownLen,
+ Align &DstAlign, Align &SrcAlign,
+ bool &IsVolatile, bool &DstAlignCanChange,
+ std::vector<LLT> &MemOps);
+
struct FPValueAndVReg {
APFloat Value;
Register VReg;
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index afe93f617a119..960b93c9a1369 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -10700,82 +10700,6 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerMulfix(MachineInstr &MI) {
return Legalized;
}
-static bool shouldLowerMemFuncForSize(const MachineFunction &MF) {
- // On Darwin, -Os means optimize for size without hurting performance, so
- // only really optimize for size when -Oz (MinSize) is used.
- if (MF.getTarget().getTargetTriple().isOSDarwin())
- return MF.getFunction().hasMinSize();
- return MF.getFunction().hasOptSize();
-}
-
-// Returns a list of types to use for memory op lowering in MemOps. A partial
-// port of findOptimalMemOpLowering in TargetLowering.
-static bool findGISelOptimalMemOpLowering(std::vector<LLT> &MemOps,
- unsigned Limit, const MemOp &Op,
- unsigned DstAS, unsigned SrcAS,
- const AttributeList &FuncAttributes,
- const TargetLowering &TLI) {
- if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign())
- return false;
-
- LLT Ty = TLI.getOptimalMemOpLLT(Op, FuncAttributes);
-
- if (Ty == LLT()) {
- // Use the largest scalar type whose alignment constraints are satisfied.
- // We only need to check DstAlign here as SrcAlign is always greater or
- // equal to DstAlign (or zero).
- Ty = LLT::integer(64);
- if (Op.isFixedDstAlign())
- while (Op.getDstAlign() < Ty.getSizeInBytes() &&
- !TLI.allowsMisalignedMemoryAccesses(Ty, DstAS, Op.getDstAlign()))
- Ty = LLT::integer(Ty.getSizeInBytes());
- assert(Ty.getSizeInBits() > 0 && "Could not find valid type");
- // FIXME: check for the largest legal type we can load/store to.
- }
-
- unsigned NumMemOps = 0;
- uint64_t Size = Op.size();
- while (Size) {
- unsigned TySize = Ty.getSizeInBytes();
- while (TySize > Size) {
- // For now, only use non-vector load / store's for the left-over pieces.
- LLT NewTy = Ty;
- // FIXME: check for mem op safety and legality of the types. Not all of
- // SDAGisms map cleanly to GISel concepts.
- if (NewTy.isVector())
- NewTy =
- NewTy.getSizeInBits() > 64 ? LLT::integer(64) : LLT::integer(32);
- NewTy = LLT::integer(llvm::bit_floor(NewTy.getSizeInBits() - 1));
- unsigned NewTySize = NewTy.getSizeInBytes();
- assert(NewTySize > 0 && "Could not find appropriate type");
-
- // If the new LLT cannot cover all of the remaining bits, then consider
- // issuing a (or a pair of) unaligned and overlapping load / store.
- unsigned Fast;
- // Need to get a VT equivalent for allowMisalignedMemoryAccesses().
- MVT VT = getMVTForLLT(Ty);
- if (NumMemOps && Op.allowOverlap() && NewTySize < Size &&
- TLI.allowsMisalignedMemoryAccesses(
- VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
- MachineMemOperand::MONone, &Fast) &&
- Fast)
- TySize = Size;
- else {
- Ty = NewTy;
- TySize = NewTySize;
- }
- }
-
- if (++NumMemOps > Limit)
- return false;
-
- MemOps.push_back(Ty);
- Size -= TySize;
- }
-
- return true;
-}
-
// Get a vectorized representation of the memset value operand, GISel edition.
static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB) {
MachineRegisterInfo &MRI = *MIB.getMRI();
@@ -10812,39 +10736,19 @@ static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB) {
LegalizerHelper::LegalizeResult
LegalizerHelper::lowerMemset(MachineInstr &MI, Register Dst, Register Val,
uint64_t KnownLen, Align Alignment,
- bool IsVolatile) {
+ bool IsVolatile, bool DstAlignCanChange,
+ ArrayRef<LLT> MemOps) {
auto &MF = *MI.getParent()->getParent();
const auto &TLI = *MF.getSubtarget().getTargetLowering();
auto &DL = MF.getDataLayout();
LLVMContext &C = MF.getFunction().getContext();
assert(KnownLen != 0 && "Have a zero length memset length!");
+ assert(!MemOps.empty() && "Expected at least one memory op");
- bool DstAlignCanChange = false;
MachineFrameInfo &MFI = MF.getFrameInfo();
- bool OptSize = shouldLowerMemFuncForSize(MF);
-
MachineInstr *FIDef = getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Dst, MRI);
- if (FIDef && !MFI.isFixedObjectIndex(FIDef->getOperand(1).getIndex()))
- DstAlignCanChange = true;
-
- unsigned Limit = TLI.getMaxStoresPerMemset(OptSize);
- std::vector<LLT> MemOps;
-
const auto &DstMMO = **MI.memoperands_begin();
- MachinePointerInfo DstPtrInfo = DstMMO.getPointerInfo();
-
- auto ValVRegAndVal = getIConstantVRegValWithLookThrough(Val, MRI);
- bool IsZeroVal = ValVRegAndVal && ValVRegAndVal->Value == 0;
-
- if (!findGISelOptimalMemOpLowering(MemOps, Limit,
- MemOp::Set(KnownLen, DstAlignCanChange,
- Alignment,
- /*IsZeroMemset=*/IsZeroVal,
- /*IsVolatile=*/IsVolatile),
- DstPtrInfo.getAddrSpace(), ~0u,
- MF.getFunction().getAttributes(), TLI))
- return UnableToLegalize;
if (DstAlignCanChange) {
// Get an estimate of the type from the LLT.
@@ -10924,41 +10828,26 @@ LegalizerHelper::lowerMemset(MachineInstr &MI, Register Dst, Register Val,
LegalizerHelper::LegalizeResult
LegalizerHelper::lowerMemcpy(MachineInstr &MI, Register Dst, Register Src,
- uint64_t KnownLen, uint64_t Limit, Align DstAlign,
- Align SrcAlign, bool IsVolatile) {
+ uint64_t KnownLen, Align DstAlign, Align SrcAlign,
+ bool IsVolatile, bool DstAlignCanChange,
+ ArrayRef<LLT> MemOps) {
auto &MF = *MI.getParent()->getParent();
- const auto &TLI = *MF.getSubtarget().getTargetLowering();
auto &DL = MF.getDataLayout();
LLVMContext &C = MF.getFunction().getContext();
assert(KnownLen != 0 && "Have a zero length memcpy length!");
+ assert(!MemOps.empty() && "Expected at least one memory op");
- bool DstAlignCanChange = false;
MachineFrameInfo &MFI = MF.getFrameInfo();
Align Alignment = std::min(DstAlign, SrcAlign);
-
MachineInstr *FIDef = getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Dst, MRI);
- if (FIDef && !MFI.isFixedObjectIndex(FIDef->getOperand(1).getIndex()))
- DstAlignCanChange = true;
// FIXME: infer better src pointer alignment like SelectionDAG does here.
// FIXME: also use the equivalent of isMemSrcFromConstant and alwaysinlining
// if the memcpy is in a tail call position.
- std::vector<LLT> MemOps;
-
const auto &DstMMO = **MI.memoperands_begin();
const auto &SrcMMO = **std::next(MI.memoperands_begin());
- MachinePointerInfo DstPtrInfo = DstMMO.getPointerInfo();
- MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo();
-
- if (!findGISelOptimalMemOpLowering(
- MemOps, Limit,
- MemOp::Copy(KnownLen, DstAlignCanChange, Alignment, SrcAlign,
- IsVolatile),
- DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
- MF.getFunction().getAttributes(), TLI))
- return UnableToLegalize;
if (DstAlignCanChange) {
// Get an estimate of the type from the LLT.
@@ -11033,41 +10922,20 @@ LegalizerHelper::lowerMemcpy(MachineInstr &MI, Register Dst, Register Src,
LegalizerHelper::LegalizeResult
LegalizerHelper::lowerMemmove(MachineInstr &MI, Register Dst, Register Src,
uint64_t KnownLen, Align DstAlign, Align SrcAlign,
- bool IsVolatile) {
+ bool IsVolatile, bool DstAlignCanChange,
+ ArrayRef<LLT> MemOps) {
auto &MF = *MI.getParent()->getParent();
- const auto &TLI = *MF.getSubtarget().getTargetLowering();
auto &DL = MF.getDataLayout();
LLVMContext &C = MF.getFunction().getContext();
assert(KnownLen != 0 && "Have a zero length memmove length!");
+ assert(!MemOps.empty() && "Expected at least one memory op");
- bool DstAlignCanChange = false;
MachineFrameInfo &MFI = MF.getFrameInfo();
- bool OptSize = shouldLowerMemFuncForSize(MF);
Align Alignment = std::min(DstAlign, SrcAlign);
-
MachineInstr *FIDef = getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Dst, MRI);
- if (FIDef && !MFI.isFixedObjectIndex(FIDef->getOperand(1).getIndex()))
- DstAlignCanChange = true;
-
- unsigned Limit = TLI.getMaxStoresPerMemmove(OptSize);
- std::vector<LLT> MemOps;
-
const auto &DstMMO = **MI.memoperands_begin();
const auto &SrcMMO = **std::next(MI.memoperands_begin());
- MachinePointerInfo DstPtrInfo = DstMMO.getPointerInfo();
- MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo();
-
- // FIXME: SelectionDAG always passes false for 'AllowOverlap', apparently due
- // to a bug in it's findOptimalMemOpLowering implementation. For now do the
- // same thing here.
- if (!findGISelOptimalMemOpLowering(
- MemOps, Limit,
- MemOp::Copy(KnownLen, DstAlignCanChange, Alignment, SrcAlign,
- /*IsVolatile*/ true),
- DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
- MF.getFunction().getAttributes(), TLI))
- return UnableToLegalize;
if (DstAlignCanChange) {
// Get an estimate of the type from the LLT.
@@ -11136,61 +11004,44 @@ LegalizerHelper::lowerMemmove(MachineInstr &MI, Register Dst, Register Src,
return Legalized;
}
-LegalizerHelper::LegalizeResult
-LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen) {
+LegalizerHelper::LegalizeResult LegalizerHelper::lowerMemCpyFamily(
+ MachineInstr &MI, Register Dst, Register Src, uint64_t KnownLen,
+ Align DstAlign, Align SrcAlign, bool IsVolatile, bool DstAlignCanChange,
+ ArrayRef<LLT> MemOps) {
const unsigned Opc = MI.getOpcode();
- // This combine is fairly complex so it's not written with a separate
- // matcher function.
assert((Opc == TargetOpcode::G_MEMCPY ||
Opc == TargetOpcode::G_MEMCPY_INLINE ||
Opc == TargetOpcode::G_MEMMOVE || Opc == TargetOpcode::G_MEMSET) &&
"Expected memcpy like instruction");
- auto MMOIt = MI.memoperands_begin();
- const MachineMemOperand *MemOp = *MMOIt;
-
- Align DstAlign = MemOp->getBaseAlign();
- Align SrcAlign;
- auto [Dst, Src, Len] = MI.getFirst3Regs();
-
- if (Opc != TargetOpcode::G_MEMSET) {
- assert(MMOIt != MI.memoperands_end() && "Expected a second MMO on MI");
- MemOp = *(++MMOIt);
- SrcAlign = MemOp->getBaseAlign();
- }
-
- // See if this is a constant length copy
- auto LenVRegAndVal = getIConstantVRegValWithLookThrough(Len, MRI);
- if (!LenVRegAndVal) {
- // FIXME: support dynamically sized G_MEMCPY_INLINE
- assert(Opc != TargetOpcode::G_MEMCPY_INLINE &&
- "inline memcpy with dynamic size is not yet supported");
- return UnableToLegalize;
- }
- uint64_t KnownLen = LenVRegAndVal->Value.getZExtValue();
-
if (KnownLen == 0) {
MI.eraseFromParent();
return Legalized;
}
- if (Opc != TargetOpcode::G_MEMCPY_INLINE && MaxLen && KnownLen > MaxLen)
- return UnableToLegalize;
-
- bool IsVolatile = MemOp->isVolatile();
if (Opc == TargetOpcode::G_MEMCPY || Opc == TargetOpcode::G_MEMCPY_INLINE) {
- auto &MF = *MI.getParent()->getParent();
- const auto &TLI = *MF.getSubtarget().getTargetLowering();
- bool OptSize = shouldLowerMemFuncForSize(MF);
- uint64_t Limit = Opc == TargetOpcode::G_MEMCPY_INLINE
- ? std::numeric_limits<uint64_t>::max()
- : TLI.getMaxStoresPerMemcpy(OptSize);
- return lowerMemcpy(MI, Dst, Src, KnownLen, Limit, DstAlign, SrcAlign,
- IsVolatile);
+ return lowerMemcpy(MI, Dst, Src, KnownLen, DstAlign, SrcAlign, IsVolatile,
+ DstAlignCanChange, MemOps);
}
if (Opc == TargetOpcode::G_MEMMOVE)
- return lowerMemmove(MI, Dst, Src, KnownLen, DstAlign, SrcAlign, IsVolatile);
+ return lowerMemmove(MI, Dst, Src, KnownLen, DstAlign, SrcAlign, IsVolatile,
+ DstAlignCanChange, MemOps);
if (Opc == TargetOpcode::G_MEMSET)
- return lowerMemset(MI, Dst, Src, KnownLen, DstAlign, IsVolatile);
+ return lowerMemset(MI, Dst, Src, KnownLen, DstAlign, IsVolatile,
+ DstAlignCanChange, MemOps);
return UnableToLegalize;
}
+
+LegalizerHelper::LegalizeResult
+LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen) {
+ Register Dst, Src;
+ uint64_t KnownLen;
+ Align DstAlign, SrcAlign;
+ bool IsVolatile, DstAlignCanChange;
+ std::vector<LLT> MemOps;
+ if (!canLowerMemCpyFamily(MI, MRI, MaxLen, Dst, Src, KnownLen, DstAlign,
+ SrcAlign, IsVolatile, DstAlignCanChange, MemOps))
+ return UnableToLegalize;
+ return lowerMemCpyFamily(MI, Dst, Src, KnownLen, DstAlign, SrcAlign,
+ IsVolatile, DstAlignCanChange, MemOps);
+}
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 7ae093282c49d..673ebd33f56aa 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -20,6 +20,7 @@
#include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
+#include "llvm/CodeGen/LowLevelTypeUtils.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
@@ -36,8 +37,10 @@
#include "llvm/Support/UndefPoison.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/Utils/SizeOpts.h"
+#include <limits>
#include <numeric>
#include <optional>
+#include <tuple>
#define DEBUG_TYPE "globalisel-utils"
@@ -2078,3 +2081,177 @@ llvm::GFConstant::getConstant(Register Const, const MachineRegisterInfo &MRI) {
return GFConstant(MayBeConstant->Value, GFConstantKind::Scalar);
}
+
+// Returns a list of types to use for memory op lowering in MemOps. A partial
+// port of findOptimalMemOpLowering in TargetLowering.
+static bool findGISelOptimalMemOpLowering(std::vector<LLT> &MemOps,
+ unsigned Limit, const MemOp &Op,
+ unsigned DstAS, unsigned SrcAS,
+ const AttributeList &FuncAttributes,
+ const TargetLowering &TLI) {
+ if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign())
+ return false;
+
+ LLT Ty = TLI.getOptimalMemOpLLT(Op, FuncAttributes);
+
+ if (Ty == LLT()) {
+ // Use the largest scalar type whose alignment constraints are satisfied.
+ // We only need to check DstAlign here as SrcAlign is always greater or
+ // equal to DstAlign (or zero).
+ Ty = LLT::integer(64);
+ if (Op.isFixedDstAlign())
+ while (Op.getDstAlign() < Ty.getSizeInBytes() &&
+ !TLI.allowsMisalignedMemoryAccesses(Ty, DstAS, Op.getDstAlign()))
+ Ty = LLT::integer(Ty.getSizeInBytes());
+ assert(Ty.getSizeInBits() > 0 && "Could not find valid type");
+ // FIXME: check for the largest legal type we can load/store to.
+ }
+
+ unsigned NumMemOps = 0;
+ uint64_t Size = Op.size();
+ while (Size) {
+ unsigned TySize = Ty.getSizeInBytes();
+ while (TySize > Size) {
+ // For now, only use non-vector load / store's for the left-over pieces.
+ LLT NewTy = Ty;
+ // FIXME: check for mem op safety and legality of the types. Not all of
+ // SDAGisms map cleanly to GISel concepts.
+ if (NewTy.isVector())
+ NewTy =
+ NewTy.getSizeInBits() > 64 ? LLT::integer(64) : LLT::integer(32);
+ NewTy = LLT::integer(llvm::bit_floor(NewTy.getSizeInBits() - 1));
+ unsigned NewTySize = NewTy.getSizeInBytes();
+ assert(NewTySize > 0 && "Could not find appropriate type");
+
+ // If the new LLT cannot cover all of the remaining bits, then consider
+ // issuing a (or a pair of) unaligned and overlapping load / store.
+ unsigned Fast;
+ // Need to get a VT equivalent for allowMisalignedMemoryAccesses().
+ MVT VT = getMVTForLLT(Ty);
+ if (NumMemOps && Op.allowOverlap() && NewTySize < Size &&
+ TLI.allowsMisalignedMemoryAccesses(
+ VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
+ MachineMemOperand::MONone, &Fast) &&
+ Fast)
+ TySize = Size;
+ else {
+ Ty = NewTy;
+ TySize = NewTySize;
+ }
+ }
+
+ if (++NumMemOps > Limit)
+ return false;
+
+ MemOps.push_back(Ty);
+ Size -= TySize;
+ }
+
+ return true;
+}
+
+bool llvm::canLowerMemCpyFamily(const MachineInstr &MI,
+ const MachineRegisterInfo &MRI, unsigned MaxLen,
+ Register &Dst, Register &Src,
+ uint64_t &KnownLen, Align &DstAlign,
+ Align &SrcAlign, bool &IsVolatile,
+ bool &DstAlignCanChange,
+ std::vector<LLT> &MemOps) {
+ const unsigned Opc = MI.getOpcode();
+ assert((Opc == TargetOpcode::G_MEMCPY ||
+ Opc == TargetOpcode::G_MEMCPY_INLINE ||
+ Opc == TargetOpcode::G_MEMMOVE || Opc == TargetOpcode::G_MEMSET) &&
+ "Expected memcpy like instruction");
+
+ auto MMOIt = MI.memoperands_begin();
+ const MachineMemOperand *MemOp = *MMOIt;
+
+ DstAlign = MemOp->getBaseAlign();
+ Register Len;
+ std::tie(Dst, Src, Len) = MI.getFirst3Regs();
+
+ if (Opc != TargetOpcode::G_MEMSET) {
+ assert(MMOIt != MI.memoperands_end() && "Expected a second MMO on MI");
+ MemOp = *(++MMOIt);
+ SrcAlign = MemOp->getBaseAlign();
+ }
+
+ // See if this is a constant length copy.
+ auto LenVRegAndVal = getIConstantVRegValWithLookThrough(Len, MRI);
+ if (!LenVRegAndVal) {
+ // FIXME: support dynamically sized G_MEMCPY_INLINE
+ assert(Opc != TargetOpcode::G_MEMCPY_INLINE &&
+ "inline memcpy with dynamic size is not yet supported");
+ return false;
+ }
+
+ KnownLen = LenVRegAndVal->Value.getZExtValue();
+ DstAlignCanChange = false;
+
+ if (KnownLen == 0)
+ return true;
+
+ if (Opc != TargetOpcode::G_MEMCPY_INLINE && MaxLen && KnownLen > MaxLen)
+ return false;
+
+ IsVolatile = MemOp->isVolatile();
+ const MachineFunction &MF = *MI.getParent()->getParent();
+ const auto &TLI = *MF.getSubtarget().getTargetLowering();
+ // On Darwin, -Os means optimize for size without hurting performance, so
+ // only really optimize for size when -Oz (MinSize) is used.
+ bool OptSize = MF.getTarget().getTargetTriple().isOSDarwin()
+ ? MF.getFunction().hasMinSize()
+ : MF.getFunction().hasOptSize();
+
+ const MachineFrameInfo &MFI = MF.getFrameInfo();
+ MachineInstr *FIDef = getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Dst, MRI);
+ if (FIDef && !MFI.isFixedObjectIndex(FIDef->getOperand(1).getIndex()))
+ DstAlignCanChange = true;
+
+ const auto &DstMMO = **MI.memoperands_begin();
+ MachinePointerInfo DstPtrInfo = DstMMO.getPointerInfo();
+
+ switch (Opc) {
+ case TargetOpcode::G_MEMCPY_INLINE:
+ case TargetOpcode::G_MEMCPY: {
+ const auto &SrcMMO = **std::next(MI.memoperands_begin());
+ MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo();
+ uint64_t Limit = Opc == TargetOpcode::G_MEMCPY_INLINE
+ ? std::numeric_limits<uint64_t>::max()
+ : TLI.getMaxStoresPerMemcpy(OptSize);
+ return findGISelOptimalMemOpLowering(
+ MemOps, Limit,
+ MemOp::Copy(KnownLen, DstAlignCanChange, std::min(DstAlign, SrcAlign),
+ SrcAlign, IsVolatile),
+ DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
+ MF.getFunction().getAttributes(), TLI);
+ }
+ case TargetOpcode::G_MEMMOVE: {
+ const auto &SrcMMO = **std::next(MI.memoperands_begin());
+ MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo();
+ unsigned Limit = TLI.getMaxStoresPerMemmove(OptSize);
+ // FIXME: SelectionDAG always passes false for 'AllowOverlap', apparently
+ // due to a bug in it's findOptimalMemOpLowering implementation. For now do
+ // the same thing here.
+ return findGISelOptimalMemOpLowering(
+ MemOps, Limit,
+ MemOp::Copy(KnownLen, DstAlignCanChange, std::min(DstAlign, SrcAlign),
+ SrcAlign, /*IsVolatile=*/true),
+ DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
+ MF.getFunction().getAttributes(), TLI);
+ }
+ case TargetOpcode::G_MEMSET: {
+ unsigned Limit = TLI.getMaxStoresPerMemset(OptSize);
+ auto ValVRegAndVal = getIConstantVRegValWithLookThrough(Src, MRI);
+ bool IsZeroVal = ValVRegAndVal && ValVRegAndVal->Value == 0;
+ return findGISelOptimalMemOpLowering(
+ MemOps, Limit,
+ MemOp::Set(KnownLen, DstAlignCanChange, DstAlign,
+ /*IsZeroMemset=*/IsZeroVal,
+ /*IsVolatile=*/IsVolatile),
+ DstPtrInfo.getAddrSpace(), ~0u, MF.getFunction().getAttributes(), TLI);
+ }
+ default:
+ llvm_unreachable("Unexpected memcpy-family opcode");
+ }
+}
>From f1d53a8b98843c53096484cb4a56a25940d9a3a5 Mon Sep 17 00:00:00 2001
From: Cullen Rhodes <cullen.rhodes at arm.com>
Date: Mon, 8 Jun 2026 06:50:11 +0000
Subject: [PATCH 2/3] [GlobalISel] Remove unused IsVolatile from memcpy-family
lowering
sanitizer-aarch64-linux-bootstrap-ubsan is failing after #201766:
failed tests:
LLVM :: CodeGen/AArch64/aarch64-mops.ll
LLVM :: CodeGen/AArch64/memsize-remarks.ll
The culprit is canLowerMemCpyFamily returning true for zero-length ops
before initializing IsVolatile. The memcpy-family lowering helpers don't
use IsVolatile, it's only needed while building the lowering plan with
findGISelOptimalMemOpLowering and shouldn't have been forwarded.
---
.../llvm/CodeGen/GlobalISel/LegalizerHelper.h | 10 ++----
llvm/include/llvm/CodeGen/GlobalISel/Utils.h | 17 ++++------
.../CodeGen/GlobalISel/LegalizerHelper.cpp | 32 +++++++++----------
llvm/lib/CodeGen/GlobalISel/Utils.cpp | 5 ++-
4 files changed, 27 insertions(+), 37 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index c9cd1099d3e9e..aa64709c6c6b4 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -305,16 +305,13 @@ class LegalizerHelper {
// Memcpy family legalization helpers.
LegalizeResult lowerMemset(MachineInstr &MI, Register Dst, Register Val,
uint64_t KnownLen, Align Alignment,
- bool IsVolatile, bool DstAlignCanChange,
- ArrayRef<LLT> MemOps);
+ bool DstAlignCanChange, ArrayRef<LLT> MemOps);
LegalizeResult lowerMemcpy(MachineInstr &MI, Register Dst, Register Src,
uint64_t KnownLen, Align DstAlign, Align SrcAlign,
- bool IsVolatile, bool DstAlignCanChange,
- ArrayRef<LLT> MemOps);
+ bool DstAlignCanChange, ArrayRef<LLT> MemOps);
LegalizeResult lowerMemmove(MachineInstr &MI, Register Dst, Register Src,
uint64_t KnownLen, Align DstAlign, Align SrcAlign,
- bool IsVolatile, bool DstAlignCanChange,
- ArrayRef<LLT> MemOps);
+ bool DstAlignCanChange, ArrayRef<LLT> MemOps);
// Implements floating-point environment read/write via library function call.
LegalizeResult createGetStateLibcall(MachineInstr &MI,
@@ -576,7 +573,6 @@ class LegalizerHelper {
LLVM_ABI LegalizeResult lowerMemCpyFamily(MachineInstr &MI, Register Dst,
Register Src, uint64_t KnownLen,
Align DstAlign, Align SrcAlign,
- bool IsVolatile,
bool DstAlignCanChange,
ArrayRef<LLT> MemOps);
LLVM_ABI LegalizeResult lowerMemCpyFamily(MachineInstr &MI,
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
index cfb7f376ff8e8..9e5b2315cad25 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
@@ -204,19 +204,16 @@ LLVM_ABI std::optional<ValueAndVReg> getAnyConstantVRegValWithLookThrough(
Register VReg, const MachineRegisterInfo &MRI,
bool LookThroughInstrs = true, bool LookThroughAnyExt = false);
-using MemCpyFamilyLoweringInfo =
- std::tuple<Register, Register, uint64_t, Align, Align, bool, bool,
- std::vector<LLT>>;
+using MemCpyFamilyLoweringInfo = std::tuple<Register, Register, uint64_t, Align,
+ Align, bool, std::vector<LLT>>;
/// Matcher for memcpy-like instructions. For non-zero lengths, \p MemOps
/// contains the load/store types to emit.
-LLVM_ABI bool canLowerMemCpyFamily(const MachineInstr &MI,
- const MachineRegisterInfo &MRI,
- unsigned MaxLen, Register &Dst,
- Register &Src, uint64_t &KnownLen,
- Align &DstAlign, Align &SrcAlign,
- bool &IsVolatile, bool &DstAlignCanChange,
- std::vector<LLT> &MemOps);
+LLVM_ABI bool
+canLowerMemCpyFamily(const MachineInstr &MI, const MachineRegisterInfo &MRI,
+ unsigned MaxLen, Register &Dst, Register &Src,
+ uint64_t &KnownLen, Align &DstAlign, Align &SrcAlign,
+ bool &DstAlignCanChange, std::vector<LLT> &MemOps);
struct FPValueAndVReg {
APFloat Value;
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 960b93c9a1369..de4d222b32793 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -10736,8 +10736,7 @@ static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB) {
LegalizerHelper::LegalizeResult
LegalizerHelper::lowerMemset(MachineInstr &MI, Register Dst, Register Val,
uint64_t KnownLen, Align Alignment,
- bool IsVolatile, bool DstAlignCanChange,
- ArrayRef<LLT> MemOps) {
+ bool DstAlignCanChange, ArrayRef<LLT> MemOps) {
auto &MF = *MI.getParent()->getParent();
const auto &TLI = *MF.getSubtarget().getTargetLowering();
auto &DL = MF.getDataLayout();
@@ -10829,8 +10828,7 @@ LegalizerHelper::lowerMemset(MachineInstr &MI, Register Dst, Register Val,
LegalizerHelper::LegalizeResult
LegalizerHelper::lowerMemcpy(MachineInstr &MI, Register Dst, Register Src,
uint64_t KnownLen, Align DstAlign, Align SrcAlign,
- bool IsVolatile, bool DstAlignCanChange,
- ArrayRef<LLT> MemOps) {
+ bool DstAlignCanChange, ArrayRef<LLT> MemOps) {
auto &MF = *MI.getParent()->getParent();
auto &DL = MF.getDataLayout();
LLVMContext &C = MF.getFunction().getContext();
@@ -10922,8 +10920,7 @@ LegalizerHelper::lowerMemcpy(MachineInstr &MI, Register Dst, Register Src,
LegalizerHelper::LegalizeResult
LegalizerHelper::lowerMemmove(MachineInstr &MI, Register Dst, Register Src,
uint64_t KnownLen, Align DstAlign, Align SrcAlign,
- bool IsVolatile, bool DstAlignCanChange,
- ArrayRef<LLT> MemOps) {
+ bool DstAlignCanChange, ArrayRef<LLT> MemOps) {
auto &MF = *MI.getParent()->getParent();
auto &DL = MF.getDataLayout();
LLVMContext &C = MF.getFunction().getContext();
@@ -11004,10 +11001,11 @@ LegalizerHelper::lowerMemmove(MachineInstr &MI, Register Dst, Register Src,
return Legalized;
}
-LegalizerHelper::LegalizeResult LegalizerHelper::lowerMemCpyFamily(
- MachineInstr &MI, Register Dst, Register Src, uint64_t KnownLen,
- Align DstAlign, Align SrcAlign, bool IsVolatile, bool DstAlignCanChange,
- ArrayRef<LLT> MemOps) {
+LegalizerHelper::LegalizeResult
+LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, Register Dst, Register Src,
+ uint64_t KnownLen, Align DstAlign,
+ Align SrcAlign, bool DstAlignCanChange,
+ ArrayRef<LLT> MemOps) {
const unsigned Opc = MI.getOpcode();
assert((Opc == TargetOpcode::G_MEMCPY ||
Opc == TargetOpcode::G_MEMCPY_INLINE ||
@@ -11020,15 +11018,15 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerMemCpyFamily(
}
if (Opc == TargetOpcode::G_MEMCPY || Opc == TargetOpcode::G_MEMCPY_INLINE) {
- return lowerMemcpy(MI, Dst, Src, KnownLen, DstAlign, SrcAlign, IsVolatile,
+ return lowerMemcpy(MI, Dst, Src, KnownLen, DstAlign, SrcAlign,
DstAlignCanChange, MemOps);
}
if (Opc == TargetOpcode::G_MEMMOVE)
- return lowerMemmove(MI, Dst, Src, KnownLen, DstAlign, SrcAlign, IsVolatile,
+ return lowerMemmove(MI, Dst, Src, KnownLen, DstAlign, SrcAlign,
DstAlignCanChange, MemOps);
if (Opc == TargetOpcode::G_MEMSET)
- return lowerMemset(MI, Dst, Src, KnownLen, DstAlign, IsVolatile,
- DstAlignCanChange, MemOps);
+ return lowerMemset(MI, Dst, Src, KnownLen, DstAlign, DstAlignCanChange,
+ MemOps);
return UnableToLegalize;
}
@@ -11037,11 +11035,11 @@ LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen) {
Register Dst, Src;
uint64_t KnownLen;
Align DstAlign, SrcAlign;
- bool IsVolatile, DstAlignCanChange;
+ bool DstAlignCanChange;
std::vector<LLT> MemOps;
if (!canLowerMemCpyFamily(MI, MRI, MaxLen, Dst, Src, KnownLen, DstAlign,
- SrcAlign, IsVolatile, DstAlignCanChange, MemOps))
+ SrcAlign, DstAlignCanChange, MemOps))
return UnableToLegalize;
return lowerMemCpyFamily(MI, Dst, Src, KnownLen, DstAlign, SrcAlign,
- IsVolatile, DstAlignCanChange, MemOps);
+ DstAlignCanChange, MemOps);
}
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 673ebd33f56aa..f91aaa0b74b16 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -2154,8 +2154,7 @@ bool llvm::canLowerMemCpyFamily(const MachineInstr &MI,
const MachineRegisterInfo &MRI, unsigned MaxLen,
Register &Dst, Register &Src,
uint64_t &KnownLen, Align &DstAlign,
- Align &SrcAlign, bool &IsVolatile,
- bool &DstAlignCanChange,
+ Align &SrcAlign, bool &DstAlignCanChange,
std::vector<LLT> &MemOps) {
const unsigned Opc = MI.getOpcode();
assert((Opc == TargetOpcode::G_MEMCPY ||
@@ -2194,7 +2193,7 @@ bool llvm::canLowerMemCpyFamily(const MachineInstr &MI,
if (Opc != TargetOpcode::G_MEMCPY_INLINE && MaxLen && KnownLen > MaxLen)
return false;
- IsVolatile = MemOp->isVolatile();
+ bool IsVolatile = MemOp->isVolatile();
const MachineFunction &MF = *MI.getParent()->getParent();
const auto &TLI = *MF.getSubtarget().getTargetLowering();
// On Darwin, -Os means optimize for size without hurting performance, so
>From 589f6dff58a77255916a1360e8f2d857bf8c6b8d Mon Sep 17 00:00:00 2001
From: Cullen Rhodes <cullen.rhodes at arm.com>
Date: Mon, 8 Jun 2026 08:57:55 +0000
Subject: [PATCH 3/3] Simplify alignment
---
.../llvm/CodeGen/GlobalISel/LegalizerHelper.h | 6 ++--
llvm/include/llvm/CodeGen/GlobalISel/Utils.h | 15 ++++----
.../CodeGen/GlobalISel/LegalizerHelper.cpp | 34 ++++++++-----------
llvm/lib/CodeGen/GlobalISel/Utils.cpp | 9 +++--
4 files changed, 32 insertions(+), 32 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index aa64709c6c6b4..8d6407a7c0115 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -307,10 +307,10 @@ class LegalizerHelper {
uint64_t KnownLen, Align Alignment,
bool DstAlignCanChange, ArrayRef<LLT> MemOps);
LegalizeResult lowerMemcpy(MachineInstr &MI, Register Dst, Register Src,
- uint64_t KnownLen, Align DstAlign, Align SrcAlign,
+ uint64_t KnownLen, Align Alignment,
bool DstAlignCanChange, ArrayRef<LLT> MemOps);
LegalizeResult lowerMemmove(MachineInstr &MI, Register Dst, Register Src,
- uint64_t KnownLen, Align DstAlign, Align SrcAlign,
+ uint64_t KnownLen, Align Alignment,
bool DstAlignCanChange, ArrayRef<LLT> MemOps);
// Implements floating-point environment read/write via library function call.
@@ -572,7 +572,7 @@ class LegalizerHelper {
LLVM_ABI LegalizeResult lowerVectorReduction(MachineInstr &MI);
LLVM_ABI LegalizeResult lowerMemCpyFamily(MachineInstr &MI, Register Dst,
Register Src, uint64_t KnownLen,
- Align DstAlign, Align SrcAlign,
+ Align Alignment,
bool DstAlignCanChange,
ArrayRef<LLT> MemOps);
LLVM_ABI LegalizeResult lowerMemCpyFamily(MachineInstr &MI,
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
index 9e5b2315cad25..bd97f7f4812f8 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
@@ -204,16 +204,17 @@ LLVM_ABI std::optional<ValueAndVReg> getAnyConstantVRegValWithLookThrough(
Register VReg, const MachineRegisterInfo &MRI,
bool LookThroughInstrs = true, bool LookThroughAnyExt = false);
-using MemCpyFamilyLoweringInfo = std::tuple<Register, Register, uint64_t, Align,
- Align, bool, std::vector<LLT>>;
+using MemCpyFamilyLoweringInfo =
+ std::tuple<Register, Register, uint64_t, Align, bool, std::vector<LLT>>;
/// Matcher for memcpy-like instructions. For non-zero lengths, \p MemOps
/// contains the load/store types to emit.
-LLVM_ABI bool
-canLowerMemCpyFamily(const MachineInstr &MI, const MachineRegisterInfo &MRI,
- unsigned MaxLen, Register &Dst, Register &Src,
- uint64_t &KnownLen, Align &DstAlign, Align &SrcAlign,
- bool &DstAlignCanChange, std::vector<LLT> &MemOps);
+LLVM_ABI bool canLowerMemCpyFamily(const MachineInstr &MI,
+ const MachineRegisterInfo &MRI,
+ unsigned MaxLen, Register &Dst,
+ Register &Src, uint64_t &KnownLen,
+ Align &Alignment, bool &DstAlignCanChange,
+ std::vector<LLT> &MemOps);
struct FPValueAndVReg {
APFloat Value;
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index de4d222b32793..7c1333e127deb 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -10827,7 +10827,7 @@ LegalizerHelper::lowerMemset(MachineInstr &MI, Register Dst, Register Val,
LegalizerHelper::LegalizeResult
LegalizerHelper::lowerMemcpy(MachineInstr &MI, Register Dst, Register Src,
- uint64_t KnownLen, Align DstAlign, Align SrcAlign,
+ uint64_t KnownLen, Align Alignment,
bool DstAlignCanChange, ArrayRef<LLT> MemOps) {
auto &MF = *MI.getParent()->getParent();
auto &DL = MF.getDataLayout();
@@ -10837,7 +10837,6 @@ LegalizerHelper::lowerMemcpy(MachineInstr &MI, Register Dst, Register Src,
assert(!MemOps.empty() && "Expected at least one memory op");
MachineFrameInfo &MFI = MF.getFrameInfo();
- Align Alignment = std::min(DstAlign, SrcAlign);
MachineInstr *FIDef = getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Dst, MRI);
// FIXME: infer better src pointer alignment like SelectionDAG does here.
@@ -10919,7 +10918,7 @@ LegalizerHelper::lowerMemcpy(MachineInstr &MI, Register Dst, Register Src,
LegalizerHelper::LegalizeResult
LegalizerHelper::lowerMemmove(MachineInstr &MI, Register Dst, Register Src,
- uint64_t KnownLen, Align DstAlign, Align SrcAlign,
+ uint64_t KnownLen, Align Alignment,
bool DstAlignCanChange, ArrayRef<LLT> MemOps) {
auto &MF = *MI.getParent()->getParent();
auto &DL = MF.getDataLayout();
@@ -10929,7 +10928,6 @@ LegalizerHelper::lowerMemmove(MachineInstr &MI, Register Dst, Register Src,
assert(!MemOps.empty() && "Expected at least one memory op");
MachineFrameInfo &MFI = MF.getFrameInfo();
- Align Alignment = std::min(DstAlign, SrcAlign);
MachineInstr *FIDef = getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Dst, MRI);
const auto &DstMMO = **MI.memoperands_begin();
const auto &SrcMMO = **std::next(MI.memoperands_begin());
@@ -11001,11 +10999,9 @@ LegalizerHelper::lowerMemmove(MachineInstr &MI, Register Dst, Register Src,
return Legalized;
}
-LegalizerHelper::LegalizeResult
-LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, Register Dst, Register Src,
- uint64_t KnownLen, Align DstAlign,
- Align SrcAlign, bool DstAlignCanChange,
- ArrayRef<LLT> MemOps) {
+LegalizerHelper::LegalizeResult LegalizerHelper::lowerMemCpyFamily(
+ MachineInstr &MI, Register Dst, Register Src, uint64_t KnownLen,
+ Align Alignment, bool DstAlignCanChange, ArrayRef<LLT> MemOps) {
const unsigned Opc = MI.getOpcode();
assert((Opc == TargetOpcode::G_MEMCPY ||
Opc == TargetOpcode::G_MEMCPY_INLINE ||
@@ -11018,14 +11014,14 @@ LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, Register Dst, Register Src,
}
if (Opc == TargetOpcode::G_MEMCPY || Opc == TargetOpcode::G_MEMCPY_INLINE) {
- return lowerMemcpy(MI, Dst, Src, KnownLen, DstAlign, SrcAlign,
- DstAlignCanChange, MemOps);
+ return lowerMemcpy(MI, Dst, Src, KnownLen, Alignment, DstAlignCanChange,
+ MemOps);
}
if (Opc == TargetOpcode::G_MEMMOVE)
- return lowerMemmove(MI, Dst, Src, KnownLen, DstAlign, SrcAlign,
- DstAlignCanChange, MemOps);
+ return lowerMemmove(MI, Dst, Src, KnownLen, Alignment, DstAlignCanChange,
+ MemOps);
if (Opc == TargetOpcode::G_MEMSET)
- return lowerMemset(MI, Dst, Src, KnownLen, DstAlign, DstAlignCanChange,
+ return lowerMemset(MI, Dst, Src, KnownLen, Alignment, DstAlignCanChange,
MemOps);
return UnableToLegalize;
}
@@ -11034,12 +11030,12 @@ LegalizerHelper::LegalizeResult
LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen) {
Register Dst, Src;
uint64_t KnownLen;
- Align DstAlign, SrcAlign;
+ Align Alignment;
bool DstAlignCanChange;
std::vector<LLT> MemOps;
- if (!canLowerMemCpyFamily(MI, MRI, MaxLen, Dst, Src, KnownLen, DstAlign,
- SrcAlign, DstAlignCanChange, MemOps))
+ if (!canLowerMemCpyFamily(MI, MRI, MaxLen, Dst, Src, KnownLen, Alignment,
+ DstAlignCanChange, MemOps))
return UnableToLegalize;
- return lowerMemCpyFamily(MI, Dst, Src, KnownLen, DstAlign, SrcAlign,
- DstAlignCanChange, MemOps);
+ return lowerMemCpyFamily(MI, Dst, Src, KnownLen, Alignment, DstAlignCanChange,
+ MemOps);
}
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index f91aaa0b74b16..4000f0027ab85 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -2153,8 +2153,8 @@ static bool findGISelOptimalMemOpLowering(std::vector<LLT> &MemOps,
bool llvm::canLowerMemCpyFamily(const MachineInstr &MI,
const MachineRegisterInfo &MRI, unsigned MaxLen,
Register &Dst, Register &Src,
- uint64_t &KnownLen, Align &DstAlign,
- Align &SrcAlign, bool &DstAlignCanChange,
+ uint64_t &KnownLen, Align &Alignment,
+ bool &DstAlignCanChange,
std::vector<LLT> &MemOps) {
const unsigned Opc = MI.getOpcode();
assert((Opc == TargetOpcode::G_MEMCPY ||
@@ -2165,7 +2165,9 @@ bool llvm::canLowerMemCpyFamily(const MachineInstr &MI,
auto MMOIt = MI.memoperands_begin();
const MachineMemOperand *MemOp = *MMOIt;
- DstAlign = MemOp->getBaseAlign();
+ Align DstAlign = MemOp->getBaseAlign();
+ Align SrcAlign;
+ Alignment = DstAlign;
Register Len;
std::tie(Dst, Src, Len) = MI.getFirst3Regs();
@@ -2173,6 +2175,7 @@ bool llvm::canLowerMemCpyFamily(const MachineInstr &MI,
assert(MMOIt != MI.memoperands_end() && "Expected a second MMO on MI");
MemOp = *(++MMOIt);
SrcAlign = MemOp->getBaseAlign();
+ Alignment = std::min(DstAlign, SrcAlign);
}
// See if this is a constant length copy.
More information about the llvm-commits
mailing list