[llvm-branch-commits] [llvm] b7afecb - Revert "[GlobalISel] Add a shared matcher for memcpy-family instructions (NFC…"
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Jun 8 00:18:51 PDT 2026
Author: Cullen Rhodes
Date: 2026-06-08T08:18:46+01:00
New Revision: b7afecbf9bd698d2d5ce1546f54e98b0bdb68c49
URL: https://github.com/llvm/llvm-project/commit/b7afecbf9bd698d2d5ce1546f54e98b0bdb68c49
DIFF: https://github.com/llvm/llvm-project/commit/b7afecbf9bd698d2d5ce1546f54e98b0bdb68c49.diff
LOG: Revert "[GlobalISel] Add a shared matcher for memcpy-family instructions (NFC…"
This reverts commit 57d94c90ec4d69d8d5a6a755ad40155b01531ed1.
Added:
Modified:
llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
llvm/include/llvm/CodeGen/GlobalISel/Utils.h
llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
llvm/lib/CodeGen/GlobalISel/Utils.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index c9cd1099d3e9e..47d7da7a0eb52 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -305,16 +305,13 @@ class LegalizerHelper {
// Memcpy family legalization helpers.
LegalizeResult lowerMemset(MachineInstr &MI, Register Dst, Register Val,
uint64_t KnownLen, Align Alignment,
- bool IsVolatile, bool DstAlignCanChange,
- ArrayRef<LLT> MemOps);
+ bool IsVolatile);
LegalizeResult lowerMemcpy(MachineInstr &MI, Register Dst, Register Src,
- uint64_t KnownLen, Align DstAlign, Align SrcAlign,
- bool IsVolatile, bool DstAlignCanChange,
- ArrayRef<LLT> MemOps);
+ uint64_t KnownLen, uint64_t Limit, Align DstAlign,
+ Align SrcAlign, bool IsVolatile);
LegalizeResult lowerMemmove(MachineInstr &MI, Register Dst, Register Src,
uint64_t KnownLen, Align DstAlign, Align SrcAlign,
- bool IsVolatile, bool DstAlignCanChange,
- ArrayRef<LLT> MemOps);
+ bool IsVolatile);
// Implements floating-point environment read/write via library function call.
LegalizeResult createGetStateLibcall(MachineInstr &MI,
@@ -573,12 +570,6 @@ class LegalizerHelper {
LLVM_ABI LegalizeResult lowerAbsDiffToMinMax(MachineInstr &MI);
LLVM_ABI LegalizeResult lowerFAbs(MachineInstr &MI);
LLVM_ABI LegalizeResult lowerVectorReduction(MachineInstr &MI);
- LLVM_ABI LegalizeResult lowerMemCpyFamily(MachineInstr &MI, Register Dst,
- Register Src, uint64_t KnownLen,
- Align DstAlign, Align SrcAlign,
- bool IsVolatile,
- bool DstAlignCanChange,
- ArrayRef<LLT> MemOps);
LLVM_ABI LegalizeResult lowerMemCpyFamily(MachineInstr &MI,
unsigned MaxLen = 0);
LLVM_ABI LegalizeResult lowerVAArg(MachineInstr &MI);
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
index cfb7f376ff8e8..4689cc4df78b3 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
@@ -25,7 +25,6 @@
#include "llvm/Support/Compiler.h"
#include <cstdint>
-#include <vector>
namespace llvm {
@@ -204,20 +203,6 @@ LLVM_ABI std::optional<ValueAndVReg> getAnyConstantVRegValWithLookThrough(
Register VReg, const MachineRegisterInfo &MRI,
bool LookThroughInstrs = true, bool LookThroughAnyExt = false);
-using MemCpyFamilyLoweringInfo =
- std::tuple<Register, Register, uint64_t, Align, Align, bool, bool,
- std::vector<LLT>>;
-
-/// Matcher for memcpy-like instructions. For non-zero lengths, \p MemOps
-/// contains the load/store types to emit.
-LLVM_ABI bool canLowerMemCpyFamily(const MachineInstr &MI,
- const MachineRegisterInfo &MRI,
- unsigned MaxLen, Register &Dst,
- Register &Src, uint64_t &KnownLen,
- Align &DstAlign, Align &SrcAlign,
- bool &IsVolatile, bool &DstAlignCanChange,
- std::vector<LLT> &MemOps);
-
struct FPValueAndVReg {
APFloat Value;
Register VReg;
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 960b93c9a1369..afe93f617a119 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -10700,6 +10700,82 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerMulfix(MachineInstr &MI) {
return Legalized;
}
+static bool shouldLowerMemFuncForSize(const MachineFunction &MF) {
+ // On Darwin, -Os means optimize for size without hurting performance, so
+ // only really optimize for size when -Oz (MinSize) is used.
+ if (MF.getTarget().getTargetTriple().isOSDarwin())
+ return MF.getFunction().hasMinSize();
+ return MF.getFunction().hasOptSize();
+}
+
+// Returns a list of types to use for memory op lowering in MemOps. A partial
+// port of findOptimalMemOpLowering in TargetLowering.
+static bool findGISelOptimalMemOpLowering(std::vector<LLT> &MemOps,
+ unsigned Limit, const MemOp &Op,
+ unsigned DstAS, unsigned SrcAS,
+ const AttributeList &FuncAttributes,
+ const TargetLowering &TLI) {
+ if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign())
+ return false;
+
+ LLT Ty = TLI.getOptimalMemOpLLT(Op, FuncAttributes);
+
+ if (Ty == LLT()) {
+ // Use the largest scalar type whose alignment constraints are satisfied.
+ // We only need to check DstAlign here as SrcAlign is always greater or
+ // equal to DstAlign (or zero).
+ Ty = LLT::integer(64);
+ if (Op.isFixedDstAlign())
+ while (Op.getDstAlign() < Ty.getSizeInBytes() &&
+ !TLI.allowsMisalignedMemoryAccesses(Ty, DstAS, Op.getDstAlign()))
+ Ty = LLT::integer(Ty.getSizeInBytes());
+ assert(Ty.getSizeInBits() > 0 && "Could not find valid type");
+ // FIXME: check for the largest legal type we can load/store to.
+ }
+
+ unsigned NumMemOps = 0;
+ uint64_t Size = Op.size();
+ while (Size) {
+ unsigned TySize = Ty.getSizeInBytes();
+ while (TySize > Size) {
+ // For now, only use non-vector load / store's for the left-over pieces.
+ LLT NewTy = Ty;
+ // FIXME: check for mem op safety and legality of the types. Not all of
+ // SDAGisms map cleanly to GISel concepts.
+ if (NewTy.isVector())
+ NewTy =
+ NewTy.getSizeInBits() > 64 ? LLT::integer(64) : LLT::integer(32);
+ NewTy = LLT::integer(llvm::bit_floor(NewTy.getSizeInBits() - 1));
+ unsigned NewTySize = NewTy.getSizeInBytes();
+ assert(NewTySize > 0 && "Could not find appropriate type");
+
+ // If the new LLT cannot cover all of the remaining bits, then consider
+ // issuing a (or a pair of) unaligned and overlapping load / store.
+ unsigned Fast;
+ // Need to get a VT equivalent for allowMisalignedMemoryAccesses().
+ MVT VT = getMVTForLLT(Ty);
+ if (NumMemOps && Op.allowOverlap() && NewTySize < Size &&
+ TLI.allowsMisalignedMemoryAccesses(
+ VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
+ MachineMemOperand::MONone, &Fast) &&
+ Fast)
+ TySize = Size;
+ else {
+ Ty = NewTy;
+ TySize = NewTySize;
+ }
+ }
+
+ if (++NumMemOps > Limit)
+ return false;
+
+ MemOps.push_back(Ty);
+ Size -= TySize;
+ }
+
+ return true;
+}
+
// Get a vectorized representation of the memset value operand, GISel edition.
static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB) {
MachineRegisterInfo &MRI = *MIB.getMRI();
@@ -10736,19 +10812,39 @@ static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB) {
LegalizerHelper::LegalizeResult
LegalizerHelper::lowerMemset(MachineInstr &MI, Register Dst, Register Val,
uint64_t KnownLen, Align Alignment,
- bool IsVolatile, bool DstAlignCanChange,
- ArrayRef<LLT> MemOps) {
+ bool IsVolatile) {
auto &MF = *MI.getParent()->getParent();
const auto &TLI = *MF.getSubtarget().getTargetLowering();
auto &DL = MF.getDataLayout();
LLVMContext &C = MF.getFunction().getContext();
assert(KnownLen != 0 && "Have a zero length memset length!");
- assert(!MemOps.empty() && "Expected at least one memory op");
+ bool DstAlignCanChange = false;
MachineFrameInfo &MFI = MF.getFrameInfo();
+ bool OptSize = shouldLowerMemFuncForSize(MF);
+
MachineInstr *FIDef = getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Dst, MRI);
+ if (FIDef && !MFI.isFixedObjectIndex(FIDef->getOperand(1).getIndex()))
+ DstAlignCanChange = true;
+
+ unsigned Limit = TLI.getMaxStoresPerMemset(OptSize);
+ std::vector<LLT> MemOps;
+
const auto &DstMMO = **MI.memoperands_begin();
+ MachinePointerInfo DstPtrInfo = DstMMO.getPointerInfo();
+
+ auto ValVRegAndVal = getIConstantVRegValWithLookThrough(Val, MRI);
+ bool IsZeroVal = ValVRegAndVal && ValVRegAndVal->Value == 0;
+
+ if (!findGISelOptimalMemOpLowering(MemOps, Limit,
+ MemOp::Set(KnownLen, DstAlignCanChange,
+ Alignment,
+ /*IsZeroMemset=*/IsZeroVal,
+ /*IsVolatile=*/IsVolatile),
+ DstPtrInfo.getAddrSpace(), ~0u,
+ MF.getFunction().getAttributes(), TLI))
+ return UnableToLegalize;
if (DstAlignCanChange) {
// Get an estimate of the type from the LLT.
@@ -10828,26 +10924,41 @@ LegalizerHelper::lowerMemset(MachineInstr &MI, Register Dst, Register Val,
LegalizerHelper::LegalizeResult
LegalizerHelper::lowerMemcpy(MachineInstr &MI, Register Dst, Register Src,
- uint64_t KnownLen, Align DstAlign, Align SrcAlign,
- bool IsVolatile, bool DstAlignCanChange,
- ArrayRef<LLT> MemOps) {
+ uint64_t KnownLen, uint64_t Limit, Align DstAlign,
+ Align SrcAlign, bool IsVolatile) {
auto &MF = *MI.getParent()->getParent();
+ const auto &TLI = *MF.getSubtarget().getTargetLowering();
auto &DL = MF.getDataLayout();
LLVMContext &C = MF.getFunction().getContext();
assert(KnownLen != 0 && "Have a zero length memcpy length!");
- assert(!MemOps.empty() && "Expected at least one memory op");
+ bool DstAlignCanChange = false;
MachineFrameInfo &MFI = MF.getFrameInfo();
Align Alignment = std::min(DstAlign, SrcAlign);
+
MachineInstr *FIDef = getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Dst, MRI);
+ if (FIDef && !MFI.isFixedObjectIndex(FIDef->getOperand(1).getIndex()))
+ DstAlignCanChange = true;
// FIXME: infer better src pointer alignment like SelectionDAG does here.
// FIXME: also use the equivalent of isMemSrcFromConstant and alwaysinlining
// if the memcpy is in a tail call position.
+ std::vector<LLT> MemOps;
+
const auto &DstMMO = **MI.memoperands_begin();
const auto &SrcMMO = **std::next(MI.memoperands_begin());
+ MachinePointerInfo DstPtrInfo = DstMMO.getPointerInfo();
+ MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo();
+
+ if (!findGISelOptimalMemOpLowering(
+ MemOps, Limit,
+ MemOp::Copy(KnownLen, DstAlignCanChange, Alignment, SrcAlign,
+ IsVolatile),
+ DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
+ MF.getFunction().getAttributes(), TLI))
+ return UnableToLegalize;
if (DstAlignCanChange) {
// Get an estimate of the type from the LLT.
@@ -10922,20 +11033,41 @@ LegalizerHelper::lowerMemcpy(MachineInstr &MI, Register Dst, Register Src,
LegalizerHelper::LegalizeResult
LegalizerHelper::lowerMemmove(MachineInstr &MI, Register Dst, Register Src,
uint64_t KnownLen, Align DstAlign, Align SrcAlign,
- bool IsVolatile, bool DstAlignCanChange,
- ArrayRef<LLT> MemOps) {
+ bool IsVolatile) {
auto &MF = *MI.getParent()->getParent();
+ const auto &TLI = *MF.getSubtarget().getTargetLowering();
auto &DL = MF.getDataLayout();
LLVMContext &C = MF.getFunction().getContext();
assert(KnownLen != 0 && "Have a zero length memmove length!");
- assert(!MemOps.empty() && "Expected at least one memory op");
+ bool DstAlignCanChange = false;
MachineFrameInfo &MFI = MF.getFrameInfo();
+ bool OptSize = shouldLowerMemFuncForSize(MF);
Align Alignment = std::min(DstAlign, SrcAlign);
+
MachineInstr *FIDef = getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Dst, MRI);
+ if (FIDef && !MFI.isFixedObjectIndex(FIDef->getOperand(1).getIndex()))
+ DstAlignCanChange = true;
+
+ unsigned Limit = TLI.getMaxStoresPerMemmove(OptSize);
+ std::vector<LLT> MemOps;
+
const auto &DstMMO = **MI.memoperands_begin();
const auto &SrcMMO = **std::next(MI.memoperands_begin());
+ MachinePointerInfo DstPtrInfo = DstMMO.getPointerInfo();
+ MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo();
+
+ // FIXME: SelectionDAG always passes false for 'AllowOverlap', apparently due
+ // to a bug in it's findOptimalMemOpLowering implementation. For now do the
+ // same thing here.
+ if (!findGISelOptimalMemOpLowering(
+ MemOps, Limit,
+ MemOp::Copy(KnownLen, DstAlignCanChange, Alignment, SrcAlign,
+ /*IsVolatile*/ true),
+ DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
+ MF.getFunction().getAttributes(), TLI))
+ return UnableToLegalize;
if (DstAlignCanChange) {
// Get an estimate of the type from the LLT.
@@ -11004,44 +11136,61 @@ LegalizerHelper::lowerMemmove(MachineInstr &MI, Register Dst, Register Src,
return Legalized;
}
-LegalizerHelper::LegalizeResult LegalizerHelper::lowerMemCpyFamily(
- MachineInstr &MI, Register Dst, Register Src, uint64_t KnownLen,
- Align DstAlign, Align SrcAlign, bool IsVolatile, bool DstAlignCanChange,
- ArrayRef<LLT> MemOps) {
+LegalizerHelper::LegalizeResult
+LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen) {
const unsigned Opc = MI.getOpcode();
+ // This combine is fairly complex so it's not written with a separate
+ // matcher function.
assert((Opc == TargetOpcode::G_MEMCPY ||
Opc == TargetOpcode::G_MEMCPY_INLINE ||
Opc == TargetOpcode::G_MEMMOVE || Opc == TargetOpcode::G_MEMSET) &&
"Expected memcpy like instruction");
+ auto MMOIt = MI.memoperands_begin();
+ const MachineMemOperand *MemOp = *MMOIt;
+
+ Align DstAlign = MemOp->getBaseAlign();
+ Align SrcAlign;
+ auto [Dst, Src, Len] = MI.getFirst3Regs();
+
+ if (Opc != TargetOpcode::G_MEMSET) {
+ assert(MMOIt != MI.memoperands_end() && "Expected a second MMO on MI");
+ MemOp = *(++MMOIt);
+ SrcAlign = MemOp->getBaseAlign();
+ }
+
+ // See if this is a constant length copy
+ auto LenVRegAndVal = getIConstantVRegValWithLookThrough(Len, MRI);
+ if (!LenVRegAndVal) {
+ // FIXME: support dynamically sized G_MEMCPY_INLINE
+ assert(Opc != TargetOpcode::G_MEMCPY_INLINE &&
+ "inline memcpy with dynamic size is not yet supported");
+ return UnableToLegalize;
+ }
+ uint64_t KnownLen = LenVRegAndVal->Value.getZExtValue();
+
if (KnownLen == 0) {
MI.eraseFromParent();
return Legalized;
}
+ if (Opc != TargetOpcode::G_MEMCPY_INLINE && MaxLen && KnownLen > MaxLen)
+ return UnableToLegalize;
+
+ bool IsVolatile = MemOp->isVolatile();
if (Opc == TargetOpcode::G_MEMCPY || Opc == TargetOpcode::G_MEMCPY_INLINE) {
- return lowerMemcpy(MI, Dst, Src, KnownLen, DstAlign, SrcAlign, IsVolatile,
- DstAlignCanChange, MemOps);
+ auto &MF = *MI.getParent()->getParent();
+ const auto &TLI = *MF.getSubtarget().getTargetLowering();
+ bool OptSize = shouldLowerMemFuncForSize(MF);
+ uint64_t Limit = Opc == TargetOpcode::G_MEMCPY_INLINE
+ ? std::numeric_limits<uint64_t>::max()
+ : TLI.getMaxStoresPerMemcpy(OptSize);
+ return lowerMemcpy(MI, Dst, Src, KnownLen, Limit, DstAlign, SrcAlign,
+ IsVolatile);
}
if (Opc == TargetOpcode::G_MEMMOVE)
- return lowerMemmove(MI, Dst, Src, KnownLen, DstAlign, SrcAlign, IsVolatile,
- DstAlignCanChange, MemOps);
+ return lowerMemmove(MI, Dst, Src, KnownLen, DstAlign, SrcAlign, IsVolatile);
if (Opc == TargetOpcode::G_MEMSET)
- return lowerMemset(MI, Dst, Src, KnownLen, DstAlign, IsVolatile,
- DstAlignCanChange, MemOps);
+ return lowerMemset(MI, Dst, Src, KnownLen, DstAlign, IsVolatile);
return UnableToLegalize;
}
-
-LegalizerHelper::LegalizeResult
-LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen) {
- Register Dst, Src;
- uint64_t KnownLen;
- Align DstAlign, SrcAlign;
- bool IsVolatile, DstAlignCanChange;
- std::vector<LLT> MemOps;
- if (!canLowerMemCpyFamily(MI, MRI, MaxLen, Dst, Src, KnownLen, DstAlign,
- SrcAlign, IsVolatile, DstAlignCanChange, MemOps))
- return UnableToLegalize;
- return lowerMemCpyFamily(MI, Dst, Src, KnownLen, DstAlign, SrcAlign,
- IsVolatile, DstAlignCanChange, MemOps);
-}
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 673ebd33f56aa..7ae093282c49d 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -20,7 +20,6 @@
#include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
-#include "llvm/CodeGen/LowLevelTypeUtils.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
@@ -37,10 +36,8 @@
#include "llvm/Support/UndefPoison.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/Utils/SizeOpts.h"
-#include <limits>
#include <numeric>
#include <optional>
-#include <tuple>
#define DEBUG_TYPE "globalisel-utils"
@@ -2081,177 +2078,3 @@ llvm::GFConstant::getConstant(Register Const, const MachineRegisterInfo &MRI) {
return GFConstant(MayBeConstant->Value, GFConstantKind::Scalar);
}
-
-// Returns a list of types to use for memory op lowering in MemOps. A partial
-// port of findOptimalMemOpLowering in TargetLowering.
-static bool findGISelOptimalMemOpLowering(std::vector<LLT> &MemOps,
- unsigned Limit, const MemOp &Op,
- unsigned DstAS, unsigned SrcAS,
- const AttributeList &FuncAttributes,
- const TargetLowering &TLI) {
- if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign())
- return false;
-
- LLT Ty = TLI.getOptimalMemOpLLT(Op, FuncAttributes);
-
- if (Ty == LLT()) {
- // Use the largest scalar type whose alignment constraints are satisfied.
- // We only need to check DstAlign here as SrcAlign is always greater or
- // equal to DstAlign (or zero).
- Ty = LLT::integer(64);
- if (Op.isFixedDstAlign())
- while (Op.getDstAlign() < Ty.getSizeInBytes() &&
- !TLI.allowsMisalignedMemoryAccesses(Ty, DstAS, Op.getDstAlign()))
- Ty = LLT::integer(Ty.getSizeInBytes());
- assert(Ty.getSizeInBits() > 0 && "Could not find valid type");
- // FIXME: check for the largest legal type we can load/store to.
- }
-
- unsigned NumMemOps = 0;
- uint64_t Size = Op.size();
- while (Size) {
- unsigned TySize = Ty.getSizeInBytes();
- while (TySize > Size) {
- // For now, only use non-vector load / store's for the left-over pieces.
- LLT NewTy = Ty;
- // FIXME: check for mem op safety and legality of the types. Not all of
- // SDAGisms map cleanly to GISel concepts.
- if (NewTy.isVector())
- NewTy =
- NewTy.getSizeInBits() > 64 ? LLT::integer(64) : LLT::integer(32);
- NewTy = LLT::integer(llvm::bit_floor(NewTy.getSizeInBits() - 1));
- unsigned NewTySize = NewTy.getSizeInBytes();
- assert(NewTySize > 0 && "Could not find appropriate type");
-
- // If the new LLT cannot cover all of the remaining bits, then consider
- // issuing a (or a pair of) unaligned and overlapping load / store.
- unsigned Fast;
- // Need to get a VT equivalent for allowMisalignedMemoryAccesses().
- MVT VT = getMVTForLLT(Ty);
- if (NumMemOps && Op.allowOverlap() && NewTySize < Size &&
- TLI.allowsMisalignedMemoryAccesses(
- VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
- MachineMemOperand::MONone, &Fast) &&
- Fast)
- TySize = Size;
- else {
- Ty = NewTy;
- TySize = NewTySize;
- }
- }
-
- if (++NumMemOps > Limit)
- return false;
-
- MemOps.push_back(Ty);
- Size -= TySize;
- }
-
- return true;
-}
-
-bool llvm::canLowerMemCpyFamily(const MachineInstr &MI,
- const MachineRegisterInfo &MRI, unsigned MaxLen,
- Register &Dst, Register &Src,
- uint64_t &KnownLen, Align &DstAlign,
- Align &SrcAlign, bool &IsVolatile,
- bool &DstAlignCanChange,
- std::vector<LLT> &MemOps) {
- const unsigned Opc = MI.getOpcode();
- assert((Opc == TargetOpcode::G_MEMCPY ||
- Opc == TargetOpcode::G_MEMCPY_INLINE ||
- Opc == TargetOpcode::G_MEMMOVE || Opc == TargetOpcode::G_MEMSET) &&
- "Expected memcpy like instruction");
-
- auto MMOIt = MI.memoperands_begin();
- const MachineMemOperand *MemOp = *MMOIt;
-
- DstAlign = MemOp->getBaseAlign();
- Register Len;
- std::tie(Dst, Src, Len) = MI.getFirst3Regs();
-
- if (Opc != TargetOpcode::G_MEMSET) {
- assert(MMOIt != MI.memoperands_end() && "Expected a second MMO on MI");
- MemOp = *(++MMOIt);
- SrcAlign = MemOp->getBaseAlign();
- }
-
- // See if this is a constant length copy.
- auto LenVRegAndVal = getIConstantVRegValWithLookThrough(Len, MRI);
- if (!LenVRegAndVal) {
- // FIXME: support dynamically sized G_MEMCPY_INLINE
- assert(Opc != TargetOpcode::G_MEMCPY_INLINE &&
- "inline memcpy with dynamic size is not yet supported");
- return false;
- }
-
- KnownLen = LenVRegAndVal->Value.getZExtValue();
- DstAlignCanChange = false;
-
- if (KnownLen == 0)
- return true;
-
- if (Opc != TargetOpcode::G_MEMCPY_INLINE && MaxLen && KnownLen > MaxLen)
- return false;
-
- IsVolatile = MemOp->isVolatile();
- const MachineFunction &MF = *MI.getParent()->getParent();
- const auto &TLI = *MF.getSubtarget().getTargetLowering();
- // On Darwin, -Os means optimize for size without hurting performance, so
- // only really optimize for size when -Oz (MinSize) is used.
- bool OptSize = MF.getTarget().getTargetTriple().isOSDarwin()
- ? MF.getFunction().hasMinSize()
- : MF.getFunction().hasOptSize();
-
- const MachineFrameInfo &MFI = MF.getFrameInfo();
- MachineInstr *FIDef = getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Dst, MRI);
- if (FIDef && !MFI.isFixedObjectIndex(FIDef->getOperand(1).getIndex()))
- DstAlignCanChange = true;
-
- const auto &DstMMO = **MI.memoperands_begin();
- MachinePointerInfo DstPtrInfo = DstMMO.getPointerInfo();
-
- switch (Opc) {
- case TargetOpcode::G_MEMCPY_INLINE:
- case TargetOpcode::G_MEMCPY: {
- const auto &SrcMMO = **std::next(MI.memoperands_begin());
- MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo();
- uint64_t Limit = Opc == TargetOpcode::G_MEMCPY_INLINE
- ? std::numeric_limits<uint64_t>::max()
- : TLI.getMaxStoresPerMemcpy(OptSize);
- return findGISelOptimalMemOpLowering(
- MemOps, Limit,
- MemOp::Copy(KnownLen, DstAlignCanChange, std::min(DstAlign, SrcAlign),
- SrcAlign, IsVolatile),
- DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
- MF.getFunction().getAttributes(), TLI);
- }
- case TargetOpcode::G_MEMMOVE: {
- const auto &SrcMMO = **std::next(MI.memoperands_begin());
- MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo();
- unsigned Limit = TLI.getMaxStoresPerMemmove(OptSize);
- // FIXME: SelectionDAG always passes false for 'AllowOverlap', apparently
- // due to a bug in it's findOptimalMemOpLowering implementation. For now do
- // the same thing here.
- return findGISelOptimalMemOpLowering(
- MemOps, Limit,
- MemOp::Copy(KnownLen, DstAlignCanChange, std::min(DstAlign, SrcAlign),
- SrcAlign, /*IsVolatile=*/true),
- DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
- MF.getFunction().getAttributes(), TLI);
- }
- case TargetOpcode::G_MEMSET: {
- unsigned Limit = TLI.getMaxStoresPerMemset(OptSize);
- auto ValVRegAndVal = getIConstantVRegValWithLookThrough(Src, MRI);
- bool IsZeroVal = ValVRegAndVal && ValVRegAndVal->Value == 0;
- return findGISelOptimalMemOpLowering(
- MemOps, Limit,
- MemOp::Set(KnownLen, DstAlignCanChange, DstAlign,
- /*IsZeroMemset=*/IsZeroVal,
- /*IsVolatile=*/IsVolatile),
- DstPtrInfo.getAddrSpace(), ~0u, MF.getFunction().getAttributes(), TLI);
- }
- default:
- llvm_unreachable("Unexpected memcpy-family opcode");
- }
-}
More information about the llvm-branch-commits
mailing list