[llvm] [CodeGen] Update for scalable MemoryType in MMO (PR #70452)
David Green via llvm-commits
llvm-commits at lists.llvm.org
Thu Dec 14 10:54:04 PST 2023
https://github.com/davemgreen updated https://github.com/llvm/llvm-project/pull/70452
>From d5a9cfef0de118657c2aa5ae9caf660c7fc03225 Mon Sep 17 00:00:00 2001
From: Harvin Iriawan <harvin.iriawan at arm.com>
Date: Wed, 25 Oct 2023 09:27:03 +0000
Subject: [PATCH 1/3] [SelectionDAG] Update for scalable MemoryType in MMO
Remove getSizeOrUnknown call when MachineMemOperand is created. For Scalable
TypeSize, the MemoryType created becomes a scalable_vector.
2 MMOs that have scalable memory access can then use the updated BasicAA that
understands scalable LocationSize
---
llvm/include/llvm/Analysis/MemoryLocation.h | 9 ++-
llvm/include/llvm/CodeGen/MachineFunction.h | 21 ++++++
llvm/include/llvm/CodeGen/MachineMemOperand.h | 16 +++--
llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp | 39 +++++++----
llvm/lib/CodeGen/MachineFunction.cpp | 20 ++++++
llvm/lib/CodeGen/MachineInstr.cpp | 43 ++++++++----
llvm/lib/CodeGen/MachineOperand.cpp | 24 ++++++-
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 70 ++++++++++++-------
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 19 +++--
llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll | 6 +-
.../RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll | 8 +--
11 files changed, 198 insertions(+), 77 deletions(-)
diff --git a/llvm/include/llvm/Analysis/MemoryLocation.h b/llvm/include/llvm/Analysis/MemoryLocation.h
index b72a27cab86b34..bea96827a7c9e3 100644
--- a/llvm/include/llvm/Analysis/MemoryLocation.h
+++ b/llvm/include/llvm/Analysis/MemoryLocation.h
@@ -291,8 +291,9 @@ class MemoryLocation {
return MemoryLocation(Ptr, LocationSize::beforeOrAfterPointer(), AATags);
}
- // Return the exact size if the exact size is known at compiletime,
- // otherwise return MemoryLocation::UnknownSize.
+ // TODO: Remove getSizeOrUnknown
+ // interface once llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest is
+ // updated
static uint64_t getSizeOrUnknown(const TypeSize &T) {
return T.isScalable() ? UnknownSize : T.getFixedValue();
}
@@ -303,6 +304,10 @@ class MemoryLocation {
const AAMDNodes &AATags = AAMDNodes())
: Ptr(Ptr), Size(Size), AATags(AATags) {}
+ explicit MemoryLocation(const Value *Ptr, TypeSize Size,
+ const AAMDNodes &AATags = AAMDNodes())
+ : Ptr(Ptr), Size(LocationSize::precise(Size)), AATags(AATags) {}
+
MemoryLocation getWithNewPtr(const Value *NewPtr) const {
MemoryLocation Copy(*this);
Copy.Ptr = NewPtr;
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index 05c9b14a423cda..e1ba0e29d6ace8 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -1032,6 +1032,13 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
+ MachineMemOperand *getMachineMemOperand(
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, TypeSize ts,
+ Align base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
+ const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
+ AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
+ AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
+
MachineMemOperand *getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy,
Align base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
@@ -1051,6 +1058,17 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
MMO, Offset, Size == ~UINT64_C(0) ? LLT() : LLT::scalar(8 * Size));
}
+ MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
+ int64_t Offset, TypeSize ts) {
+ return getMachineMemOperand(
+ MMO, Offset,
+ ts.getKnownMinValue() == ~UINT64_C(0)
+ ? LLT()
+ : ts.isScalable()
+ ? LLT::scalable_vector(1, 8 * ts.getKnownMinValue())
+ : LLT::scalar(8 * ts.getKnownMinValue()));
+ }
+
/// getMachineMemOperand - Allocate a new MachineMemOperand by copying
/// an existing one, replacing only the MachinePointerInfo and size.
/// MachineMemOperands are owned by the MachineFunction and need not be
@@ -1058,6 +1076,9 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
uint64_t Size);
+ MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
+ const MachinePointerInfo &PtrInfo,
+ TypeSize ts);
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
LLT Ty);
diff --git a/llvm/include/llvm/CodeGen/MachineMemOperand.h b/llvm/include/llvm/CodeGen/MachineMemOperand.h
index 5bfa7ffa65d519..90665199022ed0 100644
--- a/llvm/include/llvm/CodeGen/MachineMemOperand.h
+++ b/llvm/include/llvm/CodeGen/MachineMemOperand.h
@@ -192,6 +192,12 @@ class MachineMemOperand {
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
+ MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, TypeSize ts,
+ Align a, const AAMDNodes &AAInfo = AAMDNodes(),
+ const MDNode *Ranges = nullptr,
+ SyncScope::ID SSID = SyncScope::System,
+ AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
+ AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, LLT type, Align a,
const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr,
@@ -235,13 +241,15 @@ class MachineMemOperand {
LLT getMemoryType() const { return MemoryType; }
/// Return the size in bytes of the memory reference.
- uint64_t getSize() const {
- return MemoryType.isValid() ? MemoryType.getSizeInBytes() : ~UINT64_C(0);
+ TypeSize getSize() const {
+ return MemoryType.isValid() ? MemoryType.getSizeInBytes()
+ : TypeSize::Fixed(~UINT64_C(0));
}
/// Return the size in bits of the memory reference.
- uint64_t getSizeInBits() const {
- return MemoryType.isValid() ? MemoryType.getSizeInBits() : ~UINT64_C(0);
+ TypeSize getSizeInBits() const {
+ return MemoryType.isValid() ? MemoryType.getSizeInBits()
+ : TypeSize::Fixed(~UINT64_C(0));
}
LLT getType() const {
diff --git a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
index 246aa88b09acf6..b55904665f4e3f 100644
--- a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
@@ -196,7 +196,7 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
bool IsAtomic;
Register BasePtr;
int64_t Offset;
- uint64_t NumBytes;
+ TypeSize NumBytes;
MachineMemOperand *MMO;
};
@@ -212,8 +212,7 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
Offset = 0;
}
- uint64_t Size = MemoryLocation::getSizeOrUnknown(
- LS->getMMO().getMemoryType().getSizeInBytes());
+ TypeSize Size = LS->getMMO().getMemoryType().getSizeInBytes();
return {LS->isVolatile(), LS->isAtomic(), BaseReg,
Offset /*base offset*/, Size, &LS->getMMO()};
}
@@ -221,7 +220,7 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
// Default.
return {false /*isvolatile*/,
/*isAtomic*/ false, Register(),
- (int64_t)0 /*offset*/, 0 /*size*/,
+ (int64_t)0 /*offset*/, TypeSize::getFixed(0) /*size*/,
(MachineMemOperand *)nullptr};
};
MemUseCharacteristics MUC0 = getCharacteristics(&MI),
@@ -249,10 +248,20 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
return false;
}
+ // if NumBytes is scalable and offset is not 0, conservatively return may
+ // alias
+ if ((MUC0.NumBytes.isScalable() && (MUC0.Offset != 0)) ||
+ (MUC1.NumBytes.isScalable() && (MUC1.Offset != 0)))
+ return true;
+
+ const bool BothNotScalable =
+ !(MUC0.NumBytes.isScalable() || MUC1.NumBytes.isScalable());
+
// Try to prove that there is aliasing, or that there is no aliasing. Either
// way, we can return now. If nothing can be proved, proceed with more tests.
bool IsAlias;
- if (GISelAddressing::aliasIsKnownForLoadStore(MI, Other, IsAlias, MRI))
+ if (BothNotScalable &&
+ GISelAddressing::aliasIsKnownForLoadStore(MI, Other, IsAlias, MRI))
return IsAlias;
// The following all rely on MMO0 and MMO1 being valid.
@@ -262,19 +271,23 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
// FIXME: port the alignment based alias analysis from SDAG's isAlias().
int64_t SrcValOffset0 = MUC0.MMO->getOffset();
int64_t SrcValOffset1 = MUC1.MMO->getOffset();
- uint64_t Size0 = MUC0.NumBytes;
- uint64_t Size1 = MUC1.NumBytes;
+ TypeSize Size0 = MUC0.NumBytes;
+ TypeSize Size1 = MUC1.NumBytes;
if (AA && MUC0.MMO->getValue() && MUC1.MMO->getValue() &&
Size0 != MemoryLocation::UnknownSize &&
Size1 != MemoryLocation::UnknownSize) {
// Use alias analysis information.
int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1);
- int64_t Overlap0 = Size0 + SrcValOffset0 - MinOffset;
- int64_t Overlap1 = Size1 + SrcValOffset1 - MinOffset;
- if (AA->isNoAlias(MemoryLocation(MUC0.MMO->getValue(), Overlap0,
- MUC0.MMO->getAAInfo()),
- MemoryLocation(MUC1.MMO->getValue(), Overlap1,
- MUC1.MMO->getAAInfo())))
+ int64_t Overlap0 = Size0.getKnownMinValue() + SrcValOffset0 - MinOffset;
+ int64_t Overlap1 = Size1.getKnownMinValue() + SrcValOffset1 - MinOffset;
+ LocationSize Loc0 = Size0.isScalable() ? LocationSize::precise(Size0)
+ : LocationSize::precise(Overlap0);
+ LocationSize Loc1 = Size1.isScalable() ? LocationSize::precise(Size1)
+ : LocationSize::precise(Overlap1);
+
+ if (AA->isNoAlias(
+ MemoryLocation(MUC0.MMO->getValue(), Loc0, MUC0.MMO->getAAInfo()),
+ MemoryLocation(MUC1.MMO->getValue(), Loc1, MUC1.MMO->getAAInfo())))
return false;
}
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index 57af571ed9bfd5..13a54ea8eb0ae3 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -492,6 +492,16 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
SSID, Ordering, FailureOrdering);
}
+MachineMemOperand *MachineFunction::getMachineMemOperand(
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, TypeSize ts,
+ Align base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
+ SyncScope::ID SSID, AtomicOrdering Ordering,
+ AtomicOrdering FailureOrdering) {
+ return new (Allocator)
+ MachineMemOperand(PtrInfo, f, ts, base_alignment, AAInfo, Ranges, SSID,
+ Ordering, FailureOrdering);
+}
+
MachineMemOperand *MachineFunction::getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy,
Align base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
@@ -510,6 +520,16 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
MMO->getSuccessOrdering(), MMO->getFailureOrdering());
}
+MachineMemOperand *
+MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
+ const MachinePointerInfo &PtrInfo,
+ TypeSize ts) {
+ return new (Allocator)
+ MachineMemOperand(PtrInfo, MMO->getFlags(), ts, MMO->getBaseAlign(),
+ AAMDNodes(), nullptr, MMO->getSyncScopeID(),
+ MMO->getSuccessOrdering(), MMO->getFailureOrdering());
+}
+
MachineMemOperand *MachineFunction::getMachineMemOperand(
const MachineMemOperand *MMO, const MachinePointerInfo &PtrInfo, LLT Ty) {
return new (Allocator)
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 27eae372f8ad76..4a2d7303843aa1 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -1302,10 +1302,11 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
int64_t OffsetB = MMOb->getOffset();
int64_t MinOffset = std::min(OffsetA, OffsetB);
- uint64_t WidthA = MMOa->getSize();
- uint64_t WidthB = MMOb->getSize();
- bool KnownWidthA = WidthA != MemoryLocation::UnknownSize;
- bool KnownWidthB = WidthB != MemoryLocation::UnknownSize;
+ TypeSize WidthA = MMOa->getSize();
+ TypeSize WidthB = MMOb->getSize();
+ bool KnownWidthA = WidthA.getKnownMinValue() != MemoryLocation::UnknownSize;
+ bool KnownWidthB = WidthB.getKnownMinValue() != MemoryLocation::UnknownSize;
+ bool BothMMONonScalable = !WidthA.isScalable() && !WidthB.isScalable();
const Value *ValA = MMOa->getValue();
const Value *ValB = MMOb->getValue();
@@ -1321,11 +1322,12 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
SameVal = true;
}
- if (SameVal) {
+ if (SameVal && BothMMONonScalable) {
if (!KnownWidthA || !KnownWidthB)
return true;
int64_t MaxOffset = std::max(OffsetA, OffsetB);
- int64_t LowWidth = (MinOffset == OffsetA) ? WidthA : WidthB;
+ int64_t LowWidth = (MinOffset == OffsetA) ? WidthA.getKnownMinValue()
+ : WidthB.getKnownMinValue();
return (MinOffset + LowWidth > MaxOffset);
}
@@ -1338,15 +1340,30 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
assert((OffsetA >= 0) && "Negative MachineMemOperand offset");
assert((OffsetB >= 0) && "Negative MachineMemOperand offset");
- int64_t OverlapA =
- KnownWidthA ? WidthA + OffsetA - MinOffset : MemoryLocation::UnknownSize;
- int64_t OverlapB =
- KnownWidthB ? WidthB + OffsetB - MinOffset : MemoryLocation::UnknownSize;
+ // If Scalable Location Size has non-zero offset,
+ // Width + Offset does not work at the moment
+ if ((WidthA.isScalable() && (OffsetA > 0)) ||
+ (WidthB.isScalable() && (OffsetB > 0))) {
+ return true;
+ }
+
+ int64_t OverlapA = KnownWidthA
+ ? WidthA.getKnownMinValue() + OffsetA - MinOffset
+ : MemoryLocation::UnknownSize;
+ int64_t OverlapB = KnownWidthB
+ ? WidthB.getKnownMinValue() + OffsetB - MinOffset
+ : MemoryLocation::UnknownSize;
+
+ LocationSize LocA = WidthA.isScalable() && KnownWidthA
+ ? LocationSize::precise(WidthA)
+ : LocationSize(OverlapA);
+ LocationSize LocB = WidthB.isScalable() && KnownWidthB
+ ? LocationSize::precise(WidthB)
+ : LocationSize(OverlapB);
return !AA->isNoAlias(
- MemoryLocation(ValA, OverlapA, UseTBAA ? MMOa->getAAInfo() : AAMDNodes()),
- MemoryLocation(ValB, OverlapB,
- UseTBAA ? MMOb->getAAInfo() : AAMDNodes()));
+ MemoryLocation(ValA, LocA, UseTBAA ? MMOa->getAAInfo() : AAMDNodes()),
+ MemoryLocation(ValB, LocB, UseTBAA ? MMOb->getAAInfo() : AAMDNodes()));
}
bool MachineInstr::mayAlias(AAResults *AA, const MachineInstr &Other,
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index 12d6b79f735d7e..e24002fb4bf1cf 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -1098,14 +1098,29 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
s == ~UINT64_C(0) ? LLT() : LLT::scalar(8 * s), a,
AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
+MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
+ TypeSize ts, Align a,
+ const AAMDNodes &AAInfo,
+ const MDNode *Ranges, SyncScope::ID SSID,
+ AtomicOrdering Ordering,
+ AtomicOrdering FailureOrdering)
+ : MachineMemOperand(
+ ptrinfo, f,
+ ts.getKnownMinValue() == ~UINT64_C(0)
+ ? LLT()
+ : ts.isScalable()
+ ? LLT::scalable_vector(1, 8 * ts.getKnownMinValue())
+ : LLT::scalar(8 * ts.getKnownMinValue()),
+ a, AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
+
void MachineMemOperand::refineAlignment(const MachineMemOperand *MMO) {
// The Value and Offset may differ due to CSE. But the flags and size
// should be the same.
assert(MMO->getFlags() == getFlags() && "Flags mismatch!");
- assert((MMO->getSize() == ~UINT64_C(0) || getSize() == ~UINT64_C(0) ||
+ assert((MMO->getSize().getKnownMinValue() == ~UINT64_C(0) ||
+ getSize().getKnownMinValue() == ~UINT64_C(0) ||
MMO->getSize() == getSize()) &&
"Size mismatch!");
-
if (MMO->getBaseAlign() >= getBaseAlign()) {
// Update the alignment value.
BaseAlign = MMO->getBaseAlign();
@@ -1227,7 +1242,10 @@ void MachineMemOperand::print(raw_ostream &OS, ModuleSlotTracker &MST,
<< "unknown-address";
}
MachineOperand::printOperandOffset(OS, getOffset());
- if (getSize() > 0 && getAlign() != getSize())
+ if (getSize().isScalable())
+ OS << ", vscale ";
+ if (getSize().getKnownMinValue() > 0 &&
+ getAlign() != getSize().getKnownMinValue())
OS << ", align " << getAlign().value();
if (getAlign() != getBaseAlign())
OS << ", basealign " << getBaseAlign().value();
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index f35f663d6ba1b4..26cd6e723a0894 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -24201,7 +24201,7 @@ static SDValue narrowExtractedVectorLoad(SDNode *Extract, SelectionDAG &DAG) {
// TODO: Use "BaseIndexOffset" to make this more effective.
SDValue NewAddr = DAG.getMemBasePlusOffset(Ld->getBasePtr(), Offset, DL);
- uint64_t StoreSize = MemoryLocation::getSizeOrUnknown(VT.getStoreSize());
+ TypeSize StoreSize = VT.getStoreSize();
MachineFunction &MF = DAG.getMachineFunction();
MachineMemOperand *MMO;
if (Offset.isScalable()) {
@@ -27821,7 +27821,7 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
bool IsAtomic;
SDValue BasePtr;
int64_t Offset;
- std::optional<int64_t> NumBytes;
+ std::optional<TypeSize> NumBytes;
MachineMemOperand *MMO;
};
@@ -27834,13 +27834,12 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
: (LSN->getAddressingMode() == ISD::PRE_DEC)
? -1 * C->getSExtValue()
: 0;
- uint64_t Size =
- MemoryLocation::getSizeOrUnknown(LSN->getMemoryVT().getStoreSize());
+ TypeSize Size = LSN->getMemoryVT().getStoreSize();
return {LSN->isVolatile(),
LSN->isAtomic(),
LSN->getBasePtr(),
Offset /*base offset*/,
- std::optional<int64_t>(Size),
+ std::optional<TypeSize>(Size),
LSN->getMemOperand()};
}
if (const auto *LN = cast<LifetimeSDNode>(N))
@@ -27848,13 +27847,14 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
/*isAtomic*/ false,
LN->getOperand(1),
(LN->hasOffset()) ? LN->getOffset() : 0,
- (LN->hasOffset()) ? std::optional<int64_t>(LN->getSize())
- : std::optional<int64_t>(),
+ (LN->hasOffset())
+ ? std::optional<TypeSize>(TypeSize::getFixed(LN->getSize()))
+ : std::optional<TypeSize>(),
(MachineMemOperand *)nullptr};
// Default.
return {false /*isvolatile*/,
/*isAtomic*/ false, SDValue(),
- (int64_t)0 /*offset*/, std::optional<int64_t>() /*size*/,
+ (int64_t)0 /*offset*/, std::optional<TypeSize>() /*size*/,
(MachineMemOperand *)nullptr};
};
@@ -27881,10 +27881,20 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
return false;
}
+ // if NumBytes is scalable and offset is not 0, conservatively return may
+ // alias
+ if ((MUC0.NumBytes && MUC0.NumBytes.value().isScalable() &&
+ (MUC0.Offset != 0)) ||
+ (MUC1.NumBytes && MUC1.NumBytes.value().isScalable() &&
+ (MUC1.Offset != 0)))
+ return true;
// Try to prove that there is aliasing, or that there is no aliasing. Either
// way, we can return now. If nothing can be proved, proceed with more tests.
+ const bool BothNotScalable = !(MUC0.NumBytes.value().isScalable() ||
+ MUC1.NumBytes.value().isScalable());
bool IsAlias;
- if (BaseIndexOffset::computeAliasing(Op0, MUC0.NumBytes, Op1, MUC1.NumBytes,
+ if (BothNotScalable &&
+ BaseIndexOffset::computeAliasing(Op0, MUC0.NumBytes, Op1, MUC1.NumBytes,
DAG, IsAlias))
return IsAlias;
@@ -27911,17 +27921,21 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
Align OrigAlignment1 = MUC1.MMO->getBaseAlign();
auto &Size0 = MUC0.NumBytes;
auto &Size1 = MUC1.NumBytes;
- if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 &&
- Size0.has_value() && Size1.has_value() && *Size0 == *Size1 &&
- OrigAlignment0 > *Size0 && SrcValOffset0 % *Size0 == 0 &&
- SrcValOffset1 % *Size1 == 0) {
- int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.value();
- int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.value();
-
- // There is no overlap between these relatively aligned accesses of
- // similar size. Return no alias.
- if ((OffAlign0 + *Size0) <= OffAlign1 || (OffAlign1 + *Size1) <= OffAlign0)
- return false;
+
+ if (BothNotScalable) {
+ if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 &&
+ Size0.has_value() && Size1.has_value() && *Size0 == *Size1 &&
+ OrigAlignment0 > *Size0 && SrcValOffset0 % *Size0 == 0 &&
+ SrcValOffset1 % *Size1 == 0) {
+ int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.value();
+ int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.value();
+
+ // There is no overlap between these relatively aligned accesses of
+ // similar size. Return no alias.
+ if ((OffAlign0 + *Size0) <= OffAlign1 ||
+ (OffAlign1 + *Size1) <= OffAlign0)
+ return false;
+ }
}
bool UseAA = CombinerGlobalAA.getNumOccurrences() > 0
@@ -27937,12 +27951,20 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
Size1) {
// Use alias analysis information.
int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1);
- int64_t Overlap0 = *Size0 + SrcValOffset0 - MinOffset;
- int64_t Overlap1 = *Size1 + SrcValOffset1 - MinOffset;
+ int64_t Overlap0 =
+ Size0.value().getKnownMinValue() + SrcValOffset0 - MinOffset;
+ int64_t Overlap1 =
+ Size1.value().getKnownMinValue() + SrcValOffset1 - MinOffset;
+ LocationSize Loc0 = Size0.value().isScalable()
+ ? LocationSize::precise(Size0.value())
+ : LocationSize::precise(Overlap0);
+ LocationSize Loc1 = Size1.value().isScalable()
+ ? LocationSize::precise(Size1.value())
+ : LocationSize::precise(Overlap1);
if (AA->isNoAlias(
- MemoryLocation(MUC0.MMO->getValue(), Overlap0,
+ MemoryLocation(MUC0.MMO->getValue(), Loc0,
UseTBAA ? MUC0.MMO->getAAInfo() : AAMDNodes()),
- MemoryLocation(MUC1.MMO->getValue(), Overlap1,
+ MemoryLocation(MUC1.MMO->getValue(), Loc1,
UseTBAA ? MUC1.MMO->getAAInfo() : AAMDNodes())))
return false;
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 5be1892a44f6dd..be63504cbe3f47 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -8507,7 +8507,7 @@ SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
if (PtrInfo.V.isNull())
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
- uint64_t Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize());
+ TypeSize Size = MemVT.getStoreSize();
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
Alignment, AAInfo, Ranges);
@@ -8628,8 +8628,7 @@ SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
MachineFunction &MF = getMachineFunction();
- uint64_t Size =
- MemoryLocation::getSizeOrUnknown(Val.getValueType().getStoreSize());
+ TypeSize Size = Val.getValueType().getStoreSize();
MachineMemOperand *MMO =
MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo);
return getStore(Chain, dl, Val, Ptr, MMO);
@@ -8682,8 +8681,7 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
- PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()),
- Alignment, AAInfo);
+ PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment, AAInfo);
return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
}
@@ -8777,7 +8775,7 @@ SDValue SelectionDAG::getLoadVP(
if (PtrInfo.V.isNull())
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
- uint64_t Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize());
+ TypeSize Size = MemVT.getStoreSize();
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
Alignment, AAInfo, Ranges);
@@ -8930,8 +8928,7 @@ SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
- PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()),
- Alignment, AAInfo);
+ PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment, AAInfo);
return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO,
IsCompressing);
}
@@ -11782,9 +11779,9 @@ MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
// We check here that the size of the memory operand fits within the size of
// the MMO. This is because the MMO might indicate only a possible address
// range instead of specifying the affected memory addresses precisely.
- // TODO: Make MachineMemOperands aware of scalable vectors.
- assert(memvt.getStoreSize().getKnownMinValue() <= MMO->getSize() &&
- "Size mismatch!");
+ if (MMO->getType().isValid())
+ assert(TypeSize::isKnownLE(memvt.getStoreSize(), MMO->getSize()) &&
+ "Size mismatch!");
}
/// Profile - Gather unique data for the node.
diff --git a/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll b/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
index 8ed7059d2e754c..431796fac1d29a 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
@@ -3,7 +3,7 @@
define void @UphPNR(target("aarch64.svcount") %predcnt) {
entry:
; CHECK: %0:ppr = COPY $p0
-; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2)
+; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr, vscale )
; CHECK: %1:pnr_p8to15 = COPY %0
; CHECK: INLINEASM &"ld1w {z0.s,z1.s,z2.s,z3.s}, $0/z, [x10]", 1 /* sideeffect attdialect */, 458761 /* reguse:PNR_p8to15 */, %1
; CHECK: RET_ReallyLR
@@ -17,7 +17,7 @@ entry:
define void @UpaPNR(target("aarch64.svcount") %predcnt) {
entry:
; CHECK: %0:ppr = COPY $p0
-; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2)
+; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr, vscale )
; CHECK: %1:pnr = COPY %0
; CHECK: INLINEASM &"ld1w {z0.s,z1.s,z2.s,z3.s}, $0/z, [x10]", 1 /* sideeffect attdialect */, 262153 /* reguse:PNR */, %1
; CHECK: RET_ReallyLR
@@ -31,7 +31,7 @@ entry:
define void @UplPNR(target("aarch64.svcount") %predcnt) {
entry:
; CHECK: %0:ppr = COPY $p0
-; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2)
+; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr, vscale )
; CHECK: %1:pnr_3b = COPY %0
; CHECK: INLINEASM &"fadd z0.h, $0/m, z0.h, #0.5", 1 /* sideeffect attdialect */, 393225 /* reguse:PNR_3b */, %1
; CHECK: RET_ReallyLR
diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
index d612298bf50e73..2391e9fea6e3fa 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
@@ -16,8 +16,8 @@ define void @vpmerge_vpload_store(<vscale x 2 x i32> %passthru, <vscale x 2 x i3
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr = COPY $x10
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vrnov0 = COPY $v8
; CHECK-NEXT: $v0 = COPY [[COPY1]]
- ; CHECK-NEXT: [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], $v0, [[COPY]], 5 /* e32 */, 0 /* tu, mu */
- ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store unknown-size into %ir.p, align 8)
+ ; CHECK-NEXT: [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], $v0, [[COPY]], 5 /* e32 */, 0 /* tu, mu */ :: (load unknown-size from %ir.p, align 8)
+ ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store (<vscale x 1 x s64>) into %ir.p, vscale )
; CHECK-NEXT: PseudoRET
%splat = insertelement <vscale x 2 x i1> poison, i1 -1, i32 0
%mask = shufflevector <vscale x 2 x i1> %splat, <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
@@ -37,8 +37,8 @@ define void @vpselect_vpload_store(<vscale x 2 x i32> %passthru, <vscale x 2 x i
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr = COPY $x10
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vrnov0 = COPY $v8
; CHECK-NEXT: $v0 = COPY [[COPY1]]
- ; CHECK-NEXT: [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], $v0, [[COPY]], 5 /* e32 */, 1 /* ta, mu */
- ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store unknown-size into %ir.p, align 8)
+ ; CHECK-NEXT: [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], $v0, [[COPY]], 5 /* e32 */, 1 /* ta, mu */ :: (load unknown-size from %ir.p, align 8)
+ ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store (<vscale x 1 x s64>) into %ir.p, vscale )
; CHECK-NEXT: PseudoRET
%splat = insertelement <vscale x 2 x i1> poison, i1 -1, i32 0
%mask = shufflevector <vscale x 2 x i1> %splat, <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
>From 6e2a6e6166ee73807b36609ff8aec27e21136c8b Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Tue, 21 Nov 2023 08:43:31 +0000
Subject: [PATCH 2/3] Cleanup
---
llvm/include/llvm/Analysis/MemoryLocation.h | 11 ----
llvm/include/llvm/CodeGen/MachineFunction.h | 16 +++---
llvm/include/llvm/CodeGen/MachineMemOperand.h | 2 +-
.../CodeGen/SelectionDAGAddressAnalysis.h | 4 +-
llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp | 4 +-
llvm/lib/CodeGen/MachineFunction.cpp | 10 ++--
llvm/lib/CodeGen/MachineInstr.cpp | 5 +-
llvm/lib/CodeGen/MachineOperand.cpp | 16 +++---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 51 ++++++++-----------
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 6 +--
.../SelectionDAGAddressAnalysis.cpp | 17 +++----
.../Target/Hexagon/HexagonStoreWidening.cpp | 6 ++-
.../SelectionDAGAddressAnalysisTest.cpp | 44 ++++++----------
13 files changed, 77 insertions(+), 115 deletions(-)
diff --git a/llvm/include/llvm/Analysis/MemoryLocation.h b/llvm/include/llvm/Analysis/MemoryLocation.h
index bea96827a7c9e3..4680b6854abdbe 100644
--- a/llvm/include/llvm/Analysis/MemoryLocation.h
+++ b/llvm/include/llvm/Analysis/MemoryLocation.h
@@ -291,23 +291,12 @@ class MemoryLocation {
return MemoryLocation(Ptr, LocationSize::beforeOrAfterPointer(), AATags);
}
- // TODO: Remove getSizeOrUnknown
- // interface once llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest is
- // updated
- static uint64_t getSizeOrUnknown(const TypeSize &T) {
- return T.isScalable() ? UnknownSize : T.getFixedValue();
- }
-
MemoryLocation() : Ptr(nullptr), Size(LocationSize::beforeOrAfterPointer()) {}
explicit MemoryLocation(const Value *Ptr, LocationSize Size,
const AAMDNodes &AATags = AAMDNodes())
: Ptr(Ptr), Size(Size), AATags(AATags) {}
- explicit MemoryLocation(const Value *Ptr, TypeSize Size,
- const AAMDNodes &AATags = AAMDNodes())
- : Ptr(Ptr), Size(LocationSize::precise(Size)), AATags(AATags) {}
-
MemoryLocation getWithNewPtr(const Value *NewPtr) const {
MemoryLocation Copy(*this);
Copy.Ptr = NewPtr;
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index e1ba0e29d6ace8..5493f84fda35dd 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -1033,8 +1033,8 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
MachineMemOperand *getMachineMemOperand(
- MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, TypeSize ts,
- Align base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, TypeSize TS,
+ Align BaseAlignment, const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
@@ -1059,14 +1059,12 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
}
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
- int64_t Offset, TypeSize ts) {
+ int64_t Offset, TypeSize TS) {
return getMachineMemOperand(
MMO, Offset,
- ts.getKnownMinValue() == ~UINT64_C(0)
- ? LLT()
- : ts.isScalable()
- ? LLT::scalable_vector(1, 8 * ts.getKnownMinValue())
- : LLT::scalar(8 * ts.getKnownMinValue()));
+ TS.getKnownMinValue() == ~UINT64_C(0) ? LLT()
+ : TS.isScalable() ? LLT::scalable_vector(1, 8 * TS.getKnownMinValue())
+ : LLT::scalar(8 * TS.getKnownMinValue()));
}
/// getMachineMemOperand - Allocate a new MachineMemOperand by copying
@@ -1078,7 +1076,7 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
uint64_t Size);
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
- TypeSize ts);
+ TypeSize TS);
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
LLT Ty);
diff --git a/llvm/include/llvm/CodeGen/MachineMemOperand.h b/llvm/include/llvm/CodeGen/MachineMemOperand.h
index 90665199022ed0..0022f2806c7bd7 100644
--- a/llvm/include/llvm/CodeGen/MachineMemOperand.h
+++ b/llvm/include/llvm/CodeGen/MachineMemOperand.h
@@ -192,7 +192,7 @@ class MachineMemOperand {
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
- MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, TypeSize ts,
+ MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, TypeSize TS,
Align a, const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr,
SyncScope::ID SSID = SyncScope::System,
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h b/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h
index 3d0f836b0c7578..b7828d890018b5 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h
@@ -82,9 +82,9 @@ class BaseIndexOffset {
// Returns true `Op0` and `Op1` can be proven to alias/not alias, in
// which case `IsAlias` is set to true/false.
static bool computeAliasing(const SDNode *Op0,
- const std::optional<int64_t> NumBytes0,
+ const std::optional<TypeSize> NumBytes0,
const SDNode *Op1,
- const std::optional<int64_t> NumBytes1,
+ const std::optional<TypeSize> NumBytes1,
const SelectionDAG &DAG, bool &IsAlias);
/// Parses tree in N for base, index, offset addresses.
diff --git a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
index b55904665f4e3f..5474c2965726e5 100644
--- a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
@@ -220,7 +220,7 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
// Default.
return {false /*isvolatile*/,
/*isAtomic*/ false, Register(),
- (int64_t)0 /*offset*/, TypeSize::getFixed(0) /*size*/,
+ (int64_t)0 /*offset*/, TypeSize::Fixed(0) /*size*/,
(MachineMemOperand *)nullptr};
};
MemUseCharacteristics MUC0 = getCharacteristics(&MI),
@@ -248,7 +248,7 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
return false;
}
- // if NumBytes is scalable and offset is not 0, conservatively return may
+ // If NumBytes is scalable and offset is not 0, conservatively return may
// alias
if ((MUC0.NumBytes.isScalable() && (MUC0.Offset != 0)) ||
(MUC1.NumBytes.isScalable() && (MUC1.Offset != 0)))
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index 13a54ea8eb0ae3..f9d73b6f329c73 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -493,12 +493,12 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
}
MachineMemOperand *MachineFunction::getMachineMemOperand(
- MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, TypeSize ts,
- Align base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, TypeSize TS,
+ Align BaseAlignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
SyncScope::ID SSID, AtomicOrdering Ordering,
AtomicOrdering FailureOrdering) {
return new (Allocator)
- MachineMemOperand(PtrInfo, f, ts, base_alignment, AAInfo, Ranges, SSID,
+ MachineMemOperand(PtrInfo, F, TS, BaseAlignment, AAInfo, Ranges, SSID,
Ordering, FailureOrdering);
}
@@ -523,9 +523,9 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
MachineMemOperand *
MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
- TypeSize ts) {
+ TypeSize TS) {
return new (Allocator)
- MachineMemOperand(PtrInfo, MMO->getFlags(), ts, MMO->getBaseAlign(),
+ MachineMemOperand(PtrInfo, MMO->getFlags(), TS, MMO->getBaseAlign(),
AAMDNodes(), nullptr, MMO->getSyncScopeID(),
MMO->getSuccessOrdering(), MMO->getFailureOrdering());
}
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 4a2d7303843aa1..509597f4bcaee6 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -1342,10 +1342,9 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
// If Scalable Location Size has non-zero offset,
// Width + Offset does not work at the moment
- if ((WidthA.isScalable() && (OffsetA > 0)) ||
- (WidthB.isScalable() && (OffsetB > 0))) {
+ if ((WidthA.isScalable() && OffsetA > 0) ||
+ (WidthB.isScalable() && OffsetB > 0))
return true;
- }
int64_t OverlapA = KnownWidthA
? WidthA.getKnownMinValue() + OffsetA - MinOffset
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index e24002fb4bf1cf..375b694eab5462 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -1098,20 +1098,18 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
s == ~UINT64_C(0) ? LLT() : LLT::scalar(8 * s), a,
AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
-MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
- TypeSize ts, Align a,
+MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags F,
+ TypeSize TS, Align BaseAlignment,
const AAMDNodes &AAInfo,
const MDNode *Ranges, SyncScope::ID SSID,
AtomicOrdering Ordering,
AtomicOrdering FailureOrdering)
: MachineMemOperand(
- ptrinfo, f,
- ts.getKnownMinValue() == ~UINT64_C(0)
- ? LLT()
- : ts.isScalable()
- ? LLT::scalable_vector(1, 8 * ts.getKnownMinValue())
- : LLT::scalar(8 * ts.getKnownMinValue()),
- a, AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
+ ptrinfo, F,
+ TS.getKnownMinValue() == ~UINT64_C(0) ? LLT()
+ : TS.isScalable() ? LLT::scalable_vector(1, 8 * TS.getKnownMinValue())
+ : LLT::scalar(8 * TS.getKnownMinValue()),
+ BaseAlignment, AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
void MachineMemOperand::refineAlignment(const MachineMemOperand *MMO) {
// The Value and Offset may differ due to CSE. But the flags and size
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 26cd6e723a0894..a769651d279208 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -27848,7 +27848,7 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
LN->getOperand(1),
(LN->hasOffset()) ? LN->getOffset() : 0,
(LN->hasOffset())
- ? std::optional<TypeSize>(TypeSize::getFixed(LN->getSize()))
+ ? std::optional<TypeSize>(TypeSize::Fixed(LN->getSize()))
: std::optional<TypeSize>(),
(MachineMemOperand *)nullptr};
// Default.
@@ -27881,20 +27881,15 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
return false;
}
- // if NumBytes is scalable and offset is not 0, conservatively return may
+ // If NumBytes is scalable and offset is not 0, conservatively return may
// alias
- if ((MUC0.NumBytes && MUC0.NumBytes.value().isScalable() &&
- (MUC0.Offset != 0)) ||
- (MUC1.NumBytes && MUC1.NumBytes.value().isScalable() &&
- (MUC1.Offset != 0)))
+ if ((MUC0.NumBytes && MUC0.NumBytes->isScalable() && MUC0.Offset != 0) ||
+ (MUC1.NumBytes && MUC1.NumBytes->isScalable() && MUC1.Offset != 0))
return true;
// Try to prove that there is aliasing, or that there is no aliasing. Either
// way, we can return now. If nothing can be proved, proceed with more tests.
- const bool BothNotScalable = !(MUC0.NumBytes.value().isScalable() ||
- MUC1.NumBytes.value().isScalable());
bool IsAlias;
- if (BothNotScalable &&
- BaseIndexOffset::computeAliasing(Op0, MUC0.NumBytes, Op1, MUC1.NumBytes,
+ if (BaseIndexOffset::computeAliasing(Op0, MUC0.NumBytes, Op1, MUC1.NumBytes,
DAG, IsAlias))
return IsAlias;
@@ -27922,20 +27917,18 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
auto &Size0 = MUC0.NumBytes;
auto &Size1 = MUC1.NumBytes;
- if (BothNotScalable) {
- if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 &&
- Size0.has_value() && Size1.has_value() && *Size0 == *Size1 &&
- OrigAlignment0 > *Size0 && SrcValOffset0 % *Size0 == 0 &&
- SrcValOffset1 % *Size1 == 0) {
- int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.value();
- int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.value();
-
- // There is no overlap between these relatively aligned accesses of
- // similar size. Return no alias.
- if ((OffAlign0 + *Size0) <= OffAlign1 ||
- (OffAlign1 + *Size1) <= OffAlign0)
- return false;
- }
+ if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 &&
+ Size0.has_value() && Size1.has_value() && !Size0->isScalable() &&
+ !Size1->isScalable() && *Size0 == *Size1 && OrigAlignment0 > *Size0 &&
+ SrcValOffset0 % *Size0 == 0 && SrcValOffset1 % *Size1 == 0) {
+ int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.value();
+ int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.value();
+
+ // There is no overlap between these relatively aligned accesses of
+ // similar size. Return no alias.
+ if ((OffAlign0 + static_cast<int64_t>(*Size0)) <= OffAlign1 ||
+ (OffAlign1 + static_cast<int64_t>(*Size1)) <= OffAlign0)
+ return false;
}
bool UseAA = CombinerGlobalAA.getNumOccurrences() > 0
@@ -27951,14 +27944,12 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
Size1) {
// Use alias analysis information.
int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1);
- int64_t Overlap0 =
- Size0.value().getKnownMinValue() + SrcValOffset0 - MinOffset;
- int64_t Overlap1 =
- Size1.value().getKnownMinValue() + SrcValOffset1 - MinOffset;
- LocationSize Loc0 = Size0.value().isScalable()
+ int64_t Overlap0 = Size0->getKnownMinValue() + SrcValOffset0 - MinOffset;
+ int64_t Overlap1 = Size1->getKnownMinValue() + SrcValOffset1 - MinOffset;
+ LocationSize Loc0 = Size0->isScalable()
? LocationSize::precise(Size0.value())
: LocationSize::precise(Overlap0);
- LocationSize Loc1 = Size1.value().isScalable()
+ LocationSize Loc1 = Size1->isScalable()
? LocationSize::precise(Size1.value())
: LocationSize::precise(Overlap1);
if (AA->isNoAlias(
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index be63504cbe3f47..191b53f303f632 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -11779,9 +11779,9 @@ MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
// We check here that the size of the memory operand fits within the size of
// the MMO. This is because the MMO might indicate only a possible address
// range instead of specifying the affected memory addresses precisely.
- if (MMO->getType().isValid())
- assert(TypeSize::isKnownLE(memvt.getStoreSize(), MMO->getSize()) &&
- "Size mismatch!");
+ assert((!MMO->getType().isValid() ||
+ TypeSize::isKnownLE(memvt.getStoreSize(), MMO->getSize())) &&
+ "Size mismatch!");
}
/// Profile - Gather unique data for the node.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp
index 39a1e09e83c594..3572da19f1d35f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp
@@ -85,11 +85,10 @@ bool BaseIndexOffset::equalBaseIndex(const BaseIndexOffset &Other,
}
bool BaseIndexOffset::computeAliasing(const SDNode *Op0,
- const std::optional<int64_t> NumBytes0,
+ const std::optional<TypeSize> NumBytes0,
const SDNode *Op1,
- const std::optional<int64_t> NumBytes1,
+ const std::optional<TypeSize> NumBytes1,
const SelectionDAG &DAG, bool &IsAlias) {
-
BaseIndexOffset BasePtr0 = match(Op0, DAG);
BaseIndexOffset BasePtr1 = match(Op1, DAG);
@@ -103,20 +102,20 @@ bool BaseIndexOffset::computeAliasing(const SDNode *Op0,
// vector objects on the stack.
// BasePtr1 is PtrDiff away from BasePtr0. They alias if none of the
// following situations arise:
- if (PtrDiff >= 0 &&
- *NumBytes0 != static_cast<int64_t>(MemoryLocation::UnknownSize)) {
+ if (PtrDiff >= 0 && !NumBytes0->isScalable() &&
+ NumBytes0->getFixedValue() != MemoryLocation::UnknownSize) {
// [----BasePtr0----]
// [---BasePtr1--]
// ========PtrDiff========>
- IsAlias = !(*NumBytes0 <= PtrDiff);
+ IsAlias = !((int64_t)NumBytes0->getFixedValue() <= PtrDiff);
return true;
}
- if (PtrDiff < 0 &&
- *NumBytes1 != static_cast<int64_t>(MemoryLocation::UnknownSize)) {
+ if (PtrDiff < 0 && !NumBytes1->isScalable() &&
+ NumBytes1->getFixedValue() != MemoryLocation::UnknownSize) {
// [----BasePtr0----]
// [---BasePtr1--]
// =====(-PtrDiff)====>
- IsAlias = !((PtrDiff + *NumBytes1) <= 0);
+ IsAlias = !((PtrDiff + (int64_t)NumBytes1->getFixedValue()) <= 0);
return true;
}
return false;
diff --git a/llvm/lib/Target/Hexagon/HexagonStoreWidening.cpp b/llvm/lib/Target/Hexagon/HexagonStoreWidening.cpp
index a647e699a8f07a..f3facbe511a2f7 100644
--- a/llvm/lib/Target/Hexagon/HexagonStoreWidening.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonStoreWidening.cpp
@@ -172,14 +172,16 @@ bool HexagonStoreWidening::instrAliased(InstrGroup &Stores,
if (!MMO.getValue())
return true;
- MemoryLocation L(MMO.getValue(), MMO.getSize(), MMO.getAAInfo());
+ MemoryLocation L(MMO.getValue(), MMO.getSize().getFixedValue(),
+ MMO.getAAInfo());
for (auto *SI : Stores) {
const MachineMemOperand &SMO = getStoreTarget(SI);
if (!SMO.getValue())
return true;
- MemoryLocation SL(SMO.getValue(), SMO.getSize(), SMO.getAAInfo());
+ MemoryLocation SL(SMO.getValue(), SMO.getSize().getFixedValue(),
+ SMO.getAAInfo());
if (!AA->isNoAlias(L, SL))
return true;
}
diff --git a/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp b/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp
index 7426884217a08e..f711f85e1248f2 100644
--- a/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp
+++ b/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp
@@ -110,8 +110,7 @@ TEST_F(SelectionDAGAddressAnalysisTest, sameFrameObject) {
SDValue Index = DAG->getMemBasePlusOffset(FIPtr, Offset, Loc);
SDValue Store = DAG->getStore(DAG->getEntryNode(), Loc, Value, Index,
PtrInfo.getWithOffset(Offset));
- std::optional<int64_t> NumBytes = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(Store)->getMemoryVT().getStoreSize());
+ TypeSize NumBytes = cast<StoreSDNode>(Store)->getMemoryVT().getStoreSize();
bool IsAlias;
bool IsValid = BaseIndexOffset::computeAliasing(
@@ -137,7 +136,7 @@ TEST_F(SelectionDAGAddressAnalysisTest, sameFrameObjectUnknownSize) {
// Maybe unlikely that BaseIndexOffset::computeAliasing is used with the
// optional NumBytes being unset like in this test, but it would be confusing
// if that function determined IsAlias=false here.
- std::optional<int64_t> NumBytes;
+ std::optional<TypeSize> NumBytes;
bool IsAlias;
bool IsValid = BaseIndexOffset::computeAliasing(
@@ -165,10 +164,8 @@ TEST_F(SelectionDAGAddressAnalysisTest, noAliasingFrameObjects) {
PtrInfo.getWithOffset(Offset0));
SDValue Store1 = DAG->getStore(DAG->getEntryNode(), Loc, Value, Index1,
PtrInfo.getWithOffset(Offset1));
- std::optional<int64_t> NumBytes0 = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(Store0)->getMemoryVT().getStoreSize());
- std::optional<int64_t> NumBytes1 = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(Store1)->getMemoryVT().getStoreSize());
+ TypeSize NumBytes0 = cast<StoreSDNode>(Store0)->getMemoryVT().getStoreSize();
+ TypeSize NumBytes1 = cast<StoreSDNode>(Store1)->getMemoryVT().getStoreSize();
bool IsAlias;
bool IsValid = BaseIndexOffset::computeAliasing(
@@ -195,10 +192,8 @@ TEST_F(SelectionDAGAddressAnalysisTest, unknownSizeFrameObjects) {
DAG->getStore(DAG->getEntryNode(), Loc, Value, FIPtr, PtrInfo);
SDValue Store1 = DAG->getStore(DAG->getEntryNode(), Loc, Value, Index1,
MachinePointerInfo(PtrInfo.getAddrSpace()));
- std::optional<int64_t> NumBytes0 = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(Store0)->getMemoryVT().getStoreSize());
- std::optional<int64_t> NumBytes1 = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(Store1)->getMemoryVT().getStoreSize());
+ TypeSize NumBytes0 = cast<StoreSDNode>(Store0)->getMemoryVT().getStoreSize();
+ TypeSize NumBytes1 = cast<StoreSDNode>(Store1)->getMemoryVT().getStoreSize();
bool IsAlias;
bool IsValid = BaseIndexOffset::computeAliasing(
@@ -220,16 +215,14 @@ TEST_F(SelectionDAGAddressAnalysisTest, globalWithFrameObject) {
SDValue Index = DAG->getMemBasePlusOffset(FIPtr, Offset, Loc);
SDValue Store = DAG->getStore(DAG->getEntryNode(), Loc, Value, Index,
PtrInfo.getWithOffset(Offset));
- std::optional<int64_t> NumBytes = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(Store)->getMemoryVT().getStoreSize());
+ TypeSize NumBytes = cast<StoreSDNode>(Store)->getMemoryVT().getStoreSize();
EVT GTy = DAG->getTargetLoweringInfo().getValueType(DAG->getDataLayout(),
G->getType());
SDValue GValue = DAG->getConstant(0, Loc, GTy);
SDValue GAddr = DAG->getGlobalAddress(G, Loc, GTy);
SDValue GStore = DAG->getStore(DAG->getEntryNode(), Loc, GValue, GAddr,
MachinePointerInfo(G, 0));
- std::optional<int64_t> GNumBytes = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(GStore)->getMemoryVT().getStoreSize());
+ TypeSize GNumBytes = cast<StoreSDNode>(GStore)->getMemoryVT().getStoreSize();
bool IsAlias;
bool IsValid = BaseIndexOffset::computeAliasing(
@@ -248,8 +241,7 @@ TEST_F(SelectionDAGAddressAnalysisTest, globalWithAliasedGlobal) {
SDValue GAddr = DAG->getGlobalAddress(G, Loc, GTy);
SDValue GStore = DAG->getStore(DAG->getEntryNode(), Loc, GValue, GAddr,
MachinePointerInfo(G, 0));
- std::optional<int64_t> GNumBytes = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(GStore)->getMemoryVT().getStoreSize());
+ TypeSize GNumBytes = cast<StoreSDNode>(GStore)->getMemoryVT().getStoreSize();
SDValue AliasedGValue = DAG->getConstant(1, Loc, GTy);
SDValue AliasedGAddr = DAG->getGlobalAddress(AliasedG, Loc, GTy);
@@ -290,10 +282,8 @@ TEST_F(SelectionDAGAddressAnalysisTest, fixedSizeFrameObjectsWithinDiff) {
PtrInfo.getWithOffset(Offset0));
SDValue Store1 = DAG->getStore(DAG->getEntryNode(), Loc, Value1, Index1,
PtrInfo.getWithOffset(Offset1));
- std::optional<int64_t> NumBytes0 = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(Store0)->getMemoryVT().getStoreSize());
- std::optional<int64_t> NumBytes1 = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(Store1)->getMemoryVT().getStoreSize());
+ TypeSize NumBytes0 = cast<StoreSDNode>(Store0)->getMemoryVT().getStoreSize();
+ TypeSize NumBytes1 = cast<StoreSDNode>(Store1)->getMemoryVT().getStoreSize();
bool IsAlias;
bool IsValid = BaseIndexOffset::computeAliasing(
@@ -331,10 +321,8 @@ TEST_F(SelectionDAGAddressAnalysisTest, fixedSizeFrameObjectsOutOfDiff) {
PtrInfo.getWithOffset(Offset0));
SDValue Store1 = DAG->getStore(DAG->getEntryNode(), Loc, Value1, Index1,
PtrInfo.getWithOffset(Offset1));
- std::optional<int64_t> NumBytes0 = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(Store0)->getMemoryVT().getStoreSize());
- std::optional<int64_t> NumBytes1 = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(Store1)->getMemoryVT().getStoreSize());
+ TypeSize NumBytes0 = cast<StoreSDNode>(Store0)->getMemoryVT().getStoreSize();
+ TypeSize NumBytes1 = cast<StoreSDNode>(Store1)->getMemoryVT().getStoreSize();
bool IsAlias;
bool IsValid = BaseIndexOffset::computeAliasing(
@@ -365,10 +353,8 @@ TEST_F(SelectionDAGAddressAnalysisTest, twoFixedStackObjects) {
PtrInfo0.getWithOffset(Offset0));
SDValue Store1 = DAG->getStore(DAG->getEntryNode(), Loc, Value1, Index1,
PtrInfo1.getWithOffset(Offset0));
- std::optional<int64_t> NumBytes0 = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(Store0)->getMemoryVT().getStoreSize());
- std::optional<int64_t> NumBytes1 = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(Store1)->getMemoryVT().getStoreSize());
+ TypeSize NumBytes0 = cast<StoreSDNode>(Store0)->getMemoryVT().getStoreSize();
+ TypeSize NumBytes1 = cast<StoreSDNode>(Store1)->getMemoryVT().getStoreSize();
bool IsAlias;
bool IsValid = BaseIndexOffset::computeAliasing(
>From 913649416e779c5b2962b65605147fd32703f197 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Thu, 23 Nov 2023 21:59:17 +0000
Subject: [PATCH 3/3] Attempt to use LocationSize
---
.../GlobalISel/GIMatchTableExecutorImpl.h | 6 +-
.../CodeGen/GlobalISel/GenericMachineInstrs.h | 4 +-
llvm/include/llvm/CodeGen/MachineFunction.h | 60 +++++++++++------
llvm/include/llvm/CodeGen/MachineInstr.h | 9 +--
llvm/include/llvm/CodeGen/MachineMemOperand.h | 23 +++----
llvm/include/llvm/CodeGen/SelectionDAG.h | 4 +-
.../CodeGen/SelectionDAGAddressAnalysis.h | 6 +-
llvm/include/llvm/CodeGen/TargetInstrInfo.h | 2 +-
llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 26 ++++----
llvm/lib/CodeGen/DFAPacketizer.cpp | 7 +-
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 12 ++--
.../lib/CodeGen/GlobalISel/GISelKnownBits.cpp | 7 +-
.../CodeGen/GlobalISel/LegalizerHelper.cpp | 8 +--
llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp | 61 +++++++++--------
.../LiveDebugValues/InstrRefBasedImpl.cpp | 2 +-
llvm/lib/CodeGen/MIRVRegNamerUtils.cpp | 2 +-
llvm/lib/CodeGen/MachineFunction.cpp | 22 +------
llvm/lib/CodeGen/MachineInstr.cpp | 61 +++++++++--------
llvm/lib/CodeGen/MachineOperand.cpp | 28 +++-----
llvm/lib/CodeGen/MachinePipeliner.cpp | 13 ++--
llvm/lib/CodeGen/MachineScheduler.cpp | 13 ++--
llvm/lib/CodeGen/MachineStableHash.cpp | 2 +-
llvm/lib/CodeGen/MachineVerifier.cpp | 9 ++-
llvm/lib/CodeGen/ModuloSchedule.cpp | 4 +-
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 66 ++++++++++---------
.../SelectionDAG/LegalizeVectorTypes.cpp | 45 +++++++------
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 25 +++----
.../SelectionDAGAddressAnalysis.cpp | 20 +++---
.../SelectionDAG/SelectionDAGBuilder.cpp | 31 ++++-----
llvm/lib/CodeGen/TargetInstrInfo.cpp | 4 +-
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 7 +-
llvm/lib/Target/AArch64/AArch64InstrInfo.h | 2 +-
.../AArch64/AArch64LoadStoreOptimizer.cpp | 4 +-
.../GISel/AArch64InstructionSelector.cpp | 8 +--
.../GISel/AArch64PostLegalizerCombiner.cpp | 4 +-
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 2 +-
.../Target/AMDGPU/AMDGPURegisterBankInfo.cpp | 2 +-
.../lib/Target/AMDGPU/SIInsertHardClauses.cpp | 2 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 22 ++++---
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 2 +-
.../Target/AMDGPU/SILoadStoreOptimizer.cpp | 2 +-
llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp | 2 +-
llvm/lib/Target/ARM/ARMHazardRecognizer.cpp | 6 +-
llvm/lib/Target/BPF/BPFISelDAGToDAG.cpp | 4 +-
llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp | 8 +--
llvm/lib/Target/Hexagon/HexagonInstrInfo.h | 4 +-
.../Target/Hexagon/HexagonStoreWidening.cpp | 16 ++---
llvm/lib/Target/Hexagon/HexagonSubtarget.cpp | 11 ++--
llvm/lib/Target/Lanai/LanaiInstrInfo.cpp | 11 ++--
llvm/lib/Target/Lanai/LanaiInstrInfo.h | 4 +-
.../Target/Mips/MipsInstructionSelector.cpp | 5 +-
llvm/lib/Target/Mips/MipsLegalizerInfo.cpp | 2 +-
.../Target/Mips/MipsPreLegalizerCombiner.cpp | 4 +-
llvm/lib/Target/Mips/MipsRegisterBankInfo.cpp | 3 +-
.../PowerPC/GISel/PPCInstructionSelector.cpp | 2 +-
.../Target/PowerPC/PPCHazardRecognizers.cpp | 6 +-
llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 6 +-
llvm/lib/Target/PowerPC/PPCInstrInfo.cpp | 15 +++--
llvm/lib/Target/PowerPC/PPCInstrInfo.h | 4 +-
llvm/lib/Target/RISCV/RISCVInstrInfo.cpp | 15 +++--
llvm/lib/Target/RISCV/RISCVInstrInfo.h | 4 +-
llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp | 7 +-
.../WebAssemblySetP2AlignOperands.cpp | 2 +-
.../X86/X86AvoidStoreForwardingBlocks.cpp | 6 +-
llvm/lib/Target/X86/X86InstrInfo.cpp | 2 +-
llvm/lib/Target/X86/X86InstrInfo.h | 2 +-
llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll | 6 +-
llvm/test/CodeGen/AMDGPU/cluster_stores.ll | 32 ++++-----
.../RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll | 4 +-
.../SelectionDAGAddressAnalysisTest.cpp | 38 ++++++-----
.../Target/RISCV/RISCVInstrInfoTest.cpp | 2 +-
71 files changed, 446 insertions(+), 426 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
index 0a2709ef216be4..43c275aa58f538 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
@@ -647,15 +647,15 @@ bool GIMatchTableExecutor::executeMatchTable(
unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
- MMO->getSizeInBits() != Size) {
+ MMO->getSizeInBits().getValue() != Size) {
if (handleReject() == RejectAndGiveUp)
return false;
} else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
- MMO->getSizeInBits() >= Size) {
+ MMO->getSizeInBits().getValue() >= Size) {
if (handleReject() == RejectAndGiveUp)
return false;
} else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
- MMO->getSizeInBits() <= Size)
+ MMO->getSizeInBits().getValue() <= Size)
if (handleReject() == RejectAndGiveUp)
return false;
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
index 6ab1d4550c51ca..9f699bb41656c6 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
@@ -54,9 +54,9 @@ class GMemOperation : public GenericMachineInstr {
bool isUnordered() const { return getMMO().isUnordered(); }
/// Returns the size in bytes of the memory access.
- uint64_t getMemSize() const { return getMMO().getSize(); }
+ LocationSize getMemSize() const { return getMMO().getSize(); }
/// Returns the size in bits of the memory access.
- uint64_t getMemSizeInBits() const { return getMMO().getSizeInBits(); }
+ LocationSize getMemSizeInBits() const { return getMMO().getSizeInBits(); }
static bool classof(const MachineInstr *MI) {
return GenericMachineInstr::classof(MI) && MI->hasOneMemOperand();
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index 5493f84fda35dd..9d8f41a8309bd3 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -1026,25 +1026,37 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
/// MachineMemOperands are owned by the MachineFunction and need not be
/// explicitly deallocated.
MachineMemOperand *getMachineMemOperand(
- MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s,
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy,
Align base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
-
MachineMemOperand *getMachineMemOperand(
- MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, TypeSize TS,
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LocationSize TS,
Align BaseAlignment, const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
-
MachineMemOperand *getMachineMemOperand(
- MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy,
- Align base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, uint64_t TS,
+ Align BaseAlignment, const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
- AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
+ AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic) {
+ return getMachineMemOperand(PtrInfo, F, LocationSize::precise(TS),
+ BaseAlignment, AAInfo, Ranges, SSID, Ordering,
+ FailureOrdering);
+ }
+ MachineMemOperand *getMachineMemOperand(
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, TypeSize TS,
+ Align BaseAlignment, const AAMDNodes &AAInfo = AAMDNodes(),
+ const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
+ AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
+ AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic) {
+ return getMachineMemOperand(PtrInfo, F, LocationSize::precise(TS),
+ BaseAlignment, AAInfo, Ranges, SSID, Ordering,
+ FailureOrdering);
+ }
/// getMachineMemOperand - Allocate a new MachineMemOperand by copying
/// an existing one, adjusting by an offset and using the given size.
@@ -1053,18 +1065,21 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
int64_t Offset, LLT Ty);
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
- int64_t Offset, uint64_t Size) {
+ int64_t Offset, LocationSize TS) {
return getMachineMemOperand(
- MMO, Offset, Size == ~UINT64_C(0) ? LLT() : LLT::scalar(8 * Size));
+ MMO, Offset,
+ !TS.hasValue() ? LLT()
+ : TS.isScalable()
+ ? LLT::scalable_vector(1, 8 * TS.getValue().getKnownMinValue())
+ : LLT::scalar(8 * TS.getValue().getKnownMinValue()));
}
-
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
- int64_t Offset, TypeSize TS) {
- return getMachineMemOperand(
- MMO, Offset,
- TS.getKnownMinValue() == ~UINT64_C(0) ? LLT()
- : TS.isScalable() ? LLT::scalable_vector(1, 8 * TS.getKnownMinValue())
- : LLT::scalar(8 * TS.getKnownMinValue()));
+ int64_t Offset, uint64_t Size) {
+ return getMachineMemOperand(MMO, Offset, LocationSize::precise(Size));
+ }
+ MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
+ int64_t Offset, TypeSize Size) {
+ return getMachineMemOperand(MMO, Offset, LocationSize::precise(Size));
}
/// getMachineMemOperand - Allocate a new MachineMemOperand by copying
@@ -1073,13 +1088,20 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
/// explicitly deallocated.
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
- uint64_t Size);
+ LocationSize TS);
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
- TypeSize TS);
+ LLT Ty);
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
- LLT Ty);
+ uint64_t Size) {
+ return getMachineMemOperand(MMO, PtrInfo, LocationSize::precise(Size));
+ }
+ MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
+ const MachinePointerInfo &PtrInfo,
+ TypeSize Size) {
+ return getMachineMemOperand(MMO, PtrInfo, LocationSize::precise(Size));
+ }
/// Allocate a new MachineMemOperand by copying an existing one,
/// replacing only AliasAnalysis information. MachineMemOperands are owned
diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h
index bd72ac23fc9c08..f52c62c50ed08a 100644
--- a/llvm/include/llvm/CodeGen/MachineInstr.h
+++ b/llvm/include/llvm/CodeGen/MachineInstr.h
@@ -1744,16 +1744,17 @@ class MachineInstr
bool allImplicitDefsAreDead() const;
/// Return a valid size if the instruction is a spill instruction.
- std::optional<unsigned> getSpillSize(const TargetInstrInfo *TII) const;
+ std::optional<LocationSize> getSpillSize(const TargetInstrInfo *TII) const;
/// Return a valid size if the instruction is a folded spill instruction.
- std::optional<unsigned> getFoldedSpillSize(const TargetInstrInfo *TII) const;
+ std::optional<LocationSize>
+ getFoldedSpillSize(const TargetInstrInfo *TII) const;
/// Return a valid size if the instruction is a restore instruction.
- std::optional<unsigned> getRestoreSize(const TargetInstrInfo *TII) const;
+ std::optional<LocationSize> getRestoreSize(const TargetInstrInfo *TII) const;
/// Return a valid size if the instruction is a folded restore instruction.
- std::optional<unsigned>
+ std::optional<LocationSize>
getFoldedRestoreSize(const TargetInstrInfo *TII) const;
/// Copy implicit register operands from specified
diff --git a/llvm/include/llvm/CodeGen/MachineMemOperand.h b/llvm/include/llvm/CodeGen/MachineMemOperand.h
index 0022f2806c7bd7..4cd388e5263433 100644
--- a/llvm/include/llvm/CodeGen/MachineMemOperand.h
+++ b/llvm/include/llvm/CodeGen/MachineMemOperand.h
@@ -17,6 +17,7 @@
#include "llvm/ADT/BitmaskEnum.h"
#include "llvm/ADT/PointerUnion.h"
+#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/CodeGen/LowLevelType.h"
#include "llvm/CodeGen/PseudoSourceValue.h"
#include "llvm/IR/DerivedTypes.h"
@@ -186,13 +187,7 @@ class MachineMemOperand {
/// and atomic ordering requirements must also be specified. For cmpxchg
/// atomic operations the atomic ordering requirements when store does not
/// occur must also be specified.
- MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, uint64_t s,
- Align a, const AAMDNodes &AAInfo = AAMDNodes(),
- const MDNode *Ranges = nullptr,
- SyncScope::ID SSID = SyncScope::System,
- AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
- AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
- MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, TypeSize TS,
+ MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, LocationSize TS,
Align a, const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr,
SyncScope::ID SSID = SyncScope::System,
@@ -241,15 +236,17 @@ class MachineMemOperand {
LLT getMemoryType() const { return MemoryType; }
/// Return the size in bytes of the memory reference.
- TypeSize getSize() const {
- return MemoryType.isValid() ? MemoryType.getSizeInBytes()
- : TypeSize::Fixed(~UINT64_C(0));
+ LocationSize getSize() const {
+ return MemoryType.isValid()
+ ? LocationSize::precise(MemoryType.getSizeInBytes())
+ : LocationSize::beforeOrAfterPointer();
}
/// Return the size in bits of the memory reference.
- TypeSize getSizeInBits() const {
- return MemoryType.isValid() ? MemoryType.getSizeInBits()
- : TypeSize::Fixed(~UINT64_C(0));
+ LocationSize getSizeInBits() const {
+ return MemoryType.isValid()
+ ? LocationSize::precise(MemoryType.getSizeInBits())
+ : LocationSize::beforeOrAfterPointer();
}
LLT getType() const {
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index e867448b9d5512..646aadf3317116 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -1297,7 +1297,7 @@ class SelectionDAG {
EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad |
MachineMemOperand::MOStore,
- uint64_t Size = 0, const AAMDNodes &AAInfo = AAMDNodes());
+ LocationSize Size = 0, const AAMDNodes &AAInfo = AAMDNodes());
inline SDValue getMemIntrinsicNode(
unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
@@ -1305,7 +1305,7 @@ class SelectionDAG {
MaybeAlign Alignment = std::nullopt,
MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad |
MachineMemOperand::MOStore,
- uint64_t Size = 0, const AAMDNodes &AAInfo = AAMDNodes()) {
+ LocationSize Size = 0, const AAMDNodes &AAInfo = AAMDNodes()) {
// Ensure that codegen never sees alignment 0
return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, PtrInfo,
Alignment.value_or(getEVTAlign(MemVT)), Flags,
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h b/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h
index b7828d890018b5..6befd7a369827d 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h
@@ -81,10 +81,8 @@ class BaseIndexOffset {
// Returns true `Op0` and `Op1` can be proven to alias/not alias, in
// which case `IsAlias` is set to true/false.
- static bool computeAliasing(const SDNode *Op0,
- const std::optional<TypeSize> NumBytes0,
- const SDNode *Op1,
- const std::optional<TypeSize> NumBytes1,
+ static bool computeAliasing(const SDNode *Op0, const LocationSize NumBytes0,
+ const SDNode *Op1, const LocationSize NumBytes1,
const SelectionDAG &DAG, bool &IsAlias);
/// Parses tree in N for base, index, offset addresses.
diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
index 2bbe430dc68d90..995c049ef8f67d 100644
--- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
@@ -1433,7 +1433,7 @@ class TargetInstrInfo : public MCInstrInfo {
/// abstraction that supports negative offsets.
virtual bool getMemOperandsWithOffsetWidth(
const MachineInstr &MI, SmallVectorImpl<const MachineOperand *> &BaseOps,
- int64_t &Offset, bool &OffsetIsScalable, unsigned &Width,
+ int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
const TargetRegisterInfo *TRI) const {
return false;
}
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index 3a679f1576b7b6..724f5471d1ea42 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -1098,25 +1098,21 @@ static void emitComments(const MachineInstr &MI, raw_ostream &CommentOS) {
// We assume a single instruction only has a spill or reload, not
// both.
- std::optional<unsigned> Size;
+ std::optional<LocationSize> Size;
if ((Size = MI.getRestoreSize(TII))) {
- CommentOS << *Size << "-byte Reload\n";
+ CommentOS << Size->getValue() << "-byte Reload\n";
} else if ((Size = MI.getFoldedRestoreSize(TII))) {
- if (*Size) {
- if (*Size == unsigned(MemoryLocation::UnknownSize))
- CommentOS << "Unknown-size Folded Reload\n";
- else
- CommentOS << *Size << "-byte Folded Reload\n";
- }
+ if (!Size->hasValue())
+ CommentOS << "Unknown-size Folded Reload\n";
+ else if (Size->getValue())
+ CommentOS << Size->getValue() << "-byte Folded Reload\n";
} else if ((Size = MI.getSpillSize(TII))) {
- CommentOS << *Size << "-byte Spill\n";
+ CommentOS << Size->getValue() << "-byte Spill\n";
} else if ((Size = MI.getFoldedSpillSize(TII))) {
- if (*Size) {
- if (*Size == unsigned(MemoryLocation::UnknownSize))
- CommentOS << "Unknown-size Folded Spill\n";
- else
- CommentOS << *Size << "-byte Folded Spill\n";
- }
+ if (!Size->hasValue())
+ CommentOS << "Unknown-size Folded Spill\n";
+ else if (Size->getValue())
+ CommentOS << Size->getValue() << "-byte Folded Spill\n";
}
// Check for spill-induced copies
diff --git a/llvm/lib/CodeGen/DFAPacketizer.cpp b/llvm/lib/CodeGen/DFAPacketizer.cpp
index 48bb4a07662e10..c16166a1d5e1c5 100644
--- a/llvm/lib/CodeGen/DFAPacketizer.cpp
+++ b/llvm/lib/CodeGen/DFAPacketizer.cpp
@@ -252,12 +252,13 @@ void VLIWPacketizerList::PacketizeMIs(MachineBasicBlock *MBB,
bool VLIWPacketizerList::alias(const MachineMemOperand &Op1,
const MachineMemOperand &Op2,
bool UseTBAA) const {
- if (!Op1.getValue() || !Op2.getValue())
+ if (!Op1.getValue() || !Op2.getValue() || !Op1.getSize().hasValue() ||
+ !Op2.getSize().hasValue())
return true;
int64_t MinOffset = std::min(Op1.getOffset(), Op2.getOffset());
- int64_t Overlapa = Op1.getSize() + Op1.getOffset() - MinOffset;
- int64_t Overlapb = Op2.getSize() + Op2.getOffset() - MinOffset;
+ int64_t Overlapa = Op1.getSize().getValue() + Op1.getOffset() - MinOffset;
+ int64_t Overlapb = Op2.getSize().getValue() + Op2.getOffset() - MinOffset;
AliasResult AAResult =
AA->alias(MemoryLocation(Op1.getValue(), Overlapa,
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 91a64d59e154df..a320e51382dfc5 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -764,12 +764,12 @@ bool CombinerHelper::matchCombineLoadWithAndMask(MachineInstr &MI,
LLT RegTy = MRI.getType(LoadReg);
Register PtrReg = LoadMI->getPointerReg();
unsigned RegSize = RegTy.getSizeInBits();
- uint64_t LoadSizeBits = LoadMI->getMemSizeInBits();
+ LocationSize LoadSizeBits = LoadMI->getMemSizeInBits();
unsigned MaskSizeBits = MaskVal.countr_one();
// The mask may not be larger than the in-memory type, as it might cover sign
// extended bits
- if (MaskSizeBits > LoadSizeBits)
+ if (MaskSizeBits > LoadSizeBits.getValue())
return false;
// If the mask covers the whole destination register, there's nothing to
@@ -789,7 +789,8 @@ bool CombinerHelper::matchCombineLoadWithAndMask(MachineInstr &MI,
// still adjust the opcode to indicate the high bit behavior.
if (LoadMI->isSimple())
MemDesc.MemoryTy = LLT::scalar(MaskSizeBits);
- else if (LoadSizeBits > MaskSizeBits || LoadSizeBits == RegSize)
+ else if (LoadSizeBits.getValue() > MaskSizeBits ||
+ LoadSizeBits.getValue() == RegSize)
return false;
// TODO: Could check if it's legal with the reduced or original memory size.
@@ -854,7 +855,8 @@ bool CombinerHelper::matchSextTruncSextLoad(MachineInstr &MI) {
if (auto *LoadMI = getOpcodeDef<GSExtLoad>(LoadUser, MRI)) {
// If truncating more than the original extended value, abort.
auto LoadSizeBits = LoadMI->getMemSizeInBits();
- if (TruncSrc && MRI.getType(TruncSrc).getSizeInBits() < LoadSizeBits)
+ if (TruncSrc &&
+ MRI.getType(TruncSrc).getSizeInBits() < LoadSizeBits.getValue())
return false;
if (LoadSizeBits == SizeInBits)
return true;
@@ -885,7 +887,7 @@ bool CombinerHelper::matchSextInRegOfLoad(
if (!LoadDef || !MRI.hasOneNonDBGUse(DstReg))
return false;
- uint64_t MemBits = LoadDef->getMemSizeInBits();
+ uint64_t MemBits = LoadDef->getMemSizeInBits().getValue();
// If the sign extend extends from a narrower width than the load's width,
// then we can narrow the load width when we combine to a G_SEXTLOAD.
diff --git a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
index ea8c20cdcd45d6..4d6c47c0472bb6 100644
--- a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
@@ -415,7 +415,8 @@ void GISelKnownBits::computeKnownBitsImpl(Register R, KnownBits &Known,
if (DstTy.isVector())
break;
// Everything above the retrieved bits is zero
- Known.Zero.setBitsFrom((*MI.memoperands_begin())->getSizeInBits());
+ Known.Zero.setBitsFrom(
+ (*MI.memoperands_begin())->getSizeInBits().getValue());
break;
}
case TargetOpcode::G_ASHR: {
@@ -666,7 +667,7 @@ unsigned GISelKnownBits::computeNumSignBits(Register R,
// e.g. i16->i32 = '17' bits known.
const MachineMemOperand *MMO = *MI.memoperands_begin();
- return TyBits - MMO->getSizeInBits() + 1;
+ return TyBits - MMO->getSizeInBits().getValue() + 1;
}
case TargetOpcode::G_ZEXTLOAD: {
// FIXME: We need an in-memory type representation.
@@ -675,7 +676,7 @@ unsigned GISelKnownBits::computeNumSignBits(Register R,
// e.g. i16->i32 = '16' bits known.
const MachineMemOperand *MMO = *MI.memoperands_begin();
- return TyBits - MMO->getSizeInBits();
+ return TyBits - MMO->getSizeInBits().getValue();
}
case TargetOpcode::G_TRUNC: {
Register Src = MI.getOperand(1).getReg();
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 37e7153be5720e..ff985f3fe92510 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -1196,7 +1196,7 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
if (DstTy.isVector())
return UnableToLegalize;
- if (8 * LoadMI.getMemSize() != DstTy.getSizeInBits()) {
+ if (8 * LoadMI.getMemSize().getValue() != DstTy.getSizeInBits()) {
Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
MIRBuilder.buildLoad(TmpReg, LoadMI.getPointerReg(), LoadMI.getMMO());
MIRBuilder.buildAnyExt(DstReg, TmpReg);
@@ -1214,7 +1214,7 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
auto &MMO = LoadMI.getMMO();
- unsigned MemSize = MMO.getSizeInBits();
+ unsigned MemSize = MMO.getSizeInBits().getValue();
if (MemSize == NarrowSize) {
MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
@@ -1247,7 +1247,7 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
if (SrcTy.isVector() && LeftoverBits != 0)
return UnableToLegalize;
- if (8 * StoreMI.getMemSize() != SrcTy.getSizeInBits()) {
+ if (8 * StoreMI.getMemSize().getValue() != SrcTy.getSizeInBits()) {
Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
MIRBuilder.buildTrunc(TmpReg, SrcReg);
MIRBuilder.buildStore(TmpReg, StoreMI.getPointerReg(), StoreMI.getMMO());
@@ -4293,7 +4293,7 @@ LegalizerHelper::reduceLoadStoreWidth(GLoadStore &LdStMI, unsigned TypeIdx,
LLT ValTy = MRI.getType(ValReg);
// FIXME: Do we need a distinct NarrowMemory legalize action?
- if (ValTy.getSizeInBits() != 8 * LdStMI.getMemSize()) {
+ if (ValTy.getSizeInBits() != 8 * LdStMI.getMemSize().getValue()) {
LLVM_DEBUG(dbgs() << "Can't narrow extload/truncstore\n");
return UnableToLegalize;
}
diff --git a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
index 5474c2965726e5..f348c288b7ab23 100644
--- a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
@@ -117,8 +117,8 @@ bool GISelAddressing::aliasIsKnownForLoadStore(const MachineInstr &MI1,
if (!BasePtr0.BaseReg.isValid() || !BasePtr1.BaseReg.isValid())
return false;
- int64_t Size1 = LdSt1->getMemSize();
- int64_t Size2 = LdSt2->getMemSize();
+ LocationSize Size1 = LdSt1->getMemSize();
+ LocationSize Size2 = LdSt2->getMemSize();
int64_t PtrDiff;
if (BasePtr0.BaseReg == BasePtr1.BaseReg) {
@@ -128,20 +128,18 @@ bool GISelAddressing::aliasIsKnownForLoadStore(const MachineInstr &MI1,
// vector objects on the stack.
// BasePtr1 is PtrDiff away from BasePtr0. They alias if none of the
// following situations arise:
- if (PtrDiff >= 0 &&
- Size1 != static_cast<int64_t>(MemoryLocation::UnknownSize)) {
+ if (PtrDiff >= 0 && Size1.hasValue() && !Size1.isScalable()) {
// [----BasePtr0----]
// [---BasePtr1--]
// ========PtrDiff========>
- IsAlias = !(Size1 <= PtrDiff);
+ IsAlias = !((int64_t)Size1.getValue() <= PtrDiff);
return true;
}
- if (PtrDiff < 0 &&
- Size2 != static_cast<int64_t>(MemoryLocation::UnknownSize)) {
+ if (PtrDiff < 0 && Size2.hasValue() && !Size2.isScalable()) {
// [----BasePtr0----]
// [---BasePtr1--]
// =====(-PtrDiff)====>
- IsAlias = !((PtrDiff + Size2) <= 0);
+ IsAlias = !((PtrDiff + (int64_t)Size2.getValue()) <= 0);
return true;
}
return false;
@@ -196,7 +194,7 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
bool IsAtomic;
Register BasePtr;
int64_t Offset;
- TypeSize NumBytes;
+ LocationSize NumBytes;
MachineMemOperand *MMO;
};
@@ -213,14 +211,20 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
}
TypeSize Size = LS->getMMO().getMemoryType().getSizeInBytes();
- return {LS->isVolatile(), LS->isAtomic(), BaseReg,
- Offset /*base offset*/, Size, &LS->getMMO()};
+ return {LS->isVolatile(),
+ LS->isAtomic(),
+ BaseReg,
+ Offset /*base offset*/,
+ LocationSize::precise(Size),
+ &LS->getMMO()};
}
// FIXME: support recognizing lifetime instructions.
// Default.
return {false /*isvolatile*/,
- /*isAtomic*/ false, Register(),
- (int64_t)0 /*offset*/, TypeSize::Fixed(0) /*size*/,
+ /*isAtomic*/ false,
+ Register(),
+ (int64_t)0 /*offset*/,
+ LocationSize::beforeOrAfterPointer() /*size*/,
(MachineMemOperand *)nullptr};
};
MemUseCharacteristics MUC0 = getCharacteristics(&MI),
@@ -250,12 +254,12 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
// If NumBytes is scalable and offset is not 0, conservatively return may
// alias
- if ((MUC0.NumBytes.isScalable() && (MUC0.Offset != 0)) ||
- (MUC1.NumBytes.isScalable() && (MUC1.Offset != 0)))
+ if ((MUC0.NumBytes.isScalable() && MUC0.Offset != 0) ||
+ (MUC1.NumBytes.isScalable() && MUC1.Offset != 0))
return true;
const bool BothNotScalable =
- !(MUC0.NumBytes.isScalable() || MUC1.NumBytes.isScalable());
+ !MUC0.NumBytes.isScalable() && !MUC1.NumBytes.isScalable();
// Try to prove that there is aliasing, or that there is no aliasing. Either
// way, we can return now. If nothing can be proved, proceed with more tests.
@@ -271,19 +275,20 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
// FIXME: port the alignment based alias analysis from SDAG's isAlias().
int64_t SrcValOffset0 = MUC0.MMO->getOffset();
int64_t SrcValOffset1 = MUC1.MMO->getOffset();
- TypeSize Size0 = MUC0.NumBytes;
- TypeSize Size1 = MUC1.NumBytes;
- if (AA && MUC0.MMO->getValue() && MUC1.MMO->getValue() &&
- Size0 != MemoryLocation::UnknownSize &&
- Size1 != MemoryLocation::UnknownSize) {
+ LocationSize Size0 = MUC0.NumBytes;
+ LocationSize Size1 = MUC1.NumBytes;
+ if (AA && MUC0.MMO->getValue() && MUC1.MMO->getValue() && Size0.hasValue() &&
+ Size1.hasValue()) {
// Use alias analysis information.
int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1);
- int64_t Overlap0 = Size0.getKnownMinValue() + SrcValOffset0 - MinOffset;
- int64_t Overlap1 = Size1.getKnownMinValue() + SrcValOffset1 - MinOffset;
- LocationSize Loc0 = Size0.isScalable() ? LocationSize::precise(Size0)
- : LocationSize::precise(Overlap0);
- LocationSize Loc1 = Size1.isScalable() ? LocationSize::precise(Size1)
- : LocationSize::precise(Overlap1);
+ int64_t Overlap0 =
+ Size0.getValue().getKnownMinValue() + SrcValOffset0 - MinOffset;
+ int64_t Overlap1 =
+ Size1.getValue().getKnownMinValue() + SrcValOffset1 - MinOffset;
+ LocationSize Loc0 =
+ Size0.isScalable() ? Size0 : LocationSize::precise(Overlap0);
+ LocationSize Loc1 =
+ Size1.isScalable() ? Size1 : LocationSize::precise(Overlap1);
if (AA->isNoAlias(
MemoryLocation(MUC0.MMO->getValue(), Loc0, MUC0.MMO->getAAInfo()),
@@ -522,7 +527,7 @@ bool LoadStoreOpt::addStoreToCandidate(GStore &StoreMI,
return false;
// Don't allow truncating stores for now.
- if (StoreMI.getMemSizeInBits() != ValueTy.getSizeInBits())
+ if (StoreMI.getMemSizeInBits().getValue() != ValueTy.getSizeInBits())
return false;
// Avoid adding volatile or ordered stores to the candidate. We already have a
diff --git a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
index b2c2b40139eda6..7bba04f6078580 100644
--- a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
+++ b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
@@ -1356,7 +1356,7 @@ InstrRefBasedLDV::findLocationForMemOperand(const MachineInstr &MI) {
// from the stack at some point. Happily the memory operand will tell us
// the size written to the stack.
auto *MemOperand = *MI.memoperands_begin();
- unsigned SizeInBits = MemOperand->getSizeInBits();
+ unsigned SizeInBits = MemOperand->getSizeInBits().getValue();
// Find that position in the stack indexes we're tracking.
auto IdxIt = MTracker->StackSlotIdxes.find({SizeInBits, 0});
diff --git a/llvm/lib/CodeGen/MIRVRegNamerUtils.cpp b/llvm/lib/CodeGen/MIRVRegNamerUtils.cpp
index 812d57984e6cae..ccfc4565d3a9bc 100644
--- a/llvm/lib/CodeGen/MIRVRegNamerUtils.cpp
+++ b/llvm/lib/CodeGen/MIRVRegNamerUtils.cpp
@@ -123,7 +123,7 @@ std::string VRegRenamer::getInstructionOpcodeHash(MachineInstr &MI) {
llvm::transform(MI.uses(), std::back_inserter(MIOperands), GetHashableMO);
for (const auto *Op : MI.memoperands()) {
- MIOperands.push_back((unsigned)Op->getSize());
+ MIOperands.push_back((unsigned)Op->getSize().getValue());
MIOperands.push_back((unsigned)Op->getFlags());
MIOperands.push_back((unsigned)Op->getOffset());
MIOperands.push_back((unsigned)Op->getSuccessOrdering());
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index f9d73b6f329c73..174790feb29108 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -483,17 +483,7 @@ void MachineFunction::deleteMachineBasicBlock(MachineBasicBlock *MBB) {
}
MachineMemOperand *MachineFunction::getMachineMemOperand(
- MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s,
- Align base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
- SyncScope::ID SSID, AtomicOrdering Ordering,
- AtomicOrdering FailureOrdering) {
- return new (Allocator)
- MachineMemOperand(PtrInfo, f, s, base_alignment, AAInfo, Ranges,
- SSID, Ordering, FailureOrdering);
-}
-
-MachineMemOperand *MachineFunction::getMachineMemOperand(
- MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, TypeSize TS,
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LocationSize TS,
Align BaseAlignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
SyncScope::ID SSID, AtomicOrdering Ordering,
AtomicOrdering FailureOrdering) {
@@ -512,18 +502,10 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
Ordering, FailureOrdering);
}
-MachineMemOperand *MachineFunction::getMachineMemOperand(
- const MachineMemOperand *MMO, const MachinePointerInfo &PtrInfo, uint64_t Size) {
- return new (Allocator)
- MachineMemOperand(PtrInfo, MMO->getFlags(), Size, MMO->getBaseAlign(),
- AAMDNodes(), nullptr, MMO->getSyncScopeID(),
- MMO->getSuccessOrdering(), MMO->getFailureOrdering());
-}
-
MachineMemOperand *
MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
- TypeSize TS) {
+ LocationSize TS) {
return new (Allocator)
MachineMemOperand(PtrInfo, MMO->getFlags(), TS, MMO->getBaseAlign(),
AAMDNodes(), nullptr, MMO->getSyncScopeID(),
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 509597f4bcaee6..d53e026a4757a7 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -1302,10 +1302,10 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
int64_t OffsetB = MMOb->getOffset();
int64_t MinOffset = std::min(OffsetA, OffsetB);
- TypeSize WidthA = MMOa->getSize();
- TypeSize WidthB = MMOb->getSize();
- bool KnownWidthA = WidthA.getKnownMinValue() != MemoryLocation::UnknownSize;
- bool KnownWidthB = WidthB.getKnownMinValue() != MemoryLocation::UnknownSize;
+ LocationSize WidthA = MMOa->getSize();
+ LocationSize WidthB = MMOb->getSize();
+ bool KnownWidthA = WidthA.hasValue();
+ bool KnownWidthB = WidthB.hasValue();
bool BothMMONonScalable = !WidthA.isScalable() && !WidthB.isScalable();
const Value *ValA = MMOa->getValue();
@@ -1326,8 +1326,9 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
if (!KnownWidthA || !KnownWidthB)
return true;
int64_t MaxOffset = std::max(OffsetA, OffsetB);
- int64_t LowWidth = (MinOffset == OffsetA) ? WidthA.getKnownMinValue()
- : WidthB.getKnownMinValue();
+ int64_t LowWidth = (MinOffset == OffsetA)
+ ? WidthA.getValue().getKnownMinValue()
+ : WidthB.getValue().getKnownMinValue();
return (MinOffset + LowWidth > MaxOffset);
}
@@ -1346,19 +1347,19 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
(WidthB.isScalable() && OffsetB > 0))
return true;
- int64_t OverlapA = KnownWidthA
- ? WidthA.getKnownMinValue() + OffsetA - MinOffset
- : MemoryLocation::UnknownSize;
- int64_t OverlapB = KnownWidthB
- ? WidthB.getKnownMinValue() + OffsetB - MinOffset
- : MemoryLocation::UnknownSize;
+ int64_t OverlapA =
+ KnownWidthA ? WidthA.getValue().getKnownMinValue() + OffsetA - MinOffset
+ : MemoryLocation::UnknownSize;
+ int64_t OverlapB =
+ KnownWidthB ? WidthB.getValue().getKnownMinValue() + OffsetB - MinOffset
+ : MemoryLocation::UnknownSize;
- LocationSize LocA = WidthA.isScalable() && KnownWidthA
- ? LocationSize::precise(WidthA)
- : LocationSize(OverlapA);
- LocationSize LocB = WidthB.isScalable() && KnownWidthB
- ? LocationSize::precise(WidthB)
- : LocationSize(OverlapB);
+ LocationSize LocA = (WidthA.isScalable() || !KnownWidthA)
+ ? WidthA
+ : LocationSize::precise(OverlapA);
+ LocationSize LocB = (WidthB.isScalable() || !KnownWidthB)
+ ? WidthB
+ : LocationSize::precise(OverlapB);
return !AA->isNoAlias(
MemoryLocation(ValA, LocA, UseTBAA ? MMOa->getAAInfo() : AAMDNodes()),
@@ -2370,18 +2371,24 @@ void MachineInstr::changeDebugValuesDefReg(Register Reg) {
using MMOList = SmallVector<const MachineMemOperand *, 2>;
-static unsigned getSpillSlotSize(const MMOList &Accesses,
- const MachineFrameInfo &MFI) {
+static LocationSize getSpillSlotSize(const MMOList &Accesses,
+ const MachineFrameInfo &MFI) {
unsigned Size = 0;
- for (const auto *A : Accesses)
+ for (const auto *A : Accesses) {
if (MFI.isSpillSlotObjectIndex(
cast<FixedStackPseudoSourceValue>(A->getPseudoValue())
- ->getFrameIndex()))
- Size += A->getSize();
+ ->getFrameIndex())) {
+ if (A->getSize().hasValue())
+ Size += A->getSize().getValue();
+ else
+ return LocationSize::beforeOrAfterPointer();
+ }
+ }
+
return Size;
}
-std::optional<unsigned>
+std::optional<LocationSize>
MachineInstr::getSpillSize(const TargetInstrInfo *TII) const {
int FI;
if (TII->isStoreToStackSlotPostFE(*this, FI)) {
@@ -2392,7 +2399,7 @@ MachineInstr::getSpillSize(const TargetInstrInfo *TII) const {
return std::nullopt;
}
-std::optional<unsigned>
+std::optional<LocationSize>
MachineInstr::getFoldedSpillSize(const TargetInstrInfo *TII) const {
MMOList Accesses;
if (TII->hasStoreToStackSlot(*this, Accesses))
@@ -2400,7 +2407,7 @@ MachineInstr::getFoldedSpillSize(const TargetInstrInfo *TII) const {
return std::nullopt;
}
-std::optional<unsigned>
+std::optional<LocationSize>
MachineInstr::getRestoreSize(const TargetInstrInfo *TII) const {
int FI;
if (TII->isLoadFromStackSlotPostFE(*this, FI)) {
@@ -2411,7 +2418,7 @@ MachineInstr::getRestoreSize(const TargetInstrInfo *TII) const {
return std::nullopt;
}
-std::optional<unsigned>
+std::optional<LocationSize>
MachineInstr::getFoldedRestoreSize(const TargetInstrInfo *TII) const {
MMOList Accesses;
if (TII->hasLoadFromStackSlot(*this, Accesses))
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index 375b694eab5462..bc9f89d01ce8d4 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -1088,35 +1088,25 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
assert(getFailureOrdering() == FailureOrdering && "Value truncated");
}
-MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
- uint64_t s, Align a,
- const AAMDNodes &AAInfo,
- const MDNode *Ranges, SyncScope::ID SSID,
- AtomicOrdering Ordering,
- AtomicOrdering FailureOrdering)
- : MachineMemOperand(ptrinfo, f,
- s == ~UINT64_C(0) ? LLT() : LLT::scalar(8 * s), a,
- AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
-
MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags F,
- TypeSize TS, Align BaseAlignment,
+ LocationSize TS, Align BaseAlignment,
const AAMDNodes &AAInfo,
const MDNode *Ranges, SyncScope::ID SSID,
AtomicOrdering Ordering,
AtomicOrdering FailureOrdering)
: MachineMemOperand(
ptrinfo, F,
- TS.getKnownMinValue() == ~UINT64_C(0) ? LLT()
- : TS.isScalable() ? LLT::scalable_vector(1, 8 * TS.getKnownMinValue())
- : LLT::scalar(8 * TS.getKnownMinValue()),
+ !TS.hasValue() ? LLT()
+ : TS.isScalable()
+ ? LLT::scalable_vector(1, 8 * TS.getValue().getKnownMinValue())
+ : LLT::scalar(8 * TS.getValue().getKnownMinValue()),
BaseAlignment, AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
void MachineMemOperand::refineAlignment(const MachineMemOperand *MMO) {
// The Value and Offset may differ due to CSE. But the flags and size
// should be the same.
assert(MMO->getFlags() == getFlags() && "Flags mismatch!");
- assert((MMO->getSize().getKnownMinValue() == ~UINT64_C(0) ||
- getSize().getKnownMinValue() == ~UINT64_C(0) ||
+ assert((!MMO->getSize().hasValue() || !getSize().hasValue() ||
MMO->getSize() == getSize()) &&
"Size mismatch!");
if (MMO->getBaseAlign() >= getBaseAlign()) {
@@ -1240,10 +1230,8 @@ void MachineMemOperand::print(raw_ostream &OS, ModuleSlotTracker &MST,
<< "unknown-address";
}
MachineOperand::printOperandOffset(OS, getOffset());
- if (getSize().isScalable())
- OS << ", vscale ";
- if (getSize().getKnownMinValue() > 0 &&
- getAlign() != getSize().getKnownMinValue())
+ if (!getSize().hasValue() ||
+ getAlign() != getSize().getValue().getKnownMinValue())
OS << ", align " << getAlign().value();
if (getAlign() != getBaseAlign())
OS << ", basealign " << getBaseAlign().value();
diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp
index 8cd7f4ebe88d96..ed211b1858624c 100644
--- a/llvm/lib/CodeGen/MachinePipeliner.cpp
+++ b/llvm/lib/CodeGen/MachinePipeliner.cpp
@@ -2298,19 +2298,20 @@ bool SwingSchedulerDAG::isLoopCarriedDep(SUnit *Source, const SDep &Dep,
if (!LoopDefS || !TII->getIncrementValue(*LoopDefS, D))
return true;
- uint64_t AccessSizeS = (*SI->memoperands_begin())->getSize();
- uint64_t AccessSizeD = (*DI->memoperands_begin())->getSize();
+ LocationSize AccessSizeS = (*SI->memoperands_begin())->getSize();
+ LocationSize AccessSizeD = (*DI->memoperands_begin())->getSize();
// This is the main test, which checks the offset values and the loop
// increment value to determine if the accesses may be loop carried.
- if (AccessSizeS == MemoryLocation::UnknownSize ||
- AccessSizeD == MemoryLocation::UnknownSize)
+ if (!AccessSizeS.hasValue() || !AccessSizeD.hasValue())
return true;
- if (DeltaS != DeltaD || DeltaS < AccessSizeS || DeltaD < AccessSizeD)
+ if (DeltaS != DeltaD || DeltaS < AccessSizeS.getValue() ||
+ DeltaD < AccessSizeD.getValue())
return true;
- return (OffsetS + (int64_t)AccessSizeS < OffsetD + (int64_t)AccessSizeD);
+ return (OffsetS + (int64_t)AccessSizeS.getValue() <
+ OffsetD + (int64_t)AccessSizeD.getValue());
}
void SwingSchedulerDAG::postProcessDAG() {
diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index 886137d86f87de..9c9c539c02ec70 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -1697,11 +1697,11 @@ class BaseMemOpClusterMutation : public ScheduleDAGMutation {
SUnit *SU;
SmallVector<const MachineOperand *, 4> BaseOps;
int64_t Offset;
- unsigned Width;
+ LocationSize Width;
bool OffsetIsScalable;
MemOpInfo(SUnit *SU, ArrayRef<const MachineOperand *> BaseOps,
- int64_t Offset, bool OffsetIsScalable, unsigned Width)
+ int64_t Offset, bool OffsetIsScalable, LocationSize Width)
: SU(SU), BaseOps(BaseOps.begin(), BaseOps.end()), Offset(Offset),
Width(Width), OffsetIsScalable(OffsetIsScalable) {}
@@ -1825,11 +1825,12 @@ void BaseMemOpClusterMutation::clusterNeighboringMemOps(
auto MemOpb = MemOpRecords[NextIdx];
unsigned ClusterLength = 2;
- unsigned CurrentClusterBytes = MemOpa.Width + MemOpb.Width;
+ unsigned CurrentClusterBytes = MemOpa.Width.getValue().getKnownMinValue() +
+ MemOpb.Width.getValue().getKnownMinValue();
if (SUnit2ClusterInfo.count(MemOpa.SU->NodeNum)) {
ClusterLength = SUnit2ClusterInfo[MemOpa.SU->NodeNum].first + 1;
- CurrentClusterBytes =
- SUnit2ClusterInfo[MemOpa.SU->NodeNum].second + MemOpb.Width;
+ CurrentClusterBytes = SUnit2ClusterInfo[MemOpa.SU->NodeNum].second +
+ MemOpb.Width.getValue().getKnownMinValue();
}
if (!TII->shouldClusterMemOps(MemOpa.BaseOps, MemOpa.Offset,
@@ -1899,7 +1900,7 @@ void BaseMemOpClusterMutation::collectMemOpRecords(
SmallVector<const MachineOperand *, 4> BaseOps;
int64_t Offset;
bool OffsetIsScalable;
- unsigned Width;
+ LocationSize Width = 0;
if (TII->getMemOperandsWithOffsetWidth(MI, BaseOps, Offset,
OffsetIsScalable, Width, TRI)) {
MemOpRecords.push_back(
diff --git a/llvm/lib/CodeGen/MachineStableHash.cpp b/llvm/lib/CodeGen/MachineStableHash.cpp
index debb2b3809e3f7..7ac96c0b68882e 100644
--- a/llvm/lib/CodeGen/MachineStableHash.cpp
+++ b/llvm/lib/CodeGen/MachineStableHash.cpp
@@ -201,7 +201,7 @@ stable_hash llvm::stableHashValue(const MachineInstr &MI, bool HashVRegs,
for (const auto *Op : MI.memoperands()) {
if (!HashMemOperands)
break;
- HashComponents.push_back(static_cast<unsigned>(Op->getSize()));
+ HashComponents.push_back(static_cast<unsigned>(Op->getSize().getValue()));
HashComponents.push_back(static_cast<unsigned>(Op->getFlags()));
HashComponents.push_back(static_cast<unsigned>(Op->getOffset()));
HashComponents.push_back(static_cast<unsigned>(Op->getSuccessOrdering()));
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index a015d9bbd2d3f5..9537317e34e154 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -1189,13 +1189,16 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
const MachineMemOperand &MMO = **MI->memoperands_begin();
if (MI->getOpcode() == TargetOpcode::G_ZEXTLOAD ||
MI->getOpcode() == TargetOpcode::G_SEXTLOAD) {
- if (MMO.getSizeInBits() >= ValTy.getSizeInBits())
+ if (TypeSize::isKnownGE(MMO.getSizeInBits().getValue(),
+ ValTy.getSizeInBits()))
report("Generic extload must have a narrower memory type", MI);
} else if (MI->getOpcode() == TargetOpcode::G_LOAD) {
- if (MMO.getSize() > ValTy.getSizeInBytes())
+ if (TypeSize::isKnownGT(MMO.getSize().getValue(),
+ ValTy.getSizeInBytes()))
report("load memory size cannot exceed result size", MI);
} else if (MI->getOpcode() == TargetOpcode::G_STORE) {
- if (ValTy.getSizeInBytes() < MMO.getSize())
+ if (TypeSize::isKnownLT(ValTy.getSizeInBytes(),
+ MMO.getSize().getValue()))
report("store memory size cannot exceed value size", MI);
}
diff --git a/llvm/lib/CodeGen/ModuloSchedule.cpp b/llvm/lib/CodeGen/ModuloSchedule.cpp
index 0bef513342ff12..bdae94c4e6f885 100644
--- a/llvm/lib/CodeGen/ModuloSchedule.cpp
+++ b/llvm/lib/CodeGen/ModuloSchedule.cpp
@@ -979,8 +979,8 @@ void ModuloScheduleExpander::updateMemOperands(MachineInstr &NewMI,
NewMMOs.push_back(
MF.getMachineMemOperand(MMO, AdjOffset, MMO->getSize()));
} else {
- NewMMOs.push_back(
- MF.getMachineMemOperand(MMO, 0, MemoryLocation::UnknownSize));
+ NewMMOs.push_back(MF.getMachineMemOperand(
+ MMO, 0, LocationSize::beforeOrAfterPointer()));
}
}
NewMI.setMemRefs(MF, NewMMOs);
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index a769651d279208..4ea6d0e55e0ebf 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -27821,7 +27821,7 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
bool IsAtomic;
SDValue BasePtr;
int64_t Offset;
- std::optional<TypeSize> NumBytes;
+ LocationSize NumBytes;
MachineMemOperand *MMO;
};
@@ -27835,26 +27835,24 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
? -1 * C->getSExtValue()
: 0;
TypeSize Size = LSN->getMemoryVT().getStoreSize();
- return {LSN->isVolatile(),
- LSN->isAtomic(),
- LSN->getBasePtr(),
- Offset /*base offset*/,
- std::optional<TypeSize>(Size),
- LSN->getMemOperand()};
+ return {LSN->isVolatile(), LSN->isAtomic(),
+ LSN->getBasePtr(), Offset /*base offset*/,
+ LocationSize::precise(Size), LSN->getMemOperand()};
}
if (const auto *LN = cast<LifetimeSDNode>(N))
return {false /*isVolatile*/,
/*isAtomic*/ false,
LN->getOperand(1),
(LN->hasOffset()) ? LN->getOffset() : 0,
- (LN->hasOffset())
- ? std::optional<TypeSize>(TypeSize::Fixed(LN->getSize()))
- : std::optional<TypeSize>(),
+ (LN->hasOffset()) ? LocationSize::precise(LN->getSize())
+ : LocationSize::beforeOrAfterPointer(),
(MachineMemOperand *)nullptr};
// Default.
return {false /*isvolatile*/,
- /*isAtomic*/ false, SDValue(),
- (int64_t)0 /*offset*/, std::optional<TypeSize>() /*size*/,
+ /*isAtomic*/ false,
+ SDValue(),
+ (int64_t)0 /*offset*/,
+ LocationSize::beforeOrAfterPointer() /*size*/,
(MachineMemOperand *)nullptr};
};
@@ -27883,8 +27881,10 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
// If NumBytes is scalable and offset is not 0, conservatively return may
// alias
- if ((MUC0.NumBytes && MUC0.NumBytes->isScalable() && MUC0.Offset != 0) ||
- (MUC1.NumBytes && MUC1.NumBytes->isScalable() && MUC1.Offset != 0))
+ if ((MUC0.NumBytes.hasValue() && MUC0.NumBytes.isScalable() &&
+ MUC0.Offset != 0) ||
+ (MUC1.NumBytes.hasValue() && MUC1.NumBytes.isScalable() &&
+ MUC1.Offset != 0))
return true;
// Try to prove that there is aliasing, or that there is no aliasing. Either
// way, we can return now. If nothing can be proved, proceed with more tests.
@@ -27914,20 +27914,24 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
int64_t SrcValOffset1 = MUC1.MMO->getOffset();
Align OrigAlignment0 = MUC0.MMO->getBaseAlign();
Align OrigAlignment1 = MUC1.MMO->getBaseAlign();
- auto &Size0 = MUC0.NumBytes;
- auto &Size1 = MUC1.NumBytes;
+ LocationSize Size0 = MUC0.NumBytes;
+ LocationSize Size1 = MUC1.NumBytes;
if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 &&
- Size0.has_value() && Size1.has_value() && !Size0->isScalable() &&
- !Size1->isScalable() && *Size0 == *Size1 && OrigAlignment0 > *Size0 &&
- SrcValOffset0 % *Size0 == 0 && SrcValOffset1 % *Size1 == 0) {
+ Size0.hasValue() && Size1.hasValue() && !Size0.isScalable() &&
+ !Size1.isScalable() && Size0 == Size1 &&
+ OrigAlignment0 > Size0.getValue().getKnownMinValue() &&
+ SrcValOffset0 % Size0.getValue().getKnownMinValue() == 0 &&
+ SrcValOffset1 % Size1.getValue().getKnownMinValue() == 0) {
int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.value();
int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.value();
// There is no overlap between these relatively aligned accesses of
// similar size. Return no alias.
- if ((OffAlign0 + static_cast<int64_t>(*Size0)) <= OffAlign1 ||
- (OffAlign1 + static_cast<int64_t>(*Size1)) <= OffAlign0)
+ if ((OffAlign0 + static_cast<int64_t>(
+ Size0.getValue().getKnownMinValue())) <= OffAlign1 ||
+ (OffAlign1 + static_cast<int64_t>(
+ Size1.getValue().getKnownMinValue())) <= OffAlign0)
return false;
}
@@ -27940,18 +27944,18 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
UseAA = false;
#endif
- if (UseAA && AA && MUC0.MMO->getValue() && MUC1.MMO->getValue() && Size0 &&
- Size1) {
+ if (UseAA && AA && MUC0.MMO->getValue() && MUC1.MMO->getValue() &&
+ Size0.hasValue() && Size1.hasValue()) {
// Use alias analysis information.
int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1);
- int64_t Overlap0 = Size0->getKnownMinValue() + SrcValOffset0 - MinOffset;
- int64_t Overlap1 = Size1->getKnownMinValue() + SrcValOffset1 - MinOffset;
- LocationSize Loc0 = Size0->isScalable()
- ? LocationSize::precise(Size0.value())
- : LocationSize::precise(Overlap0);
- LocationSize Loc1 = Size1->isScalable()
- ? LocationSize::precise(Size1.value())
- : LocationSize::precise(Overlap1);
+ int64_t Overlap0 =
+ Size0.getValue().getKnownMinValue() + SrcValOffset0 - MinOffset;
+ int64_t Overlap1 =
+ Size1.getValue().getKnownMinValue() + SrcValOffset1 - MinOffset;
+ LocationSize Loc0 =
+ Size0.isScalable() ? Size0 : LocationSize::precise(Overlap0);
+ LocationSize Loc1 =
+ Size1.isScalable() ? Size1 : LocationSize::precise(Overlap1);
if (AA->isNoAlias(
MemoryLocation(MUC0.MMO->getValue(), Loc0,
UseTBAA ? MUC0.MMO->getAAInfo() : AAMDNodes()),
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 66461b26468f79..5eae715cfd8ac5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -1963,7 +1963,8 @@ void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
LD->getPointerInfo(), MachineMemOperand::MOLoad,
- MemoryLocation::UnknownSize, Alignment, LD->getAAInfo(), LD->getRanges());
+ LocationSize::beforeOrAfterPointer(), Alignment, LD->getAAInfo(),
+ LD->getRanges());
Lo =
DAG.getLoadVP(LD->getAddressingMode(), ExtType, LoVT, dl, Ch, Ptr, Offset,
@@ -1986,8 +1987,8 @@ void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
LoMemVT.getStoreSize().getFixedValue());
MMO = DAG.getMachineFunction().getMachineMemOperand(
- MPI, MachineMemOperand::MOLoad, MemoryLocation::UnknownSize, Alignment,
- LD->getAAInfo(), LD->getRanges());
+ MPI, MachineMemOperand::MOLoad, LocationSize::beforeOrAfterPointer(),
+ Alignment, LD->getAAInfo(), LD->getRanges());
Hi = DAG.getLoadVP(LD->getAddressingMode(), ExtType, HiVT, dl, Ch, Ptr,
Offset, MaskHi, EVLHi, HiMemVT, MMO,
@@ -2066,8 +2067,8 @@ void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(SLD->getPointerInfo().getAddrSpace()),
- MachineMemOperand::MOLoad, MemoryLocation::UnknownSize, Alignment,
- SLD->getAAInfo(), SLD->getRanges());
+ MachineMemOperand::MOLoad, LocationSize::beforeOrAfterPointer(),
+ Alignment, SLD->getAAInfo(), SLD->getRanges());
Hi = DAG.getStridedLoadVP(SLD->getAddressingMode(), SLD->getExtensionType(),
HiVT, DL, SLD->getChain(), Ptr, SLD->getOffset(),
@@ -2126,7 +2127,7 @@ void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MLD->getPointerInfo(), MachineMemOperand::MOLoad,
- MemoryLocation::UnknownSize, Alignment, MLD->getAAInfo(),
+ LocationSize::beforeOrAfterPointer(), Alignment, MLD->getAAInfo(),
MLD->getRanges());
Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT,
@@ -2150,8 +2151,8 @@ void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
LoMemVT.getStoreSize().getFixedValue());
MMO = DAG.getMachineFunction().getMachineMemOperand(
- MPI, MachineMemOperand::MOLoad, MemoryLocation::UnknownSize, Alignment,
- MLD->getAAInfo(), MLD->getRanges());
+ MPI, MachineMemOperand::MOLoad, LocationSize::beforeOrAfterPointer(),
+ Alignment, MLD->getAAInfo(), MLD->getRanges());
Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi,
HiMemVT, MMO, MLD->getAddressingMode(), ExtType,
@@ -2213,7 +2214,8 @@ void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
N->getPointerInfo(), MachineMemOperand::MOLoad,
- MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
+ LocationSize::beforeOrAfterPointer(), Alignment, N->getAAInfo(),
+ N->getRanges());
if (auto *MGT = dyn_cast<MaskedGatherSDNode>(N)) {
SDValue PassThru = MGT->getPassThru();
@@ -2880,10 +2882,10 @@ void DAGTypeLegalizer::SplitVecRes_VP_REVERSE(SDNode *N, SDValue &Lo,
auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
- PtrInfo, MachineMemOperand::MOStore, MemoryLocation::UnknownSize,
+ PtrInfo, MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer(),
Alignment);
MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
- PtrInfo, MachineMemOperand::MOLoad, MemoryLocation::UnknownSize,
+ PtrInfo, MachineMemOperand::MOLoad, LocationSize::beforeOrAfterPointer(),
Alignment);
unsigned EltWidth = VT.getScalarSizeInBits() / 8;
@@ -3474,7 +3476,8 @@ SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
SDValue Lo, Hi;
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
N->getPointerInfo(), MachineMemOperand::MOStore,
- MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
+ LocationSize::beforeOrAfterPointer(), Alignment, N->getAAInfo(),
+ N->getRanges());
Lo = DAG.getStoreVP(Ch, DL, DataLo, Ptr, Offset, MaskLo, EVLLo, LoMemVT, MMO,
N->getAddressingMode(), N->isTruncatingStore(),
@@ -3497,8 +3500,8 @@ SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
LoMemVT.getStoreSize().getFixedValue());
MMO = DAG.getMachineFunction().getMachineMemOperand(
- MPI, MachineMemOperand::MOStore, MemoryLocation::UnknownSize, Alignment,
- N->getAAInfo(), N->getRanges());
+ MPI, MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer(),
+ Alignment, N->getAAInfo(), N->getRanges());
Hi = DAG.getStoreVP(Ch, DL, DataHi, Ptr, Offset, MaskHi, EVLHi, HiMemVT, MMO,
N->getAddressingMode(), N->isTruncatingStore(),
@@ -3570,8 +3573,8 @@ SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(N->getPointerInfo().getAddrSpace()),
- MachineMemOperand::MOStore, MemoryLocation::UnknownSize, Alignment,
- N->getAAInfo(), N->getRanges());
+ MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer(),
+ Alignment, N->getAAInfo(), N->getRanges());
SDValue Hi = DAG.getStridedStoreVP(
N->getChain(), DL, HiData, Ptr, N->getOffset(), N->getStride(), HiMask,
@@ -3622,7 +3625,8 @@ SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
SDValue Lo, Hi, Res;
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
N->getPointerInfo(), MachineMemOperand::MOStore,
- MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
+ LocationSize::beforeOrAfterPointer(), Alignment, N->getAAInfo(),
+ N->getRanges());
Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO,
N->getAddressingMode(), N->isTruncatingStore(),
@@ -3647,8 +3651,8 @@ SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
LoMemVT.getStoreSize().getFixedValue());
MMO = DAG.getMachineFunction().getMachineMemOperand(
- MPI, MachineMemOperand::MOStore, MemoryLocation::UnknownSize, Alignment,
- N->getAAInfo(), N->getRanges());
+ MPI, MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer(),
+ Alignment, N->getAAInfo(), N->getRanges());
Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO,
N->getAddressingMode(), N->isTruncatingStore(),
@@ -3712,7 +3716,8 @@ SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
SDValue Lo;
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
N->getPointerInfo(), MachineMemOperand::MOStore,
- MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
+ LocationSize::beforeOrAfterPointer(), Alignment, N->getAAInfo(),
+ N->getRanges());
if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) {
SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 191b53f303f632..ea8b28c5cce7be 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -8341,11 +8341,10 @@ SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
SDValue SelectionDAG::getMemIntrinsicNode(
unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
- MachineMemOperand::Flags Flags, uint64_t Size, const AAMDNodes &AAInfo) {
- if (!Size && MemVT.isScalableVector())
- Size = MemoryLocation::UnknownSize;
- else if (!Size)
- Size = MemVT.getStoreSize();
+ MachineMemOperand::Flags Flags, LocationSize Size,
+ const AAMDNodes &AAInfo) {
+ if (Size.hasValue() && !Size.getValue())
+ Size = LocationSize::precise(MemVT.getStoreSize());
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO =
@@ -9027,10 +9026,10 @@ SDValue SelectionDAG::getStridedLoadVP(
if (PtrInfo.V.isNull())
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
- uint64_t Size = MemoryLocation::UnknownSize;
MachineFunction &MF = getMachineFunction();
- MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
- Alignment, AAInfo, Ranges);
+ MachineMemOperand *MMO = MF.getMachineMemOperand(
+ PtrInfo, MMOFlags, LocationSize::beforeOrAfterPointer(), Alignment,
+ AAInfo, Ranges);
return getStridedLoadVP(AM, ExtType, VT, DL, Chain, Ptr, Offset, Stride, Mask,
EVL, MemVT, MMO, IsExpanding);
}
@@ -9179,7 +9178,8 @@ SDValue SelectionDAG::getTruncStridedStoreVP(
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
- PtrInfo, MMOFlags, MemoryLocation::UnknownSize, Alignment, AAInfo);
+ PtrInfo, MMOFlags, LocationSize::beforeOrAfterPointer(), Alignment,
+ AAInfo);
return getTruncStridedStoreVP(Chain, DL, Val, Ptr, Stride, Mask, EVL, SVT,
MMO, IsCompressing);
}
@@ -11779,9 +11779,10 @@ MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
// We check here that the size of the memory operand fits within the size of
// the MMO. This is because the MMO might indicate only a possible address
// range instead of specifying the affected memory addresses precisely.
- assert((!MMO->getType().isValid() ||
- TypeSize::isKnownLE(memvt.getStoreSize(), MMO->getSize())) &&
- "Size mismatch!");
+ assert(
+ (!MMO->getType().isValid() ||
+ TypeSize::isKnownLE(memvt.getStoreSize(), MMO->getSize().getValue())) &&
+ "Size mismatch!");
}
/// Profile - Gather unique data for the node.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp
index 3572da19f1d35f..2a003e6d463e79 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp
@@ -85,9 +85,9 @@ bool BaseIndexOffset::equalBaseIndex(const BaseIndexOffset &Other,
}
bool BaseIndexOffset::computeAliasing(const SDNode *Op0,
- const std::optional<TypeSize> NumBytes0,
+ const LocationSize NumBytes0,
const SDNode *Op1,
- const std::optional<TypeSize> NumBytes1,
+ const LocationSize NumBytes1,
const SelectionDAG &DAG, bool &IsAlias) {
BaseIndexOffset BasePtr0 = match(Op0, DAG);
BaseIndexOffset BasePtr1 = match(Op1, DAG);
@@ -95,27 +95,23 @@ bool BaseIndexOffset::computeAliasing(const SDNode *Op0,
if (!(BasePtr0.getBase().getNode() && BasePtr1.getBase().getNode()))
return false;
int64_t PtrDiff;
- if (NumBytes0 && NumBytes1 &&
- BasePtr0.equalBaseIndex(BasePtr1, DAG, PtrDiff)) {
+ if (BasePtr0.equalBaseIndex(BasePtr1, DAG, PtrDiff)) {
// If the size of memory access is unknown, do not use it to analysis.
- // One example of unknown size memory access is to load/store scalable
- // vector objects on the stack.
// BasePtr1 is PtrDiff away from BasePtr0. They alias if none of the
// following situations arise:
- if (PtrDiff >= 0 && !NumBytes0->isScalable() &&
- NumBytes0->getFixedValue() != MemoryLocation::UnknownSize) {
+ if (PtrDiff >= 0 && NumBytes0.hasValue() && !NumBytes0.isScalable()) {
// [----BasePtr0----]
// [---BasePtr1--]
// ========PtrDiff========>
- IsAlias = !((int64_t)NumBytes0->getFixedValue() <= PtrDiff);
+ IsAlias = !((int64_t)NumBytes0.getValue().getFixedValue() <= PtrDiff);
return true;
}
- if (PtrDiff < 0 && !NumBytes1->isScalable() &&
- NumBytes1->getFixedValue() != MemoryLocation::UnknownSize) {
+ if (PtrDiff < 0 && NumBytes1.hasValue() && !NumBytes1.isScalable()) {
// [----BasePtr0----]
// [---BasePtr1--]
// =====(-PtrDiff)====>
- IsAlias = !((PtrDiff + (int64_t)NumBytes1->getFixedValue()) <= 0);
+ IsAlias =
+ !((PtrDiff + (int64_t)NumBytes1.getValue().getFixedValue()) <= 0);
return true;
}
return false;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 12ed4a82ee91a5..eb2dad25412325 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -4562,7 +4562,7 @@ void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
- MemoryLocation::UnknownSize, *Alignment, I.getAAMetadata());
+ LocationSize::beforeOrAfterPointer(), *Alignment, I.getAAMetadata());
SDValue StoreNode =
DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, VT, MMO,
ISD::UNINDEXED, false /* Truncating */, IsCompressing);
@@ -4666,9 +4666,7 @@ void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(AS), MachineMemOperand::MOStore,
- // TODO: Make MachineMemOperands aware of scalable
- // vectors.
- MemoryLocation::UnknownSize, Alignment, I.getAAMetadata());
+ LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata());
if (!UniformBase) {
Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
Index = getValue(Ptr);
@@ -4737,7 +4735,7 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
- MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
+ LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
SDValue Load =
DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO,
@@ -4773,9 +4771,8 @@ void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(AS), MachineMemOperand::MOLoad,
- // TODO: Make MachineMemOperands aware of scalable
- // vectors.
- MemoryLocation::UnknownSize, Alignment, I.getAAMetadata(), Ranges);
+ LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata(),
+ Ranges);
if (!UniformBase) {
Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
@@ -6688,7 +6685,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
auto MPI =
MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
- MPI, MachineMemOperand::MOStore, MemoryLocation::UnknownSize,
+ MPI, MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer(),
TempAlign);
Chain = DAG.getGetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
Res = DAG.getLoad(EnvVT, sdl, Chain, Temp, MPI);
@@ -6717,7 +6714,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
Chain = DAG.getStore(Chain, sdl, Env, Temp, MPI, TempAlign,
MachineMemOperand::MOStore);
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
- MPI, MachineMemOperand::MOLoad, MemoryLocation::UnknownSize,
+ MPI, MachineMemOperand::MOLoad, LocationSize::beforeOrAfterPointer(),
TempAlign);
Chain = DAG.getSetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
}
@@ -7860,7 +7857,7 @@ void SelectionDAGBuilder::visitVPLoad(
SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
- MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
+ LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
MMO, false /*IsExpanding */);
if (AddToChain)
@@ -7883,8 +7880,8 @@ void SelectionDAGBuilder::visitVPGather(
unsigned AS =
PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
- MachinePointerInfo(AS), MachineMemOperand::MOLoad,
- MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
+ MachinePointerInfo(AS), MachineMemOperand::MOLoad,
+ LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
SDValue Base, Index, Scale;
ISD::MemIndexType IndexType;
bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
@@ -7924,7 +7921,7 @@ void SelectionDAGBuilder::visitVPStore(
SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
- MemoryLocation::UnknownSize, *Alignment, AAInfo);
+ LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
ST = DAG.getStoreVP(getMemoryRoot(), DL, OpValues[0], Ptr, Offset,
OpValues[2], OpValues[3], VT, MMO, ISD::UNINDEXED,
/* IsTruncating */ false, /*IsCompressing*/ false);
@@ -7947,7 +7944,7 @@ void SelectionDAGBuilder::visitVPScatter(
PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(AS), MachineMemOperand::MOStore,
- MemoryLocation::UnknownSize, *Alignment, AAInfo);
+ LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
SDValue Base, Index, Scale;
ISD::MemIndexType IndexType;
bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
@@ -7989,7 +7986,7 @@ void SelectionDAGBuilder::visitVPStridedLoad(
SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
- MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
+ LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
SDValue LD = DAG.getStridedLoadVP(VT, DL, InChain, OpValues[0], OpValues[1],
OpValues[2], OpValues[3], MMO,
@@ -8011,7 +8008,7 @@ void SelectionDAGBuilder::visitVPStridedStore(
AAMDNodes AAInfo = VPIntrin.getAAMetadata();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
- MemoryLocation::UnknownSize, *Alignment, AAInfo);
+ LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
SDValue ST = DAG.getStridedStoreVP(
getMemoryRoot(), DL, OpValues[0], OpValues[1],
diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp
index 4783742a14ad7d..c0e8d085346103 100644
--- a/llvm/lib/CodeGen/TargetInstrInfo.cpp
+++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp
@@ -1365,7 +1365,7 @@ bool TargetInstrInfo::getMemOperandWithOffset(
const MachineInstr &MI, const MachineOperand *&BaseOp, int64_t &Offset,
bool &OffsetIsScalable, const TargetRegisterInfo *TRI) const {
SmallVector<const MachineOperand *, 4> BaseOps;
- unsigned Width;
+ LocationSize Width = 0;
if (!getMemOperandsWithOffsetWidth(MI, BaseOps, Offset, OffsetIsScalable,
Width, TRI) ||
BaseOps.size() != 1)
@@ -1554,7 +1554,7 @@ TargetInstrInfo::describeLoadedValue(const MachineInstr &MI,
SmallVector<uint64_t, 8> Ops;
DIExpression::appendOffset(Ops, Offset);
Ops.push_back(dwarf::DW_OP_deref_size);
- Ops.push_back(MMO->getSize());
+ Ops.push_back(MMO->getSize().getValue());
Expr = DIExpression::prependOpcodes(Expr, Ops);
return ParamLoadedValue(*BaseOp, Expr);
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 50cbd3672fbd0d..471440fa206465 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -2671,7 +2671,7 @@ bool AArch64InstrInfo::isCandidateToMergeOrPair(const MachineInstr &MI) const {
bool AArch64InstrInfo::getMemOperandsWithOffsetWidth(
const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
- int64_t &Offset, bool &OffsetIsScalable, unsigned &Width,
+ int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
const TargetRegisterInfo *TRI) const {
if (!LdSt.mayLoadOrStore())
return false;
@@ -2683,10 +2683,7 @@ bool AArch64InstrInfo::getMemOperandsWithOffsetWidth(
return false;
// The maximum vscale is 16 under AArch64, return the maximal extent for the
// vector.
- Width = WidthN.isScalable()
- ? WidthN.getKnownMinValue() * AArch64::SVEMaxBitsPerVector /
- AArch64::SVEBitsPerBlock
- : WidthN.getKnownMinValue();
+ Width = LocationSize::precise(WidthN);
BaseOps.push_back(BaseOp);
return true;
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index e97ff0a9758d69..191d4527fb534d 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -155,7 +155,7 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
bool getMemOperandsWithOffsetWidth(
const MachineInstr &MI, SmallVectorImpl<const MachineOperand *> &BaseOps,
- int64_t &Offset, bool &OffsetIsScalable, unsigned &Width,
+ int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
const TargetRegisterInfo *TRI) const override;
/// If \p OffsetIsScalable is set to 'true', the offset is scaled by `vscale`.
diff --git a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
index dc6d5b8950c34a..5da38f8e3adc39 100644
--- a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
@@ -2322,7 +2322,9 @@ bool AArch64LoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) {
// Get the needed alignments to check them if
// ldp-aligned-only/stp-aligned-only features are opted.
uint64_t MemAlignment = MemOp ? MemOp->getAlign().value() : -1;
- uint64_t TypeAlignment = MemOp ? Align(MemOp->getSize()).value() : -1;
+ uint64_t TypeAlignment = MemOp && MemOp->getSize().hasValue()
+ ? Align(MemOp->getSize().getValue()).value()
+ : -1;
// If a load arrives and ldp-aligned-only feature is opted, check that the
// alignment of the source pointer is at least double the alignment of the
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index bdaae4dd724d53..c601bb6e59aab0 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -2881,8 +2881,8 @@ bool AArch64InstructionSelector::select(MachineInstr &I) {
return false;
}
- uint64_t MemSizeInBytes = LdSt.getMemSize();
- unsigned MemSizeInBits = LdSt.getMemSizeInBits();
+ uint64_t MemSizeInBytes = LdSt.getMemSize().getValue();
+ unsigned MemSizeInBits = LdSt.getMemSizeInBits().getValue();
AtomicOrdering Order = LdSt.getMMO().getSuccessOrdering();
// Need special instructions for atomics that affect ordering.
@@ -3332,7 +3332,7 @@ bool AArch64InstructionSelector::select(MachineInstr &I) {
RBI.getRegBank(SrcReg, MRI, TRI)->getID() == AArch64::GPRRegBankID;
if (LoadMI && IsGPR) {
const MachineMemOperand *MemOp = *LoadMI->memoperands_begin();
- unsigned BytesLoaded = MemOp->getSize();
+ unsigned BytesLoaded = MemOp->getSize().getValue();
if (BytesLoaded < 4 && SrcTy.getSizeInBytes() == BytesLoaded)
return selectCopy(I, TII, MRI, TRI, RBI);
}
@@ -7767,7 +7767,7 @@ bool AArch64InstructionSelector::isLoadStoreOfNumBytes(
return false;
assert(MI.hasOneMemOperand() &&
"Expected load/store to have only one mem op!");
- return (*MI.memoperands_begin())->getSize() == NumBytes;
+ return (*MI.memoperands_begin())->getSize().getValue() == NumBytes;
}
bool AArch64InstructionSelector::isDef32(const MachineInstr &MI) const {
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
index 51c52aad359497..967ac79744b4a8 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
@@ -311,7 +311,7 @@ bool matchSplitStoreZero128(MachineInstr &MI, MachineRegisterInfo &MRI) {
LLT ValTy = MRI.getType(Store.getValueReg());
if (!ValTy.isVector() || ValTy.getSizeInBits() != 128)
return false;
- if (ValTy.getSizeInBits() != Store.getMemSizeInBits())
+ if (ValTy.getSizeInBits() != Store.getMemSizeInBits().getValue())
return false; // Don't split truncating stores.
if (!MRI.hasOneNonDBGUse(Store.getValueReg()))
return false;
@@ -658,7 +658,7 @@ bool AArch64PostLegalizerCombiner::optimizeConsecutiveMemOpAddressing(
APInt Offset;
LLT StoredValTy = MRI.getType(St->getValueReg());
unsigned ValSize = StoredValTy.getSizeInBits();
- if (ValSize < 32 || ValSize != St->getMMO().getSizeInBits())
+ if (ValSize < 32 || ValSize != St->getMMO().getSizeInBits().getValue())
continue;
Register PtrReg = St->getPointerReg();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 489b4f5a8d86a5..fb098a0db3f589 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -5562,7 +5562,7 @@ bool AMDGPULegalizerInfo::legalizeBufferStore(MachineInstr &MI,
Register RSrc = MI.getOperand(2).getReg();
MachineMemOperand *MMO = *MI.memoperands_begin();
- const int MemSize = MMO->getSize();
+ const int MemSize = MMO->getSize().getValue();
unsigned ImmOffset;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 03b6d19b2b3c06..081fb37a1443bf 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -1065,7 +1065,7 @@ bool AMDGPURegisterBankInfo::applyMappingLoad(
return false;
MachineMemOperand *MMO = *MI.memoperands_begin();
- const unsigned MemSize = 8 * MMO->getSize();
+ const unsigned MemSize = 8 * MMO->getSize().getValue();
// Scalar loads of size 8 or 16 bit with proper alignment may be widened to
// 32 bit. Check to see if we need to widen the memory access, 8 or 16 bit
// scalar loads should have a load size of 32 but memory access size of less
diff --git a/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp b/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp
index 442ae4dd7b34fe..1dcd6bb108d4fb 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp
@@ -212,7 +212,7 @@ class SIInsertHardClauses : public MachineFunctionPass {
int64_t Dummy1;
bool Dummy2;
- unsigned Dummy3;
+ LocationSize Dummy3 = 0;
SmallVector<const MachineOperand *, 4> BaseOps;
if (Type <= LAST_REAL_HARDCLAUSE_TYPE) {
if (!SII->getMemOperandsWithOffsetWidth(MI, BaseOps, Dummy1, Dummy2,
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index d4746b559d9256..2d0943ca6f37ed 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -355,7 +355,7 @@ static bool isStride64(unsigned Opc) {
bool SIInstrInfo::getMemOperandsWithOffsetWidth(
const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
- int64_t &Offset, bool &OffsetIsScalable, unsigned &Width,
+ int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
const TargetRegisterInfo *TRI) const {
if (!LdSt.mayLoadOrStore())
return false;
@@ -417,9 +417,10 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
if (DataOpIdx == -1) {
DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
- Width = getOpSize(LdSt, DataOpIdx);
+ unsigned Width2 = getOpSize(LdSt, DataOpIdx);
DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data1);
- Width += getOpSize(LdSt, DataOpIdx);
+ Width2 += getOpSize(LdSt, DataOpIdx);
+ Width = LocationSize::precise(Width2);
} else {
Width = getOpSize(LdSt, DataOpIdx);
}
@@ -3605,19 +3606,20 @@ memOpsHaveSameBaseOperands(ArrayRef<const MachineOperand *> BaseOps1,
return true;
}
-static bool offsetsDoNotOverlap(int WidthA, int OffsetA,
- int WidthB, int OffsetB) {
+static bool offsetsDoNotOverlap(LocationSize WidthA, int OffsetA,
+ LocationSize WidthB, int OffsetB) {
int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
- int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
- return LowOffset + LowWidth <= HighOffset;
+ LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
+ return LowWidth.hasValue() &&
+ LowOffset + (int)LowWidth.getValue() <= HighOffset;
}
bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
const MachineInstr &MIb) const {
SmallVector<const MachineOperand *, 4> BaseOps0, BaseOps1;
int64_t Offset0, Offset1;
- unsigned Dummy0, Dummy1;
+ LocationSize Dummy0 = 0, Dummy1 = 0;
bool Offset0IsScalable, Offset1IsScalable;
if (!getMemOperandsWithOffsetWidth(MIa, BaseOps0, Offset0, Offset0IsScalable,
Dummy0, &RI) ||
@@ -3632,8 +3634,8 @@ bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
// FIXME: Handle ds_read2 / ds_write2.
return false;
}
- unsigned Width0 = MIa.memoperands().front()->getSize();
- unsigned Width1 = MIb.memoperands().front()->getSize();
+ LocationSize Width0 = MIa.memoperands().front()->getSize();
+ LocationSize Width1 = MIb.memoperands().front()->getSize();
return offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1);
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 32a81d67cc246c..722acebc19806e 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -227,7 +227,7 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
bool getMemOperandsWithOffsetWidth(
const MachineInstr &LdSt,
SmallVectorImpl<const MachineOperand *> &BaseOps, int64_t &Offset,
- bool &OffsetIsScalable, unsigned &Width,
+ bool &OffsetIsScalable, LocationSize &Width,
const TargetRegisterInfo *TRI) const final;
bool shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
diff --git a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
index 17105965471f65..fc14f147a26888 100644
--- a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
@@ -825,7 +825,7 @@ SILoadStoreOptimizer::combineKnownAdjacentMMOs(const CombineInfo &CI,
const MachineMemOperand *MMOa = *CI.I->memoperands_begin();
const MachineMemOperand *MMOb = *Paired.I->memoperands_begin();
- unsigned Size = MMOa->getSize() + MMOb->getSize();
+ unsigned Size = MMOa->getSize().getValue() + MMOb->getSize().getValue();
// A base pointer for the combined operation is the same as the leading
// operation's pointer.
diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
index b85107ec471911..a79b8f3d4ea4cd 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -3710,7 +3710,7 @@ unsigned ARMBaseInstrInfo::getNumLDMAddresses(const MachineInstr &MI) const {
for (MachineInstr::mmo_iterator I = MI.memoperands_begin(),
E = MI.memoperands_end();
I != E; ++I) {
- Size += (*I)->getSize();
+ Size += (*I)->getSize().getValue();
}
// FIXME: The scheduler currently can't handle values larger than 16. But
// the values can actually go up to 32 for floating-point load/store
diff --git a/llvm/lib/Target/ARM/ARMHazardRecognizer.cpp b/llvm/lib/Target/ARM/ARMHazardRecognizer.cpp
index 9b26aac6c0b71e..34b6f0575f7276 100644
--- a/llvm/lib/Target/ARM/ARMHazardRecognizer.cpp
+++ b/llvm/lib/Target/ARM/ARMHazardRecognizer.cpp
@@ -191,7 +191,7 @@ ARMBankConflictHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
auto BasePseudoVal0 = MO0->getPseudoValue();
int64_t Offset0 = 0;
- if (MO0->getSize() > 4)
+ if (!MO0->getSize().hasValue() || MO0->getSize().getValue() > 4)
return NoHazard;
bool SPvalid = false;
@@ -259,8 +259,8 @@ void ARMBankConflictHazardRecognizer::EmitInstruction(SUnit *SU) {
return;
auto MO = *MI.memoperands().begin();
- uint64_t Size1 = MO->getSize();
- if (Size1 > 4)
+ LocationSize Size1 = MO->getSize();
+ if (Size1.hasValue() && Size1.getValue() > 4)
return;
Accesses.push_back(&MI);
}
diff --git a/llvm/lib/Target/BPF/BPFISelDAGToDAG.cpp b/llvm/lib/Target/BPF/BPFISelDAGToDAG.cpp
index 909c7c005735b6..4a69fc69e5cbab 100644
--- a/llvm/lib/Target/BPF/BPFISelDAGToDAG.cpp
+++ b/llvm/lib/Target/BPF/BPFISelDAGToDAG.cpp
@@ -242,7 +242,9 @@ void BPFDAGToDAGISel::PreprocessLoad(SDNode *Node,
bool to_replace = false;
SDLoc DL(Node);
const LoadSDNode *LD = cast<LoadSDNode>(Node);
- uint64_t size = LD->getMemOperand()->getSize();
+ if (!LD->getMemOperand()->getSize().hasValue())
+ return;
+ uint64_t size = LD->getMemOperand()->getSize().getValue();
if (!size || size > 8 || (size & (size - 1)) || !LD->isSimple())
return;
diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
index 1689b8f1e132d5..08c91f8b17427b 100644
--- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
@@ -3042,7 +3042,7 @@ bool HexagonInstrInfo::addLatencyToSchedule(const MachineInstr &MI1,
/// Get the base register and byte offset of a load/store instr.
bool HexagonInstrInfo::getMemOperandsWithOffsetWidth(
const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
- int64_t &Offset, bool &OffsetIsScalable, unsigned &Width,
+ int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
const TargetRegisterInfo *TRI) const {
OffsetIsScalable = false;
const MachineOperand *BaseOp = getBaseAndOffset(LdSt, Offset, Width);
@@ -3258,9 +3258,9 @@ unsigned HexagonInstrInfo::getAddrMode(const MachineInstr &MI) const {
// returned in Offset and the access size is returned in AccessSize.
// If the base operand has a subregister or the offset field does not contain
// an immediate value, return nullptr.
-MachineOperand *HexagonInstrInfo::getBaseAndOffset(const MachineInstr &MI,
- int64_t &Offset,
- unsigned &AccessSize) const {
+MachineOperand *
+HexagonInstrInfo::getBaseAndOffset(const MachineInstr &MI, int64_t &Offset,
+ LocationSize &AccessSize) const {
// Return if it is not a base+offset type instruction or a MemOp.
if (getAddrMode(MI) != HexagonII::BaseImmOffset &&
getAddrMode(MI) != HexagonII::BaseLongOffset &&
diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.h b/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
index 645b57f4664df2..0216644c8ba533 100644
--- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
@@ -208,7 +208,7 @@ class HexagonInstrInfo : public HexagonGenInstrInfo {
bool getMemOperandsWithOffsetWidth(
const MachineInstr &LdSt,
SmallVectorImpl<const MachineOperand *> &BaseOps, int64_t &Offset,
- bool &OffsetIsScalable, unsigned &Width,
+ bool &OffsetIsScalable, LocationSize &Width,
const TargetRegisterInfo *TRI) const override;
/// Reverses the branch condition of the specified condition list,
@@ -437,7 +437,7 @@ class HexagonInstrInfo : public HexagonGenInstrInfo {
unsigned getAddrMode(const MachineInstr &MI) const;
MachineOperand *getBaseAndOffset(const MachineInstr &MI, int64_t &Offset,
- unsigned &AccessSize) const;
+ LocationSize &AccessSize) const;
SmallVector<MachineInstr*,2> getBranchingInstrs(MachineBasicBlock& MBB) const;
unsigned getCExtOpNum(const MachineInstr &MI) const;
HexagonII::CompoundGroup
diff --git a/llvm/lib/Target/Hexagon/HexagonStoreWidening.cpp b/llvm/lib/Target/Hexagon/HexagonStoreWidening.cpp
index f3facbe511a2f7..9d8e5c53b8227a 100644
--- a/llvm/lib/Target/Hexagon/HexagonStoreWidening.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonStoreWidening.cpp
@@ -172,16 +172,14 @@ bool HexagonStoreWidening::instrAliased(InstrGroup &Stores,
if (!MMO.getValue())
return true;
- MemoryLocation L(MMO.getValue(), MMO.getSize().getFixedValue(),
- MMO.getAAInfo());
+ MemoryLocation L(MMO.getValue(), MMO.getSize(), MMO.getAAInfo());
for (auto *SI : Stores) {
const MachineMemOperand &SMO = getStoreTarget(SI);
if (!SMO.getValue())
return true;
- MemoryLocation SL(SMO.getValue(), SMO.getSize().getFixedValue(),
- SMO.getAAInfo());
+ MemoryLocation SL(SMO.getValue(), SMO.getSize(), SMO.getAAInfo());
if (!AA->isNoAlias(L, SL))
return true;
}
@@ -294,8 +292,8 @@ bool HexagonStoreWidening::storesAreAdjacent(const MachineInstr *S1,
int Off1 = S1->getOperand(1).getImm();
int Off2 = S2->getOperand(1).getImm();
- return (Off1 >= 0) ? Off1+S1MO.getSize() == unsigned(Off2)
- : int(Off1+S1MO.getSize()) == Off2;
+ return (Off1 >= 0) ? Off1 + S1MO.getSize().getValue() == unsigned(Off2)
+ : int(Off1 + S1MO.getSize().getValue()) == Off2;
}
/// Given a sequence of adjacent stores, and a maximum size of a single wide
@@ -317,7 +315,7 @@ bool HexagonStoreWidening::selectStores(InstrGroup::iterator Begin,
assert(!FirstMI->memoperands_empty() && "Expecting some memory operands");
const MachineMemOperand &FirstMMO = getStoreTarget(FirstMI);
unsigned Alignment = FirstMMO.getAlign().value();
- unsigned SizeAccum = FirstMMO.getSize();
+ unsigned SizeAccum = FirstMMO.getSize().getValue();
unsigned FirstOffset = getStoreOffset(FirstMI);
// The initial value of SizeAccum should always be a power of 2.
@@ -359,7 +357,7 @@ bool HexagonStoreWidening::selectStores(InstrGroup::iterator Begin,
if (!storesAreAdjacent(S1, S2))
break;
- unsigned S2Size = getStoreTarget(S2).getSize();
+ unsigned S2Size = getStoreTarget(S2).getSize().getValue();
if (SizeAccum + S2Size > std::min(MaxSize, Alignment))
break;
@@ -407,7 +405,7 @@ bool HexagonStoreWidening::createWideStores(InstrGroup &OG, InstrGroup &NG,
MachineOperand &SO = MI->getOperand(2); // Source.
assert(SO.isImm() && "Expecting an immediate operand");
- unsigned NBits = MMO.getSize()*8;
+ unsigned NBits = MMO.getSize().getValue() * 8;
unsigned Mask = (0xFFFFFFFFU >> (32-NBits));
unsigned Val = (SO.getImm() & Mask) << Shift;
Acc |= Val;
diff --git a/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp b/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp
index 4df811f188df68..2d320e6b0cad75 100644
--- a/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp
@@ -395,10 +395,11 @@ void HexagonSubtarget::BankConflictMutation::apply(ScheduleDAGInstrs *DAG) {
HII.getAddrMode(L0) != HexagonII::BaseImmOffset)
continue;
int64_t Offset0;
- unsigned Size0;
+ LocationSize Size0 = 0;
MachineOperand *BaseOp0 = HII.getBaseAndOffset(L0, Offset0, Size0);
// Is the access size is longer than the L1 cache line, skip the check.
- if (BaseOp0 == nullptr || !BaseOp0->isReg() || Size0 >= 32)
+ if (BaseOp0 == nullptr || !BaseOp0->isReg() || !Size0.hasValue() ||
+ Size0.getValue() >= 32)
continue;
// Scan only up to 32 instructions ahead (to avoid n^2 complexity).
for (unsigned j = i+1, m = std::min(i+32, e); j != m; ++j) {
@@ -408,10 +409,10 @@ void HexagonSubtarget::BankConflictMutation::apply(ScheduleDAGInstrs *DAG) {
HII.getAddrMode(L1) != HexagonII::BaseImmOffset)
continue;
int64_t Offset1;
- unsigned Size1;
+ LocationSize Size1 = 0;
MachineOperand *BaseOp1 = HII.getBaseAndOffset(L1, Offset1, Size1);
- if (BaseOp1 == nullptr || !BaseOp1->isReg() || Size1 >= 32 ||
- BaseOp0->getReg() != BaseOp1->getReg())
+ if (BaseOp1 == nullptr || !BaseOp1->isReg() || !Size0.hasValue() ||
+ Size1.getValue() >= 32 || BaseOp0->getReg() != BaseOp1->getReg())
continue;
// Check bits 3 and 4 of the offset: if they differ, a bank conflict
// is unlikely.
diff --git a/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp b/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp
index aa7e8846406dd8..53889a81bd7476 100644
--- a/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp
+++ b/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp
@@ -102,14 +102,15 @@ bool LanaiInstrInfo::areMemAccessesTriviallyDisjoint(
const TargetRegisterInfo *TRI = &getRegisterInfo();
const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
int64_t OffsetA = 0, OffsetB = 0;
- unsigned int WidthA = 0, WidthB = 0;
+ LocationSize WidthA = 0, WidthB = 0;
if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) &&
getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) {
if (BaseOpA->isIdenticalTo(*BaseOpB)) {
int LowOffset = std::min(OffsetA, OffsetB);
int HighOffset = std::max(OffsetA, OffsetB);
- int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
- if (LowOffset + LowWidth <= HighOffset)
+ LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
+ if (LowWidth.hasValue() &&
+ LowOffset + (int)LowWidth.getValue() <= HighOffset)
return true;
}
}
@@ -752,7 +753,7 @@ unsigned LanaiInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
bool LanaiInstrInfo::getMemOperandWithOffsetWidth(
const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset,
- unsigned &Width, const TargetRegisterInfo * /*TRI*/) const {
+ LocationSize &Width, const TargetRegisterInfo * /*TRI*/) const {
// Handle only loads/stores with base register followed by immediate offset
// and with add as ALU op.
if (LdSt.getNumOperands() != 4)
@@ -793,7 +794,7 @@ bool LanaiInstrInfo::getMemOperandWithOffsetWidth(
bool LanaiInstrInfo::getMemOperandsWithOffsetWidth(
const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
- int64_t &Offset, bool &OffsetIsScalable, unsigned &Width,
+ int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
const TargetRegisterInfo *TRI) const {
switch (LdSt.getOpcode()) {
default:
diff --git a/llvm/lib/Target/Lanai/LanaiInstrInfo.h b/llvm/lib/Target/Lanai/LanaiInstrInfo.h
index 62f6240c6e4681..710118be27dde7 100644
--- a/llvm/lib/Target/Lanai/LanaiInstrInfo.h
+++ b/llvm/lib/Target/Lanai/LanaiInstrInfo.h
@@ -70,12 +70,12 @@ class LanaiInstrInfo : public LanaiGenInstrInfo {
bool getMemOperandsWithOffsetWidth(
const MachineInstr &LdSt,
SmallVectorImpl<const MachineOperand *> &BaseOps, int64_t &Offset,
- bool &OffsetIsScalable, unsigned &Width,
+ bool &OffsetIsScalable, LocationSize &Width,
const TargetRegisterInfo *TRI) const override;
bool getMemOperandWithOffsetWidth(const MachineInstr &LdSt,
const MachineOperand *&BaseOp,
- int64_t &Offset, unsigned &Width,
+ int64_t &Offset, LocationSize &Width,
const TargetRegisterInfo *TRI) const;
std::pair<unsigned, unsigned>
diff --git a/llvm/lib/Target/Mips/MipsInstructionSelector.cpp b/llvm/lib/Target/Mips/MipsInstructionSelector.cpp
index 4478a574a24088..6cb8fbafd01660 100644
--- a/llvm/lib/Target/Mips/MipsInstructionSelector.cpp
+++ b/llvm/lib/Target/Mips/MipsInstructionSelector.cpp
@@ -184,7 +184,8 @@ MipsInstructionSelector::selectLoadStoreOpCode(MachineInstr &I,
const Register ValueReg = I.getOperand(0).getReg();
const LLT Ty = MRI.getType(ValueReg);
const unsigned TySize = Ty.getSizeInBits();
- const unsigned MemSizeInBytes = (*I.memoperands_begin())->getSize();
+ const unsigned MemSizeInBytes =
+ (*I.memoperands_begin())->getSize().getValue();
unsigned Opc = I.getOpcode();
const bool isStore = Opc == TargetOpcode::G_STORE;
@@ -462,7 +463,7 @@ bool MipsInstructionSelector::select(MachineInstr &I) {
}
// Unaligned memory access
- if (MMO->getAlign() < MMO->getSize() &&
+ if ((!MMO->getSize().hasValue() || MMO->getAlign() < MMO->getSize().getValue()) &&
!STI.systemSupportsUnalignedAccess()) {
if (MMO->getSize() != 4 || !isRegInGprb(I.getOperand(0).getReg(), MRI))
return false;
diff --git a/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp b/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp
index 14f26201e6c09b..d7fcd8b8d754c5 100644
--- a/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp
+++ b/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp
@@ -343,7 +343,7 @@ bool MipsLegalizerInfo::legalizeCustom(LegalizerHelper &Helper,
switch (MI.getOpcode()) {
case G_LOAD:
case G_STORE: {
- unsigned MemSize = (**MI.memoperands_begin()).getSize();
+ unsigned MemSize = (**MI.memoperands_begin()).getSize().getValue();
Register Val = MI.getOperand(0).getReg();
unsigned Size = MRI.getType(Val).getSizeInBits();
diff --git a/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp b/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
index acf0d6312ef51b..26926d38b290c7 100644
--- a/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
@@ -70,9 +70,9 @@ class MipsPreLegalizerCombinerImpl : public Combiner {
// subtarget doesn't support them.
auto MMO = *MI.memoperands_begin();
const MipsSubtarget &STI = MI.getMF()->getSubtarget<MipsSubtarget>();
- if (!isPowerOf2_64(MMO->getSize()))
+ if (!MMO->getSize().hasValue() || !isPowerOf2_64(MMO->getSize().getValue()))
return false;
- bool isUnaligned = MMO->getAlign() < MMO->getSize();
+ bool isUnaligned = MMO->getAlign() < MMO->getSize().getValue();
if (!STI.systemSupportsUnalignedAccess() && isUnaligned)
return false;
diff --git a/llvm/lib/Target/Mips/MipsRegisterBankInfo.cpp b/llvm/lib/Target/Mips/MipsRegisterBankInfo.cpp
index b38ca3f09ffbe2..9770ed1bc3027c 100644
--- a/llvm/lib/Target/Mips/MipsRegisterBankInfo.cpp
+++ b/llvm/lib/Target/Mips/MipsRegisterBankInfo.cpp
@@ -155,7 +155,8 @@ static bool isGprbTwoInstrUnalignedLoadOrStore(const MachineInstr *MI) {
auto MMO = *MI->memoperands_begin();
const MipsSubtarget &STI = MI->getMF()->getSubtarget<MipsSubtarget>();
if (MMO->getSize() == 4 && (!STI.systemSupportsUnalignedAccess() &&
- MMO->getAlign() < MMO->getSize()))
+ (!MMO->getSize().hasValue() ||
+ MMO->getAlign() < MMO->getSize().getValue())))
return true;
}
return false;
diff --git a/llvm/lib/Target/PowerPC/GISel/PPCInstructionSelector.cpp b/llvm/lib/Target/PowerPC/GISel/PPCInstructionSelector.cpp
index 3fd7a1ad9efa59..fdffbcd71d0490 100644
--- a/llvm/lib/Target/PowerPC/GISel/PPCInstructionSelector.cpp
+++ b/llvm/lib/Target/PowerPC/GISel/PPCInstructionSelector.cpp
@@ -739,7 +739,7 @@ bool PPCInstructionSelector::select(MachineInstr &I) {
auto SelectLoadStoreAddressingMode = [&]() -> MachineInstr * {
const unsigned NewOpc = selectLoadStoreOp(
I.getOpcode(), RBI.getRegBank(LdSt.getReg(0), MRI, TRI)->getID(),
- LdSt.getMemSizeInBits());
+ LdSt.getMemSizeInBits().getValue());
if (NewOpc == I.getOpcode())
return nullptr;
diff --git a/llvm/lib/Target/PowerPC/PPCHazardRecognizers.cpp b/llvm/lib/Target/PowerPC/PPCHazardRecognizers.cpp
index ffaa3e05c84790..44549d08ec8825 100644
--- a/llvm/lib/Target/PowerPC/PPCHazardRecognizers.cpp
+++ b/llvm/lib/Target/PowerPC/PPCHazardRecognizers.cpp
@@ -374,8 +374,8 @@ getHazardType(SUnit *SU, int Stalls) {
// overlapping address.
if (isLoad && NumStores && !MI->memoperands_empty()) {
MachineMemOperand *MO = *MI->memoperands_begin();
- if (isLoadOfStoredAddress(MO->getSize(),
- MO->getOffset(), MO->getValue()))
+ if (isLoadOfStoredAddress(MO->getSize().getValue(), MO->getOffset(),
+ MO->getValue()))
return NoopHazard;
}
@@ -401,7 +401,7 @@ void PPCHazardRecognizer970::EmitInstruction(SUnit *SU) {
// Track the address stored to.
if (isStore && NumStores < 4 && !MI->memoperands_empty()) {
MachineMemOperand *MO = *MI->memoperands_begin();
- StoreSize[NumStores] = MO->getSize();
+ StoreSize[NumStores] = MO->getSize().getValue();
StoreOffset[NumStores] = MO->getOffset();
StoreValue[NumStores] = MO->getValue();
++NumStores;
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index acaf98b62fd94d..c77dff5fbb7901 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -8493,7 +8493,7 @@ bool PPCTargetLowering::directMoveIsProfitable(const SDValue &Op) const {
// If there is no LXSIBZX/LXSIHZX, like Power8,
// prefer direct move if the memory size is 1 or 2 bytes.
MachineMemOperand *MMO = cast<LoadSDNode>(Origin)->getMemOperand();
- if (!Subtarget.hasP9Vector() && MMO->getSize() <= 2)
+ if (!Subtarget.hasP9Vector() && MMO->getSize().getValue() <= 2)
return true;
for (SDNode::use_iterator UI = Origin->use_begin(),
@@ -15034,7 +15034,7 @@ SDValue PPCTargetLowering::expandVSXLoadForLE(SDNode *N,
// If the MMO suggests this isn't a load of a full vector, leave
// things alone. For a built-in, we have to make the change for
// correctness, so if there is a size problem that will be a bug.
- if (MMO->getSize() < 16)
+ if (MMO->getSize().getValue() < 16)
return SDValue();
break;
}
@@ -15102,7 +15102,7 @@ SDValue PPCTargetLowering::expandVSXStoreForLE(SDNode *N,
// If the MMO suggests this isn't a store of a full vector, leave
// things alone. For a built-in, we have to make the change for
// correctness, so if there is a size problem that will be a bug.
- if (MMO->getSize() < 16)
+ if (MMO->getSize().getValue() < 16)
return SDValue();
break;
}
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
index d0a6cced1b1974..8673c9df008c74 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
@@ -2827,7 +2827,7 @@ bool PPCInstrInfo::optimizeCmpPostRA(MachineInstr &CmpMI) const {
bool PPCInstrInfo::getMemOperandsWithOffsetWidth(
const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
- int64_t &Offset, bool &OffsetIsScalable, unsigned &Width,
+ int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
const TargetRegisterInfo *TRI) const {
const MachineOperand *BaseOp;
OffsetIsScalable = false;
@@ -2919,7 +2919,7 @@ bool PPCInstrInfo::shouldClusterMemOps(
return false;
int64_t Offset1 = 0, Offset2 = 0;
- unsigned Width1 = 0, Width2 = 0;
+ LocationSize Width1 = 0, Width2 = 0;
const MachineOperand *Base1 = nullptr, *Base2 = nullptr;
if (!getMemOperandWithOffsetWidth(FirstLdSt, Base1, Offset1, Width1, TRI) ||
!getMemOperandWithOffsetWidth(SecondLdSt, Base2, Offset2, Width2, TRI) ||
@@ -2930,7 +2930,7 @@ bool PPCInstrInfo::shouldClusterMemOps(
"getMemOperandWithOffsetWidth return incorrect base op");
// The caller should already have ordered FirstMemOp/SecondMemOp by offset.
assert(Offset1 <= Offset2 && "Caller should have ordered offsets.");
- return Offset1 + Width1 == Offset2;
+ return Offset1 + (int64_t)Width1.getValue() == Offset2;
}
/// GetInstSize - Return the number of bytes of code the specified
@@ -5467,7 +5467,7 @@ MachineInstr *PPCInstrInfo::findLoopInstr(
// memory width. Width is the size of memory that is being loaded/stored.
bool PPCInstrInfo::getMemOperandWithOffsetWidth(
const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset,
- unsigned &Width, const TargetRegisterInfo *TRI) const {
+ LocationSize &Width, const TargetRegisterInfo *TRI) const {
if (!LdSt.mayLoadOrStore() || LdSt.getNumExplicitOperands() != 3)
return false;
@@ -5505,14 +5505,15 @@ bool PPCInstrInfo::areMemAccessesTriviallyDisjoint(
const TargetRegisterInfo *TRI = &getRegisterInfo();
const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
int64_t OffsetA = 0, OffsetB = 0;
- unsigned int WidthA = 0, WidthB = 0;
+ LocationSize WidthA = 0, WidthB = 0;
if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) &&
getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) {
if (BaseOpA->isIdenticalTo(*BaseOpB)) {
int LowOffset = std::min(OffsetA, OffsetB);
int HighOffset = std::max(OffsetA, OffsetB);
- int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
- if (LowOffset + LowWidth <= HighOffset)
+ LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
+ if (LowWidth.hasValue() &&
+ LowOffset + (int)LowWidth.getValue() <= HighOffset)
return true;
}
}
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.h b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
index 75f9cd1c206d82..1f45a630207441 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
@@ -543,7 +543,7 @@ class PPCInstrInfo : public PPCGenInstrInfo {
/// loaded/stored (e.g. 1, 2, 4, 8).
bool getMemOperandWithOffsetWidth(const MachineInstr &LdSt,
const MachineOperand *&BaseOp,
- int64_t &Offset, unsigned &Width,
+ int64_t &Offset, LocationSize &Width,
const TargetRegisterInfo *TRI) const;
bool optimizeCmpPostRA(MachineInstr &MI) const;
@@ -553,7 +553,7 @@ class PPCInstrInfo : public PPCGenInstrInfo {
bool getMemOperandsWithOffsetWidth(
const MachineInstr &LdSt,
SmallVectorImpl<const MachineOperand *> &BaseOps, int64_t &Offset,
- bool &OffsetIsScalable, unsigned &Width,
+ bool &OffsetIsScalable, LocationSize &Width,
const TargetRegisterInfo *TRI) const override;
/// Returns true if the two given memory operations should be scheduled
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 1dcff7eb563e20..ba565686d7285a 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -655,7 +655,7 @@ void RISCVInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
if (IsScalableVector) {
MachineMemOperand *MMO = MF->getMachineMemOperand(
MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore,
- MemoryLocation::UnknownSize, MFI.getObjectAlign(FI));
+ LocationSize::beforeOrAfterPointer(), MFI.getObjectAlign(FI));
MFI.setStackID(FI, TargetStackID::ScalableVector);
BuildMI(MBB, I, DebugLoc(), get(Opcode))
@@ -738,7 +738,7 @@ void RISCVInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
if (IsScalableVector) {
MachineMemOperand *MMO = MF->getMachineMemOperand(
MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOLoad,
- MemoryLocation::UnknownSize, MFI.getObjectAlign(FI));
+ LocationSize::beforeOrAfterPointer(), MFI.getObjectAlign(FI));
MFI.setStackID(FI, TargetStackID::ScalableVector);
BuildMI(MBB, I, DebugLoc(), get(Opcode), DstReg)
@@ -2196,7 +2196,7 @@ MachineInstr *RISCVInstrInfo::emitLdStWithAddr(MachineInstr &MemI,
bool RISCVInstrInfo::getMemOperandsWithOffsetWidth(
const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
- int64_t &Offset, bool &OffsetIsScalable, unsigned &Width,
+ int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
const TargetRegisterInfo *TRI) const {
if (!LdSt.mayLoadOrStore())
return false;
@@ -2296,7 +2296,7 @@ bool RISCVInstrInfo::shouldClusterMemOps(
// function) and set it as appropriate.
bool RISCVInstrInfo::getMemOperandWithOffsetWidth(
const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset,
- unsigned &Width, const TargetRegisterInfo *TRI) const {
+ LocationSize &Width, const TargetRegisterInfo *TRI) const {
if (!LdSt.mayLoadOrStore())
return false;
@@ -2335,14 +2335,15 @@ bool RISCVInstrInfo::areMemAccessesTriviallyDisjoint(
const TargetRegisterInfo *TRI = STI.getRegisterInfo();
const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
int64_t OffsetA = 0, OffsetB = 0;
- unsigned int WidthA = 0, WidthB = 0;
+ LocationSize WidthA = 0, WidthB = 0;
if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) &&
getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) {
if (BaseOpA->isIdenticalTo(*BaseOpB)) {
int LowOffset = std::min(OffsetA, OffsetB);
int HighOffset = std::max(OffsetA, OffsetB);
- int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
- if (LowOffset + LowWidth <= HighOffset)
+ LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
+ if (LowWidth.hasValue() &&
+ LowOffset + (int)LowWidth.getValue() <= HighOffset)
return true;
}
}
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
index 7e1d3f31180650..5d11ed66f5420a 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
@@ -154,7 +154,7 @@ class RISCVInstrInfo : public RISCVGenInstrInfo {
bool getMemOperandsWithOffsetWidth(
const MachineInstr &MI, SmallVectorImpl<const MachineOperand *> &BaseOps,
- int64_t &Offset, bool &OffsetIsScalable, unsigned &Width,
+ int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
const TargetRegisterInfo *TRI) const override;
bool shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
@@ -166,7 +166,7 @@ class RISCVInstrInfo : public RISCVGenInstrInfo {
bool getMemOperandWithOffsetWidth(const MachineInstr &LdSt,
const MachineOperand *&BaseOp,
- int64_t &Offset, unsigned &Width,
+ int64_t &Offset, LocationSize &Width,
const TargetRegisterInfo *TRI) const;
bool areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
index ac8c395f9064fb..335e9311594702 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
@@ -2023,11 +2023,12 @@ areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
}
if (SameVal) {
int OffsetA = MMOa->getOffset(), OffsetB = MMOb->getOffset();
- int WidthA = MMOa->getSize(), WidthB = MMOb->getSize();
+ LocationSize WidthA = MMOa->getSize(), WidthB = MMOb->getSize();
int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
- int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
- if (LowOffset + LowWidth <= HighOffset)
+ LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
+ if (LowWidth.hasValue() &&
+ LowOffset + (int)LowWidth.getValue() <= HighOffset)
return true;
}
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblySetP2AlignOperands.cpp b/llvm/lib/Target/WebAssembly/WebAssemblySetP2AlignOperands.cpp
index a453e7388e274f..5f5f3a58da3341 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblySetP2AlignOperands.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblySetP2AlignOperands.cpp
@@ -59,7 +59,7 @@ static void rewriteP2Align(MachineInstr &MI, unsigned OperandNo) {
"ISel should set p2align operands to 0");
assert(MI.hasOneMemOperand() &&
"Load and store instructions have exactly one mem operand");
- assert((*MI.memoperands_begin())->getSize() ==
+ assert((*MI.memoperands_begin())->getSize().getValue() ==
(UINT64_C(1) << WebAssembly::GetDefaultP2Align(MI.getOpcode())) &&
"Default p2align value should be natural");
assert(MI.getDesc().operands()[OperandNo].OperandType ==
diff --git a/llvm/lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp b/llvm/lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp
index 04931afdec51c3..a72eeb53915d65 100644
--- a/llvm/lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp
+++ b/llvm/lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp
@@ -521,8 +521,8 @@ bool X86AvoidSFBPass::alias(const MachineMemOperand &Op1,
return true;
int64_t MinOffset = std::min(Op1.getOffset(), Op2.getOffset());
- int64_t Overlapa = Op1.getSize() + Op1.getOffset() - MinOffset;
- int64_t Overlapb = Op2.getSize() + Op2.getOffset() - MinOffset;
+ int64_t Overlapa = Op1.getSize().getValue() + Op1.getOffset() - MinOffset;
+ int64_t Overlapb = Op2.getSize().getValue() + Op2.getOffset() - MinOffset;
return !AA->isNoAlias(
MemoryLocation(Op1.getValue(), Overlapa, Op1.getAAInfo()),
@@ -688,7 +688,7 @@ bool X86AvoidSFBPass::runOnMachineFunction(MachineFunction &MF) {
!isRelevantAddressingMode(PBInst) || !PBInst->hasOneMemOperand())
continue;
int64_t PBstDispImm = getDispOperand(PBInst).getImm();
- unsigned PBstSize = (*PBInst->memoperands_begin())->getSize();
+ unsigned PBstSize = (*PBInst->memoperands_begin())->getSize().getValue();
// This check doesn't cover all cases, but it will suffice for now.
// TODO: take branch probability into consideration, if the blocking
// store is in an unreached block, breaking the memcopy could lose
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 7b607c6f198c72..9567b9541b1f82 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -4446,7 +4446,7 @@ bool X86InstrInfo::preservesZeroValueInReg(
bool X86InstrInfo::getMemOperandsWithOffsetWidth(
const MachineInstr &MemOp, SmallVectorImpl<const MachineOperand *> &BaseOps,
- int64_t &Offset, bool &OffsetIsScalable, unsigned &Width,
+ int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
const TargetRegisterInfo *TRI) const {
const MCInstrDesc &Desc = MemOp.getDesc();
int MemRefBegin = X86II::getMemoryOperandNo(Desc.TSFlags);
diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h
index eac8d79eb8a32a..4e31617e828d2b 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/llvm/lib/Target/X86/X86InstrInfo.h
@@ -354,7 +354,7 @@ class X86InstrInfo final : public X86GenInstrInfo {
bool getMemOperandsWithOffsetWidth(
const MachineInstr &LdSt,
SmallVectorImpl<const MachineOperand *> &BaseOps, int64_t &Offset,
- bool &OffsetIsScalable, unsigned &Width,
+ bool &OffsetIsScalable, LocationSize &Width,
const TargetRegisterInfo *TRI) const override;
bool analyzeBranchPredicate(MachineBasicBlock &MBB,
TargetInstrInfo::MachineBranchPredicate &MBP,
diff --git a/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll b/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
index 431796fac1d29a..a3bd78e1cc0b2d 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
@@ -3,7 +3,7 @@
define void @UphPNR(target("aarch64.svcount") %predcnt) {
entry:
; CHECK: %0:ppr = COPY $p0
-; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr, vscale )
+; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr)
; CHECK: %1:pnr_p8to15 = COPY %0
; CHECK: INLINEASM &"ld1w {z0.s,z1.s,z2.s,z3.s}, $0/z, [x10]", 1 /* sideeffect attdialect */, 458761 /* reguse:PNR_p8to15 */, %1
; CHECK: RET_ReallyLR
@@ -17,7 +17,7 @@ entry:
define void @UpaPNR(target("aarch64.svcount") %predcnt) {
entry:
; CHECK: %0:ppr = COPY $p0
-; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr, vscale )
+; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr)
; CHECK: %1:pnr = COPY %0
; CHECK: INLINEASM &"ld1w {z0.s,z1.s,z2.s,z3.s}, $0/z, [x10]", 1 /* sideeffect attdialect */, 262153 /* reguse:PNR */, %1
; CHECK: RET_ReallyLR
@@ -31,7 +31,7 @@ entry:
define void @UplPNR(target("aarch64.svcount") %predcnt) {
entry:
; CHECK: %0:ppr = COPY $p0
-; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr, vscale )
+; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr)
; CHECK: %1:pnr_3b = COPY %0
; CHECK: INLINEASM &"fadd z0.h, $0/m, z0.h, #0.5", 1 /* sideeffect attdialect */, 393225 /* reguse:PNR_3b */, %1
; CHECK: RET_ReallyLR
diff --git a/llvm/test/CodeGen/AMDGPU/cluster_stores.ll b/llvm/test/CodeGen/AMDGPU/cluster_stores.ll
index e96e59042413d5..92a544dcfb5d86 100644
--- a/llvm/test/CodeGen/AMDGPU/cluster_stores.ll
+++ b/llvm/test/CodeGen/AMDGPU/cluster_stores.ll
@@ -11,11 +11,11 @@
; DBG-LABEL: cluster_load_cluster_store:
-; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
-; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
-; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
-; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
-; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(16)
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(4)
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(4)
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(4)
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(4)
; DBG: Cluster ld/st SU([[L1:[0-9]+]]) - SU([[L2:[0-9]+]])
; DBG: Cluster ld/st SU([[L2]]) - SU([[L3:[0-9]+]])
@@ -136,11 +136,11 @@ bb:
; DBG-LABEL: cluster_load_valu_cluster_store:
-; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
-; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
-; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
-; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
-; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(16)
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(4)
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(4)
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(4)
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(4)
; DBG: Cluster ld/st SU([[L1:[0-9]+]]) - SU([[L2:[0-9]+]])
; DBG: Cluster ld/st SU([[L2]]) - SU([[L3:[0-9]+]])
@@ -266,8 +266,8 @@ bb:
; Cluster loads from the same texture with different coordinates
; DBG-LABEL: cluster_image_load:
-; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
-; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(16)
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(16)
; DBG: {{^}}Cluster ld/st [[SU1:SU\([0-9]+\)]] - [[SU2:SU\([0-9]+\)]]
; DBG: {{^}}[[SU1]]: {{.*}} IMAGE_LOAD
; DBG: {{^}}[[SU2]]: {{.*}} IMAGE_LOAD
@@ -335,8 +335,8 @@ entry:
; Don't cluster loads from different textures
; DBG-LABEL: no_cluster_image_load:
-; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
-; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(16)
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(16)
; DBG-NOT: {{^}}Cluster ld/st
define amdgpu_ps void @no_cluster_image_load(<8 x i32> inreg %src1, <8 x i32> inreg %src2, <8 x i32> inreg %dst, i32 %x, i32 %y) {
; GFX9-LABEL: no_cluster_image_load:
@@ -387,8 +387,8 @@ entry:
; Cluster loads from the same texture and sampler with different coordinates
; DBG-LABEL: cluster_image_sample:
-; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
-; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(16)
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(16)
; DBG: {{^}}Cluster ld/st [[SU1:SU\([0-9]+\)]] - [[SU2:SU\([0-9]+\)]]
; DBG: {{^}}[[SU1]]: {{.*}} IMAGE_SAMPLE
; DBG: {{^}}[[SU2]]: {{.*}} IMAGE_SAMPLE
diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
index 2391e9fea6e3fa..f3251974b40a32 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
@@ -17,7 +17,7 @@ define void @vpmerge_vpload_store(<vscale x 2 x i32> %passthru, <vscale x 2 x i3
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vrnov0 = COPY $v8
; CHECK-NEXT: $v0 = COPY [[COPY1]]
; CHECK-NEXT: [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], $v0, [[COPY]], 5 /* e32 */, 0 /* tu, mu */ :: (load unknown-size from %ir.p, align 8)
- ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store (<vscale x 1 x s64>) into %ir.p, vscale )
+ ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store (<vscale x 1 x s64>) into %ir.p)
; CHECK-NEXT: PseudoRET
%splat = insertelement <vscale x 2 x i1> poison, i1 -1, i32 0
%mask = shufflevector <vscale x 2 x i1> %splat, <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
@@ -38,7 +38,7 @@ define void @vpselect_vpload_store(<vscale x 2 x i32> %passthru, <vscale x 2 x i
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vrnov0 = COPY $v8
; CHECK-NEXT: $v0 = COPY [[COPY1]]
; CHECK-NEXT: [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], $v0, [[COPY]], 5 /* e32 */, 1 /* ta, mu */ :: (load unknown-size from %ir.p, align 8)
- ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store (<vscale x 1 x s64>) into %ir.p, vscale )
+ ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store (<vscale x 1 x s64>) into %ir.p)
; CHECK-NEXT: PseudoRET
%splat = insertelement <vscale x 2 x i1> poison, i1 -1, i32 0
%mask = shufflevector <vscale x 2 x i1> %splat, <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
diff --git a/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp b/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp
index f711f85e1248f2..1f2b8c1754f6ef 100644
--- a/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp
+++ b/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp
@@ -114,7 +114,8 @@ TEST_F(SelectionDAGAddressAnalysisTest, sameFrameObject) {
bool IsAlias;
bool IsValid = BaseIndexOffset::computeAliasing(
- Store.getNode(), NumBytes, Store.getNode(), NumBytes, *DAG, IsAlias);
+ Store.getNode(), LocationSize::precise(NumBytes), Store.getNode(),
+ LocationSize::precise(NumBytes), *DAG, IsAlias);
EXPECT_TRUE(IsValid);
EXPECT_TRUE(IsAlias);
@@ -133,14 +134,10 @@ TEST_F(SelectionDAGAddressAnalysisTest, sameFrameObjectUnknownSize) {
SDValue Store = DAG->getStore(DAG->getEntryNode(), Loc, Value, Index,
PtrInfo.getWithOffset(Offset));
- // Maybe unlikely that BaseIndexOffset::computeAliasing is used with the
- // optional NumBytes being unset like in this test, but it would be confusing
- // if that function determined IsAlias=false here.
- std::optional<TypeSize> NumBytes;
-
bool IsAlias;
bool IsValid = BaseIndexOffset::computeAliasing(
- Store.getNode(), NumBytes, Store.getNode(), NumBytes, *DAG, IsAlias);
+ Store.getNode(), LocationSize::beforeOrAfterPointer(), Store.getNode(),
+ LocationSize::beforeOrAfterPointer(), *DAG, IsAlias);
EXPECT_FALSE(IsValid);
}
@@ -169,7 +166,8 @@ TEST_F(SelectionDAGAddressAnalysisTest, noAliasingFrameObjects) {
bool IsAlias;
bool IsValid = BaseIndexOffset::computeAliasing(
- Store0.getNode(), NumBytes0, Store1.getNode(), NumBytes1, *DAG, IsAlias);
+ Store0.getNode(), LocationSize::precise(NumBytes0), Store1.getNode(),
+ LocationSize::precise(NumBytes1), *DAG, IsAlias);
EXPECT_TRUE(IsValid);
EXPECT_FALSE(IsAlias);
@@ -197,7 +195,8 @@ TEST_F(SelectionDAGAddressAnalysisTest, unknownSizeFrameObjects) {
bool IsAlias;
bool IsValid = BaseIndexOffset::computeAliasing(
- Store0.getNode(), NumBytes0, Store1.getNode(), NumBytes1, *DAG, IsAlias);
+ Store0.getNode(), LocationSize::precise(NumBytes0), Store1.getNode(),
+ LocationSize::precise(NumBytes1), *DAG, IsAlias);
EXPECT_FALSE(IsValid);
}
@@ -226,7 +225,8 @@ TEST_F(SelectionDAGAddressAnalysisTest, globalWithFrameObject) {
bool IsAlias;
bool IsValid = BaseIndexOffset::computeAliasing(
- Store.getNode(), NumBytes, GStore.getNode(), GNumBytes, *DAG, IsAlias);
+ Store.getNode(), LocationSize::precise(NumBytes), GStore.getNode(),
+ LocationSize::precise(GNumBytes), *DAG, IsAlias);
EXPECT_TRUE(IsValid);
EXPECT_FALSE(IsAlias);
@@ -250,9 +250,9 @@ TEST_F(SelectionDAGAddressAnalysisTest, globalWithAliasedGlobal) {
MachinePointerInfo(AliasedG, 0));
bool IsAlias;
- bool IsValid = BaseIndexOffset::computeAliasing(GStore.getNode(), GNumBytes,
- AliasedGStore.getNode(),
- GNumBytes, *DAG, IsAlias);
+ bool IsValid = BaseIndexOffset::computeAliasing(
+ GStore.getNode(), LocationSize::precise(GNumBytes),
+ AliasedGStore.getNode(), LocationSize::precise(GNumBytes), *DAG, IsAlias);
// With some deeper analysis we could detect if G and AliasedG is aliasing or
// not. But computeAliasing is currently defensive and assumes that a
@@ -287,12 +287,14 @@ TEST_F(SelectionDAGAddressAnalysisTest, fixedSizeFrameObjectsWithinDiff) {
bool IsAlias;
bool IsValid = BaseIndexOffset::computeAliasing(
- Store0.getNode(), NumBytes0, Store1.getNode(), NumBytes1, *DAG, IsAlias);
+ Store0.getNode(), LocationSize::precise(NumBytes0), Store1.getNode(),
+ LocationSize::precise(NumBytes1), *DAG, IsAlias);
EXPECT_TRUE(IsValid);
EXPECT_FALSE(IsAlias);
IsValid = BaseIndexOffset::computeAliasing(
- Store1.getNode(), NumBytes1, Store0.getNode(), NumBytes0, *DAG, IsAlias);
+ Store1.getNode(), LocationSize::precise(NumBytes1), Store0.getNode(),
+ LocationSize::precise(NumBytes0), *DAG, IsAlias);
EXPECT_TRUE(IsValid);
EXPECT_FALSE(IsAlias);
}
@@ -326,7 +328,8 @@ TEST_F(SelectionDAGAddressAnalysisTest, fixedSizeFrameObjectsOutOfDiff) {
bool IsAlias;
bool IsValid = BaseIndexOffset::computeAliasing(
- Store0.getNode(), NumBytes0, Store1.getNode(), NumBytes1, *DAG, IsAlias);
+ Store0.getNode(), LocationSize::precise(NumBytes0), Store1.getNode(),
+ LocationSize::precise(NumBytes1), *DAG, IsAlias);
EXPECT_TRUE(IsValid);
EXPECT_TRUE(IsAlias);
}
@@ -358,7 +361,8 @@ TEST_F(SelectionDAGAddressAnalysisTest, twoFixedStackObjects) {
bool IsAlias;
bool IsValid = BaseIndexOffset::computeAliasing(
- Store0.getNode(), NumBytes0, Store1.getNode(), NumBytes1, *DAG, IsAlias);
+ Store0.getNode(), LocationSize::precise(NumBytes0), Store1.getNode(),
+ LocationSize::precise(NumBytes1), *DAG, IsAlias);
EXPECT_TRUE(IsValid);
EXPECT_FALSE(IsAlias);
}
diff --git a/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp b/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp
index b4c96a9c2a62ce..9196a21dd0de19 100644
--- a/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp
+++ b/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp
@@ -98,7 +98,7 @@ TEST_P(RISCVInstrInfoTest, GetMemOperandsWithOffsetWidth) {
DebugLoc DL;
SmallVector<const MachineOperand *> BaseOps;
- unsigned Width;
+ LocationSize Width = 0;
int64_t Offset;
bool OffsetIsScalable;
More information about the llvm-commits
mailing list