[llvm] [CodeGen] Update for scalable MemoryType in MMO (PR #70452)
David Green via llvm-commits
llvm-commits at lists.llvm.org
Fri Mar 8 06:04:23 PST 2024
https://github.com/davemgreen updated https://github.com/llvm/llvm-project/pull/70452
>From 6f1afaa836d24c0df697372d953a5129c101e819 Mon Sep 17 00:00:00 2001
From: Harvin Iriawan <harvin.iriawan at arm.com>
Date: Wed, 25 Oct 2023 09:27:03 +0000
Subject: [PATCH 1/4] [SelectionDAG] Update for scalable MemoryType in MMO
Remove getSizeOrUnknown call when MachineMemOperand is created. For Scalable
TypeSize, the MemoryType created becomes a scalable_vector.
2 MMOs that have scalable memory access can then use the updated BasicAA that
understands scalable LocationSize
---
llvm/include/llvm/Analysis/MemoryLocation.h | 9 ++-
llvm/include/llvm/CodeGen/MachineFunction.h | 21 ++++++
llvm/include/llvm/CodeGen/MachineMemOperand.h | 16 +++-
llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp | 39 ++++++----
llvm/lib/CodeGen/MachineFunction.cpp | 20 +++++
llvm/lib/CodeGen/MachineInstr.cpp | 43 +++++++----
llvm/lib/CodeGen/MachineOperand.cpp | 24 +++++-
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 75 ++++++++++++-------
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 19 ++---
llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll | 6 +-
.../RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll | 8 +-
11 files changed, 199 insertions(+), 81 deletions(-)
diff --git a/llvm/include/llvm/Analysis/MemoryLocation.h b/llvm/include/llvm/Analysis/MemoryLocation.h
index b72a27cab86b34..bea96827a7c9e3 100644
--- a/llvm/include/llvm/Analysis/MemoryLocation.h
+++ b/llvm/include/llvm/Analysis/MemoryLocation.h
@@ -291,8 +291,9 @@ class MemoryLocation {
return MemoryLocation(Ptr, LocationSize::beforeOrAfterPointer(), AATags);
}
- // Return the exact size if the exact size is known at compiletime,
- // otherwise return MemoryLocation::UnknownSize.
+ // TODO: Remove getSizeOrUnknown
+ // interface once llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest is
+ // updated
static uint64_t getSizeOrUnknown(const TypeSize &T) {
return T.isScalable() ? UnknownSize : T.getFixedValue();
}
@@ -303,6 +304,10 @@ class MemoryLocation {
const AAMDNodes &AATags = AAMDNodes())
: Ptr(Ptr), Size(Size), AATags(AATags) {}
+ explicit MemoryLocation(const Value *Ptr, TypeSize Size,
+ const AAMDNodes &AATags = AAMDNodes())
+ : Ptr(Ptr), Size(LocationSize::precise(Size)), AATags(AATags) {}
+
MemoryLocation getWithNewPtr(const Value *NewPtr) const {
MemoryLocation Copy(*this);
Copy.Ptr = NewPtr;
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index a2c90c9f42f38f..657dce3297a782 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -1032,6 +1032,13 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
+ MachineMemOperand *getMachineMemOperand(
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, TypeSize ts,
+ Align base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
+ const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
+ AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
+ AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
+
MachineMemOperand *getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy,
Align base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
@@ -1051,6 +1058,17 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
MMO, Offset, Size == ~UINT64_C(0) ? LLT() : LLT::scalar(8 * Size));
}
+ MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
+ int64_t Offset, TypeSize ts) {
+ return getMachineMemOperand(
+ MMO, Offset,
+ ts.getKnownMinValue() == ~UINT64_C(0)
+ ? LLT()
+ : ts.isScalable()
+ ? LLT::scalable_vector(1, 8 * ts.getKnownMinValue())
+ : LLT::scalar(8 * ts.getKnownMinValue()));
+ }
+
/// getMachineMemOperand - Allocate a new MachineMemOperand by copying
/// an existing one, replacing only the MachinePointerInfo and size.
/// MachineMemOperands are owned by the MachineFunction and need not be
@@ -1058,6 +1076,9 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
uint64_t Size);
+ MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
+ const MachinePointerInfo &PtrInfo,
+ TypeSize ts);
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
LLT Ty);
diff --git a/llvm/include/llvm/CodeGen/MachineMemOperand.h b/llvm/include/llvm/CodeGen/MachineMemOperand.h
index 12c18aaea5b26c..65593ad0480318 100644
--- a/llvm/include/llvm/CodeGen/MachineMemOperand.h
+++ b/llvm/include/llvm/CodeGen/MachineMemOperand.h
@@ -192,6 +192,12 @@ class MachineMemOperand {
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
+ MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, TypeSize ts,
+ Align a, const AAMDNodes &AAInfo = AAMDNodes(),
+ const MDNode *Ranges = nullptr,
+ SyncScope::ID SSID = SyncScope::System,
+ AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
+ AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, LLT type, Align a,
const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr,
@@ -235,13 +241,15 @@ class MachineMemOperand {
LLT getMemoryType() const { return MemoryType; }
/// Return the size in bytes of the memory reference.
- uint64_t getSize() const {
- return MemoryType.isValid() ? MemoryType.getSizeInBytes() : ~UINT64_C(0);
+ TypeSize getSize() const {
+ return MemoryType.isValid() ? MemoryType.getSizeInBytes()
+ : TypeSize::getFixed(~UINT64_C(0));
}
/// Return the size in bits of the memory reference.
- uint64_t getSizeInBits() const {
- return MemoryType.isValid() ? MemoryType.getSizeInBits() : ~UINT64_C(0);
+ TypeSize getSizeInBits() const {
+ return MemoryType.isValid() ? MemoryType.getSizeInBits()
+ : TypeSize::getFixed(~UINT64_C(0));
}
LLT getType() const {
diff --git a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
index b5c9d3e912cc20..e3b940298754a6 100644
--- a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
@@ -215,13 +215,8 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
}
TypeSize Size = LS->getMMO().getMemoryType().getSizeInBytes();
- return {LS->isVolatile(),
- LS->isAtomic(),
- BaseReg,
- Offset /*base offset*/,
- Size.isScalable() ? LocationSize::beforeOrAfterPointer()
- : LocationSize::precise(Size),
- &LS->getMMO()};
+ return {LS->isVolatile(), LS->isAtomic(), BaseReg,
+ Offset /*base offset*/, LocationSize::precise(Size), &LS->getMMO()};
}
// FIXME: support recognizing lifetime instructions.
// Default.
@@ -257,10 +252,20 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
return false;
}
+ // if NumBytes is scalable and offset is not 0, conservatively return may
+ // alias
+ if ((MUC0.NumBytes.isScalable() && (MUC0.Offset != 0)) ||
+ (MUC1.NumBytes.isScalable() && (MUC1.Offset != 0)))
+ return true;
+
+ const bool BothNotScalable =
+ !(MUC0.NumBytes.isScalable() || MUC1.NumBytes.isScalable());
+
// Try to prove that there is aliasing, or that there is no aliasing. Either
// way, we can return now. If nothing can be proved, proceed with more tests.
bool IsAlias;
- if (GISelAddressing::aliasIsKnownForLoadStore(MI, Other, IsAlias, MRI))
+ if (BothNotScalable &&
+ GISelAddressing::aliasIsKnownForLoadStore(MI, Other, IsAlias, MRI))
return IsAlias;
// The following all rely on MMO0 and MMO1 being valid.
@@ -276,12 +281,18 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
Size1.hasValue()) {
// Use alias analysis information.
int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1);
- int64_t Overlap0 = Size0.getValue() + SrcValOffset0 - MinOffset;
- int64_t Overlap1 = Size1.getValue() + SrcValOffset1 - MinOffset;
- if (AA->isNoAlias(MemoryLocation(MUC0.MMO->getValue(), Overlap0,
- MUC0.MMO->getAAInfo()),
- MemoryLocation(MUC1.MMO->getValue(), Overlap1,
- MUC1.MMO->getAAInfo())))
+ int64_t Overlap0 =
+ Size0.getValue().getKnownMinValue() + SrcValOffset0 - MinOffset;
+ int64_t Overlap1 =
+ Size1.getValue().getKnownMinValue() + SrcValOffset1 - MinOffset;
+ LocationSize Loc0 =
+ Size0.isScalable() ? Size0 : LocationSize::precise(Overlap0);
+ LocationSize Loc1 =
+ Size1.isScalable() ? Size1 : LocationSize::precise(Overlap1);
+
+ if (AA->isNoAlias(
+ MemoryLocation(MUC0.MMO->getValue(), Loc0, MUC0.MMO->getAAInfo()),
+ MemoryLocation(MUC1.MMO->getValue(), Loc1, MUC1.MMO->getAAInfo())))
return false;
}
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index 323f1a6a1b2bd7..ed9f6977e81061 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -493,6 +493,16 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
SSID, Ordering, FailureOrdering);
}
+MachineMemOperand *MachineFunction::getMachineMemOperand(
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, TypeSize ts,
+ Align base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
+ SyncScope::ID SSID, AtomicOrdering Ordering,
+ AtomicOrdering FailureOrdering) {
+ return new (Allocator)
+ MachineMemOperand(PtrInfo, f, ts, base_alignment, AAInfo, Ranges, SSID,
+ Ordering, FailureOrdering);
+}
+
MachineMemOperand *MachineFunction::getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy,
Align base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
@@ -511,6 +521,16 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
MMO->getSuccessOrdering(), MMO->getFailureOrdering());
}
+MachineMemOperand *
+MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
+ const MachinePointerInfo &PtrInfo,
+ TypeSize ts) {
+ return new (Allocator)
+ MachineMemOperand(PtrInfo, MMO->getFlags(), ts, MMO->getBaseAlign(),
+ AAMDNodes(), nullptr, MMO->getSyncScopeID(),
+ MMO->getSuccessOrdering(), MMO->getFailureOrdering());
+}
+
MachineMemOperand *MachineFunction::getMachineMemOperand(
const MachineMemOperand *MMO, const MachinePointerInfo &PtrInfo, LLT Ty) {
return new (Allocator)
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 6654e1d6ceceea..2fefa2245f2af1 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -1302,10 +1302,11 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
int64_t OffsetB = MMOb->getOffset();
int64_t MinOffset = std::min(OffsetA, OffsetB);
- uint64_t WidthA = MMOa->getSize();
- uint64_t WidthB = MMOb->getSize();
- bool KnownWidthA = WidthA != MemoryLocation::UnknownSize;
- bool KnownWidthB = WidthB != MemoryLocation::UnknownSize;
+ TypeSize WidthA = MMOa->getSize();
+ TypeSize WidthB = MMOb->getSize();
+ bool KnownWidthA = WidthA.getKnownMinValue() != MemoryLocation::UnknownSize;
+ bool KnownWidthB = WidthB.getKnownMinValue() != MemoryLocation::UnknownSize;
+ bool BothMMONonScalable = !WidthA.isScalable() && !WidthB.isScalable();
const Value *ValA = MMOa->getValue();
const Value *ValB = MMOb->getValue();
@@ -1321,11 +1322,12 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
SameVal = true;
}
- if (SameVal) {
+ if (SameVal && BothMMONonScalable) {
if (!KnownWidthA || !KnownWidthB)
return true;
int64_t MaxOffset = std::max(OffsetA, OffsetB);
- int64_t LowWidth = (MinOffset == OffsetA) ? WidthA : WidthB;
+ int64_t LowWidth = (MinOffset == OffsetA) ? WidthA.getKnownMinValue()
+ : WidthB.getKnownMinValue();
return (MinOffset + LowWidth > MaxOffset);
}
@@ -1338,15 +1340,30 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
assert((OffsetA >= 0) && "Negative MachineMemOperand offset");
assert((OffsetB >= 0) && "Negative MachineMemOperand offset");
- int64_t OverlapA =
- KnownWidthA ? WidthA + OffsetA - MinOffset : MemoryLocation::UnknownSize;
- int64_t OverlapB =
- KnownWidthB ? WidthB + OffsetB - MinOffset : MemoryLocation::UnknownSize;
+ // If Scalable Location Size has non-zero offset,
+ // Width + Offset does not work at the moment
+ if ((WidthA.isScalable() && (OffsetA > 0)) ||
+ (WidthB.isScalable() && (OffsetB > 0))) {
+ return true;
+ }
+
+ int64_t OverlapA = KnownWidthA
+ ? WidthA.getKnownMinValue() + OffsetA - MinOffset
+ : MemoryLocation::UnknownSize;
+ int64_t OverlapB = KnownWidthB
+ ? WidthB.getKnownMinValue() + OffsetB - MinOffset
+ : MemoryLocation::UnknownSize;
+
+ LocationSize LocA = WidthA.isScalable() && KnownWidthA
+ ? LocationSize::precise(WidthA)
+ : LocationSize(OverlapA);
+ LocationSize LocB = WidthB.isScalable() && KnownWidthB
+ ? LocationSize::precise(WidthB)
+ : LocationSize(OverlapB);
return !AA->isNoAlias(
- MemoryLocation(ValA, OverlapA, UseTBAA ? MMOa->getAAInfo() : AAMDNodes()),
- MemoryLocation(ValB, OverlapB,
- UseTBAA ? MMOb->getAAInfo() : AAMDNodes()));
+ MemoryLocation(ValA, LocA, UseTBAA ? MMOa->getAAInfo() : AAMDNodes()),
+ MemoryLocation(ValB, LocB, UseTBAA ? MMOb->getAAInfo() : AAMDNodes()));
}
bool MachineInstr::mayAlias(AAResults *AA, const MachineInstr &Other,
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index c7c0a1c20d57f4..cb3051e4c99df4 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -1111,14 +1111,29 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
s == ~UINT64_C(0) ? LLT() : LLT::scalar(8 * s), a,
AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
+MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
+ TypeSize ts, Align a,
+ const AAMDNodes &AAInfo,
+ const MDNode *Ranges, SyncScope::ID SSID,
+ AtomicOrdering Ordering,
+ AtomicOrdering FailureOrdering)
+ : MachineMemOperand(
+ ptrinfo, f,
+ ts.getKnownMinValue() == ~UINT64_C(0)
+ ? LLT()
+ : ts.isScalable()
+ ? LLT::scalable_vector(1, 8 * ts.getKnownMinValue())
+ : LLT::scalar(8 * ts.getKnownMinValue()),
+ a, AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
+
void MachineMemOperand::refineAlignment(const MachineMemOperand *MMO) {
// The Value and Offset may differ due to CSE. But the flags and size
// should be the same.
assert(MMO->getFlags() == getFlags() && "Flags mismatch!");
- assert((MMO->getSize() == ~UINT64_C(0) || getSize() == ~UINT64_C(0) ||
+ assert((MMO->getSize().getKnownMinValue() == ~UINT64_C(0) ||
+ getSize().getKnownMinValue() == ~UINT64_C(0) ||
MMO->getSize() == getSize()) &&
"Size mismatch!");
-
if (MMO->getBaseAlign() >= getBaseAlign()) {
// Update the alignment value.
BaseAlign = MMO->getBaseAlign();
@@ -1240,7 +1255,10 @@ void MachineMemOperand::print(raw_ostream &OS, ModuleSlotTracker &MST,
<< "unknown-address";
}
MachineOperand::printOperandOffset(OS, getOffset());
- if (getSize() > 0 && getAlign() != getSize())
+ if (getSize().isScalable())
+ OS << ", vscale ";
+ if (getSize().getKnownMinValue() > 0 &&
+ getAlign() != getSize().getKnownMinValue())
OS << ", align " << getAlign().value();
if (getAlign() != getBaseAlign())
OS << ", basealign " << getBaseAlign().value();
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 5476ef87971436..e92c1cedc6b246 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -24222,7 +24222,7 @@ static SDValue narrowExtractedVectorLoad(SDNode *Extract, SelectionDAG &DAG) {
// TODO: Use "BaseIndexOffset" to make this more effective.
SDValue NewAddr = DAG.getMemBasePlusOffset(Ld->getBasePtr(), Offset, DL);
- uint64_t StoreSize = MemoryLocation::getSizeOrUnknown(VT.getStoreSize());
+ TypeSize StoreSize = VT.getStoreSize();
MachineFunction &MF = DAG.getMachineFunction();
MachineMemOperand *MMO;
if (Offset.isScalable()) {
@@ -27863,18 +27863,17 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
if (const auto *LSN = dyn_cast<LSBaseSDNode>(N)) {
int64_t Offset = 0;
if (auto *C = dyn_cast<ConstantSDNode>(LSN->getOffset()))
- Offset = (LSN->getAddressingMode() == ISD::PRE_INC) ? C->getSExtValue()
- : (LSN->getAddressingMode() == ISD::PRE_DEC)
- ? -1 * C->getSExtValue()
- : 0;
- uint64_t Size =
- MemoryLocation::getSizeOrUnknown(LSN->getMemoryVT().getStoreSize());
+ Offset = (LSN->getAddressingMode() == ISD::PRE_INC)
+ ? C->getSExtValue()
+ : (LSN->getAddressingMode() == ISD::PRE_DEC)
+ ? -1 * C->getSExtValue()
+ : 0;
+ TypeSize Size = LSN->getMemoryVT().getStoreSize();
return {LSN->isVolatile(),
LSN->isAtomic(),
LSN->getBasePtr(),
Offset /*base offset*/,
- Size != ~UINT64_C(0) ? LocationSize::precise(Size)
- : LocationSize::beforeOrAfterPointer(),
+ LocationSize::precise(Size),
LSN->getMemOperand()};
}
if (const auto *LN = cast<LifetimeSDNode>(N))
@@ -27917,10 +27916,20 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
return false;
}
+ // if NumBytes is scalable and offset is not 0, conservatively return may
+ // alias
+ if ((MUC0.NumBytes.hasValue() && MUC0.NumBytes.isScalable() &&
+ (MUC0.Offset != 0)) ||
+ (MUC1.NumBytes.hasValue() && MUC1.NumBytes.isScalable() &&
+ (MUC1.Offset != 0)))
+ return true;
// Try to prove that there is aliasing, or that there is no aliasing. Either
// way, we can return now. If nothing can be proved, proceed with more tests.
+ const bool BothNotScalable = !(MUC0.NumBytes.isScalable() ||
+ MUC1.NumBytes.isScalable());
bool IsAlias;
- if (BaseIndexOffset::computeAliasing(Op0, MUC0.NumBytes, Op1, MUC1.NumBytes,
+ if (BothNotScalable &&
+ BaseIndexOffset::computeAliasing(Op0, MUC0.NumBytes, Op1, MUC1.NumBytes,
DAG, IsAlias))
return IsAlias;
@@ -27947,19 +27956,23 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
Align OrigAlignment1 = MUC1.MMO->getBaseAlign();
LocationSize Size0 = MUC0.NumBytes;
LocationSize Size1 = MUC1.NumBytes;
- if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 &&
- Size0.hasValue() && Size1.hasValue() && Size0 == Size1 &&
- OrigAlignment0 > Size0.getValue() &&
- SrcValOffset0 % Size0.getValue() == 0 &&
- SrcValOffset1 % Size1.getValue() == 0) {
- int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.value();
- int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.value();
-
- // There is no overlap between these relatively aligned accesses of
- // similar size. Return no alias.
- if ((OffAlign0 + (int64_t)Size0.getValue()) <= OffAlign1 ||
- (OffAlign1 + (int64_t)Size1.getValue()) <= OffAlign0)
- return false;
+
+ if (BothNotScalable) {
+ if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 &&
+ Size0.hasValue() && Size1.hasValue() && !Size0.isScalable() &&
+ !Size1.isScalable() && Size0.getValue() == Size1.getValue() &&
+ OrigAlignment0 > Size0.getValue() &&
+ SrcValOffset0 % Size0.getValue() == 0 &&
+ SrcValOffset1 % Size1.getValue() == 0) {
+ int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.value();
+ int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.value();
+
+ // There is no overlap between these relatively aligned accesses of
+ // similar size. Return no alias.
+ if ((OffAlign0 + Size0.getValue()) <= OffAlign1 ||
+ (OffAlign1 + Size1.getValue()) <= OffAlign0)
+ return false;
+ }
}
bool UseAA = CombinerGlobalAA.getNumOccurrences() > 0
@@ -27975,12 +27988,20 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
Size0.hasValue() && Size1.hasValue()) {
// Use alias analysis information.
int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1);
- int64_t Overlap0 = Size0.getValue() + SrcValOffset0 - MinOffset;
- int64_t Overlap1 = Size1.getValue() + SrcValOffset1 - MinOffset;
+ int64_t Overlap0 =
+ Size0.getValue().getKnownMinValue() + SrcValOffset0 - MinOffset;
+ int64_t Overlap1 =
+ Size1.getValue().getKnownMinValue() + SrcValOffset1 - MinOffset;
+ LocationSize Loc0 = Size0.getValue().isScalable()
+ ? LocationSize::precise(Size0.getValue())
+ : LocationSize::precise(Overlap0);
+ LocationSize Loc1 = Size1.getValue().isScalable()
+ ? LocationSize::precise(Size1.getValue())
+ : LocationSize::precise(Overlap1);
if (AA->isNoAlias(
- MemoryLocation(MUC0.MMO->getValue(), Overlap0,
+ MemoryLocation(MUC0.MMO->getValue(), Loc0,
UseTBAA ? MUC0.MMO->getAAInfo() : AAMDNodes()),
- MemoryLocation(MUC1.MMO->getValue(), Overlap1,
+ MemoryLocation(MUC1.MMO->getValue(), Loc1,
UseTBAA ? MUC1.MMO->getAAInfo() : AAMDNodes())))
return false;
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index f7ace79e8c51d4..d0eebd3cac89c2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -8556,7 +8556,7 @@ SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
if (PtrInfo.V.isNull())
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
- uint64_t Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize());
+ TypeSize Size = MemVT.getStoreSize();
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
Alignment, AAInfo, Ranges);
@@ -8677,8 +8677,7 @@ SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
MachineFunction &MF = getMachineFunction();
- uint64_t Size =
- MemoryLocation::getSizeOrUnknown(Val.getValueType().getStoreSize());
+ TypeSize Size = Val.getValueType().getStoreSize();
MachineMemOperand *MMO =
MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo);
return getStore(Chain, dl, Val, Ptr, MMO);
@@ -8731,8 +8730,7 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
- PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()),
- Alignment, AAInfo);
+ PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment, AAInfo);
return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
}
@@ -8826,7 +8824,7 @@ SDValue SelectionDAG::getLoadVP(
if (PtrInfo.V.isNull())
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
- uint64_t Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize());
+ TypeSize Size = MemVT.getStoreSize();
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
Alignment, AAInfo, Ranges);
@@ -8979,8 +8977,7 @@ SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
- PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()),
- Alignment, AAInfo);
+ PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment, AAInfo);
return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO,
IsCompressing);
}
@@ -11715,9 +11712,9 @@ MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
// We check here that the size of the memory operand fits within the size of
// the MMO. This is because the MMO might indicate only a possible address
// range instead of specifying the affected memory addresses precisely.
- // TODO: Make MachineMemOperands aware of scalable vectors.
- assert(memvt.getStoreSize().getKnownMinValue() <= MMO->getSize() &&
- "Size mismatch!");
+ if (MMO->getType().isValid())
+ assert(TypeSize::isKnownLE(memvt.getStoreSize(), MMO->getSize()) &&
+ "Size mismatch!");
}
/// Profile - Gather unique data for the node.
diff --git a/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll b/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
index 58299696e78fc2..39c546e59afbf5 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
@@ -3,7 +3,7 @@
define void @UphPNR(target("aarch64.svcount") %predcnt) {
entry:
; CHECK: %0:ppr = COPY $p0
-; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2)
+; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr, vscale )
; CHECK: %1:pnr_p8to15 = COPY %0
; CHECK: INLINEASM &"ld1w {z0.s,z1.s,z2.s,z3.s}, $0/z, [x10]", 1 /* sideeffect attdialect */, {{[0-9]+}} /* reguse:PNR_p8to15 */, %1
; CHECK: RET_ReallyLR
@@ -17,7 +17,7 @@ entry:
define void @UpaPNR(target("aarch64.svcount") %predcnt) {
entry:
; CHECK: %0:ppr = COPY $p0
-; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2)
+; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr, vscale )
; CHECK: %1:pnr = COPY %0
; CHECK: INLINEASM &"ld1w {z0.s,z1.s,z2.s,z3.s}, $0/z, [x10]", 1 /* sideeffect attdialect */, {{[0-9]+}} /* reguse:PNR */, %1
; CHECK: RET_ReallyLR
@@ -31,7 +31,7 @@ entry:
define void @UplPNR(target("aarch64.svcount") %predcnt) {
entry:
; CHECK: %0:ppr = COPY $p0
-; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2)
+; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr, vscale )
; CHECK: %1:pnr_3b = COPY %0
; CHECK: INLINEASM &"fadd z0.h, $0/m, z0.h, #0.5", 1 /* sideeffect attdialect */, {{[0-9]+}} /* reguse:PNR_3b */, %1
; CHECK: RET_ReallyLR
diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
index 0544204cce792a..05814a0b3f5e70 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
@@ -16,8 +16,8 @@ define void @vpmerge_vpload_store(<vscale x 2 x i32> %passthru, ptr %p, <vscale
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr = COPY $x10
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vrnov0 = COPY $v8
; CHECK-NEXT: $v0 = COPY [[COPY1]]
- ; CHECK-NEXT: [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], $v0, [[COPY]], 5 /* e32 */, 0 /* tu, mu */
- ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store unknown-size into %ir.p, align 8)
+ ; CHECK-NEXT: [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], $v0, [[COPY]], 5 /* e32 */, 0 /* tu, mu */ :: (load unknown-size from %ir.p, align 8)
+ ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store (<vscale x 1 x s64>) into %ir.p, vscale )
; CHECK-NEXT: PseudoRET
%splat = insertelement <vscale x 2 x i1> poison, i1 -1, i32 0
%mask = shufflevector <vscale x 2 x i1> %splat, <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
@@ -37,8 +37,8 @@ define void @vpselect_vpload_store(<vscale x 2 x i32> %passthru, ptr %p, <vscale
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr = COPY $x10
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vrnov0 = COPY $v8
; CHECK-NEXT: $v0 = COPY [[COPY1]]
- ; CHECK-NEXT: [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], $v0, [[COPY]], 5 /* e32 */, 1 /* ta, mu */
- ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store unknown-size into %ir.p, align 8)
+ ; CHECK-NEXT: [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], $v0, [[COPY]], 5 /* e32 */, 1 /* ta, mu */ :: (load unknown-size from %ir.p, align 8)
+ ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store (<vscale x 1 x s64>) into %ir.p, vscale )
; CHECK-NEXT: PseudoRET
%splat = insertelement <vscale x 2 x i1> poison, i1 -1, i32 0
%mask = shufflevector <vscale x 2 x i1> %splat, <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
>From bfb675341dd089f52809e046d50edc2bcb6df848 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Tue, 21 Nov 2023 08:43:31 +0000
Subject: [PATCH 2/4] Cleanup
---
llvm/include/llvm/Analysis/MemoryLocation.h | 11 -------
llvm/include/llvm/CodeGen/MachineFunction.h | 16 +++++-----
llvm/include/llvm/CodeGen/MachineMemOperand.h | 2 +-
llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp | 2 +-
llvm/lib/CodeGen/MachineFunction.cpp | 10 +++----
llvm/lib/CodeGen/MachineInstr.cpp | 5 ++--
llvm/lib/CodeGen/MachineOperand.cpp | 16 +++++-----
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 29 +++++++------------
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 6 ++--
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 2 +-
.../Target/Hexagon/HexagonStoreWidening.cpp | 6 ++--
11 files changed, 42 insertions(+), 63 deletions(-)
diff --git a/llvm/include/llvm/Analysis/MemoryLocation.h b/llvm/include/llvm/Analysis/MemoryLocation.h
index bea96827a7c9e3..4680b6854abdbe 100644
--- a/llvm/include/llvm/Analysis/MemoryLocation.h
+++ b/llvm/include/llvm/Analysis/MemoryLocation.h
@@ -291,23 +291,12 @@ class MemoryLocation {
return MemoryLocation(Ptr, LocationSize::beforeOrAfterPointer(), AATags);
}
- // TODO: Remove getSizeOrUnknown
- // interface once llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest is
- // updated
- static uint64_t getSizeOrUnknown(const TypeSize &T) {
- return T.isScalable() ? UnknownSize : T.getFixedValue();
- }
-
MemoryLocation() : Ptr(nullptr), Size(LocationSize::beforeOrAfterPointer()) {}
explicit MemoryLocation(const Value *Ptr, LocationSize Size,
const AAMDNodes &AATags = AAMDNodes())
: Ptr(Ptr), Size(Size), AATags(AATags) {}
- explicit MemoryLocation(const Value *Ptr, TypeSize Size,
- const AAMDNodes &AATags = AAMDNodes())
- : Ptr(Ptr), Size(LocationSize::precise(Size)), AATags(AATags) {}
-
MemoryLocation getWithNewPtr(const Value *NewPtr) const {
MemoryLocation Copy(*this);
Copy.Ptr = NewPtr;
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index 657dce3297a782..bc93bef12dbc54 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -1033,8 +1033,8 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
MachineMemOperand *getMachineMemOperand(
- MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, TypeSize ts,
- Align base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, TypeSize TS,
+ Align BaseAlignment, const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
@@ -1059,14 +1059,12 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
}
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
- int64_t Offset, TypeSize ts) {
+ int64_t Offset, TypeSize TS) {
return getMachineMemOperand(
MMO, Offset,
- ts.getKnownMinValue() == ~UINT64_C(0)
- ? LLT()
- : ts.isScalable()
- ? LLT::scalable_vector(1, 8 * ts.getKnownMinValue())
- : LLT::scalar(8 * ts.getKnownMinValue()));
+ TS.getKnownMinValue() == ~UINT64_C(0) ? LLT()
+ : TS.isScalable() ? LLT::scalable_vector(1, 8 * TS.getKnownMinValue())
+ : LLT::scalar(8 * TS.getKnownMinValue()));
}
/// getMachineMemOperand - Allocate a new MachineMemOperand by copying
@@ -1078,7 +1076,7 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
uint64_t Size);
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
- TypeSize ts);
+ TypeSize TS);
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
LLT Ty);
diff --git a/llvm/include/llvm/CodeGen/MachineMemOperand.h b/llvm/include/llvm/CodeGen/MachineMemOperand.h
index 65593ad0480318..a4f3792309399b 100644
--- a/llvm/include/llvm/CodeGen/MachineMemOperand.h
+++ b/llvm/include/llvm/CodeGen/MachineMemOperand.h
@@ -192,7 +192,7 @@ class MachineMemOperand {
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
- MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, TypeSize ts,
+ MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, TypeSize TS,
Align a, const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr,
SyncScope::ID SSID = SyncScope::System,
diff --git a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
index e3b940298754a6..7756f6ed0d53e9 100644
--- a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
@@ -252,7 +252,7 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
return false;
}
- // if NumBytes is scalable and offset is not 0, conservatively return may
+ // If NumBytes is scalable and offset is not 0, conservatively return may
// alias
if ((MUC0.NumBytes.isScalable() && (MUC0.Offset != 0)) ||
(MUC1.NumBytes.isScalable() && (MUC1.Offset != 0)))
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index ed9f6977e81061..06f89badb65b96 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -494,12 +494,12 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
}
MachineMemOperand *MachineFunction::getMachineMemOperand(
- MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, TypeSize ts,
- Align base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, TypeSize TS,
+ Align BaseAlignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
SyncScope::ID SSID, AtomicOrdering Ordering,
AtomicOrdering FailureOrdering) {
return new (Allocator)
- MachineMemOperand(PtrInfo, f, ts, base_alignment, AAInfo, Ranges, SSID,
+ MachineMemOperand(PtrInfo, F, TS, BaseAlignment, AAInfo, Ranges, SSID,
Ordering, FailureOrdering);
}
@@ -524,9 +524,9 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
MachineMemOperand *
MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
- TypeSize ts) {
+ TypeSize TS) {
return new (Allocator)
- MachineMemOperand(PtrInfo, MMO->getFlags(), ts, MMO->getBaseAlign(),
+ MachineMemOperand(PtrInfo, MMO->getFlags(), TS, MMO->getBaseAlign(),
AAMDNodes(), nullptr, MMO->getSyncScopeID(),
MMO->getSuccessOrdering(), MMO->getFailureOrdering());
}
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 2fefa2245f2af1..6aebd71d92028a 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -1342,10 +1342,9 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
// If Scalable Location Size has non-zero offset,
// Width + Offset does not work at the moment
- if ((WidthA.isScalable() && (OffsetA > 0)) ||
- (WidthB.isScalable() && (OffsetB > 0))) {
+ if ((WidthA.isScalable() && OffsetA > 0) ||
+ (WidthB.isScalable() && OffsetB > 0))
return true;
- }
int64_t OverlapA = KnownWidthA
? WidthA.getKnownMinValue() + OffsetA - MinOffset
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index cb3051e4c99df4..2823760a94b385 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -1111,20 +1111,18 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
s == ~UINT64_C(0) ? LLT() : LLT::scalar(8 * s), a,
AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
-MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
- TypeSize ts, Align a,
+MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags F,
+ TypeSize TS, Align BaseAlignment,
const AAMDNodes &AAInfo,
const MDNode *Ranges, SyncScope::ID SSID,
AtomicOrdering Ordering,
AtomicOrdering FailureOrdering)
: MachineMemOperand(
- ptrinfo, f,
- ts.getKnownMinValue() == ~UINT64_C(0)
- ? LLT()
- : ts.isScalable()
- ? LLT::scalable_vector(1, 8 * ts.getKnownMinValue())
- : LLT::scalar(8 * ts.getKnownMinValue()),
- a, AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
+ ptrinfo, F,
+ TS.getKnownMinValue() == ~UINT64_C(0) ? LLT()
+ : TS.isScalable() ? LLT::scalable_vector(1, 8 * TS.getKnownMinValue())
+ : LLT::scalar(8 * TS.getKnownMinValue()),
+ BaseAlignment, AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
void MachineMemOperand::refineAlignment(const MachineMemOperand *MMO) {
// The Value and Offset may differ due to CSE. But the flags and size
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index e92c1cedc6b246..7ba974286c67c5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -27916,20 +27916,17 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
return false;
}
- // if NumBytes is scalable and offset is not 0, conservatively return may
+ // If NumBytes is scalable and offset is not 0, conservatively return may
// alias
if ((MUC0.NumBytes.hasValue() && MUC0.NumBytes.isScalable() &&
- (MUC0.Offset != 0)) ||
+ MUC0.Offset != 0) ||
(MUC1.NumBytes.hasValue() && MUC1.NumBytes.isScalable() &&
- (MUC1.Offset != 0)))
+ MUC1.Offset != 0))
return true;
// Try to prove that there is aliasing, or that there is no aliasing. Either
// way, we can return now. If nothing can be proved, proceed with more tests.
- const bool BothNotScalable = !(MUC0.NumBytes.isScalable() ||
- MUC1.NumBytes.isScalable());
bool IsAlias;
- if (BothNotScalable &&
- BaseIndexOffset::computeAliasing(Op0, MUC0.NumBytes, Op1, MUC1.NumBytes,
+ if (BaseIndexOffset::computeAliasing(Op0, MUC0.NumBytes, Op1, MUC1.NumBytes,
DAG, IsAlias))
return IsAlias;
@@ -27957,22 +27954,18 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
LocationSize Size0 = MUC0.NumBytes;
LocationSize Size1 = MUC1.NumBytes;
- if (BothNotScalable) {
- if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 &&
- Size0.hasValue() && Size1.hasValue() && !Size0.isScalable() &&
- !Size1.isScalable() && Size0.getValue() == Size1.getValue() &&
- OrigAlignment0 > Size0.getValue() &&
- SrcValOffset0 % Size0.getValue() == 0 &&
- SrcValOffset1 % Size1.getValue() == 0) {
- int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.value();
- int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.value();
+ if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 &&
+ Size0.hasValue() && Size1.hasValue() && !Size0.isScalable() &&
+ !Size1.isScalable() && Size0.getValue() == Size1.getValue() && OrigAlignment0 > Size0.getValue() &&
+ SrcValOffset0 % Size0.getValue() == 0 && SrcValOffset1 % Size1.getValue() == 0) {
+ int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.value();
+ int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.value();
// There is no overlap between these relatively aligned accesses of
// similar size. Return no alias.
if ((OffAlign0 + Size0.getValue()) <= OffAlign1 ||
(OffAlign1 + Size1.getValue()) <= OffAlign0)
return false;
- }
}
bool UseAA = CombinerGlobalAA.getNumOccurrences() > 0
@@ -27995,7 +27988,7 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
LocationSize Loc0 = Size0.getValue().isScalable()
? LocationSize::precise(Size0.getValue())
: LocationSize::precise(Overlap0);
- LocationSize Loc1 = Size1.getValue().isScalable()
+ LocationSize Loc1 = Size1.isScalable()
? LocationSize::precise(Size1.getValue())
: LocationSize::precise(Overlap1);
if (AA->isNoAlias(
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index d0eebd3cac89c2..641a596f36eaf3 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -11712,9 +11712,9 @@ MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
// We check here that the size of the memory operand fits within the size of
// the MMO. This is because the MMO might indicate only a possible address
// range instead of specifying the affected memory addresses precisely.
- if (MMO->getType().isValid())
- assert(TypeSize::isKnownLE(memvt.getStoreSize(), MMO->getSize()) &&
- "Size mismatch!");
+ assert((!MMO->getType().isValid() ||
+ TypeSize::isKnownLE(memvt.getStoreSize(), MMO->getSize())) &&
+ "Size mismatch!");
}
/// Profile - Gather unique data for the node.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index d1af74e6e0d352..0e8ff46c66b0fb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -3550,7 +3550,7 @@ bool AMDGPUDAGToDAGISel::isUniformLoad(const SDNode *N) const {
if (N->isDivergent() && !AMDGPUInstrInfo::isUniformMMO(MMO))
return false;
- return Ld->getAlign() >= Align(std::min(MMO->getSize(), uint64_t(4))) &&
+ return Ld->getAlign() >= Align(std::min(MMO->getSize().getKnownMinValue(), uint64_t(4))) &&
((Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) ||
(Subtarget->getScalarizeGlobalBehavior() &&
diff --git a/llvm/lib/Target/Hexagon/HexagonStoreWidening.cpp b/llvm/lib/Target/Hexagon/HexagonStoreWidening.cpp
index a647e699a8f07a..f3facbe511a2f7 100644
--- a/llvm/lib/Target/Hexagon/HexagonStoreWidening.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonStoreWidening.cpp
@@ -172,14 +172,16 @@ bool HexagonStoreWidening::instrAliased(InstrGroup &Stores,
if (!MMO.getValue())
return true;
- MemoryLocation L(MMO.getValue(), MMO.getSize(), MMO.getAAInfo());
+ MemoryLocation L(MMO.getValue(), MMO.getSize().getFixedValue(),
+ MMO.getAAInfo());
for (auto *SI : Stores) {
const MachineMemOperand &SMO = getStoreTarget(SI);
if (!SMO.getValue())
return true;
- MemoryLocation SL(SMO.getValue(), SMO.getSize(), SMO.getAAInfo());
+ MemoryLocation SL(SMO.getValue(), SMO.getSize().getFixedValue(),
+ SMO.getAAInfo());
if (!AA->isNoAlias(L, SL))
return true;
}
>From bb7072ce7590ce2d809fb5f247760d7d22c55858 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Thu, 23 Nov 2023 21:59:17 +0000
Subject: [PATCH 3/4] Attempt to use LocationSize
---
.../GlobalISel/GIMatchTableExecutorImpl.h | 6 +-
.../CodeGen/GlobalISel/GenericMachineInstrs.h | 4 +-
llvm/include/llvm/CodeGen/MachineFunction.h | 60 +++++++++++++------
llvm/include/llvm/CodeGen/MachineMemOperand.h | 23 ++++---
llvm/include/llvm/CodeGen/SelectionDAG.h | 4 +-
llvm/lib/CodeGen/DFAPacketizer.cpp | 7 ++-
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 12 ++--
.../lib/CodeGen/GlobalISel/GISelKnownBits.cpp | 7 ++-
.../CodeGen/GlobalISel/LegalizerHelper.cpp | 8 +--
llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp | 26 ++++----
.../LiveDebugValues/InstrRefBasedImpl.cpp | 2 +-
llvm/lib/CodeGen/MIRVRegNamerUtils.cpp | 2 +-
llvm/lib/CodeGen/MachineFunction.cpp | 22 +------
llvm/lib/CodeGen/MachineInstr.cpp | 58 +++++++++---------
llvm/lib/CodeGen/MachineOperand.cpp | 28 +++------
llvm/lib/CodeGen/MachinePipeliner.cpp | 13 ++--
llvm/lib/CodeGen/MachineStableHash.cpp | 2 +-
llvm/lib/CodeGen/MachineVerifier.cpp | 9 ++-
llvm/lib/CodeGen/ModuloSchedule.cpp | 4 +-
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 37 ++++++------
.../SelectionDAG/LegalizeVectorTypes.cpp | 45 +++++++-------
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 16 ++---
.../SelectionDAGAddressAnalysis.cpp | 2 -
.../SelectionDAG/SelectionDAGBuilder.cpp | 31 +++++-----
llvm/lib/CodeGen/TargetInstrInfo.cpp | 2 +-
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 5 +-
.../GISel/AArch64InstructionSelector.cpp | 8 +--
.../GISel/AArch64PostLegalizerCombiner.cpp | 4 +-
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 2 +-
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 2 +-
.../Target/AMDGPU/AMDGPURegisterBankInfo.cpp | 4 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 18 +++---
.../Target/AMDGPU/SILoadStoreOptimizer.cpp | 2 +-
llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp | 2 +-
llvm/lib/Target/ARM/ARMHazardRecognizer.cpp | 6 +-
llvm/lib/Target/BPF/BPFISelDAGToDAG.cpp | 4 +-
.../Target/Hexagon/HexagonStoreWidening.cpp | 16 +++--
.../Target/Mips/MipsInstructionSelector.cpp | 5 +-
llvm/lib/Target/Mips/MipsLegalizerInfo.cpp | 2 +-
.../Target/Mips/MipsPreLegalizerCombiner.cpp | 4 +-
llvm/lib/Target/Mips/MipsRegisterBankInfo.cpp | 3 +-
.../PowerPC/GISel/PPCInstructionSelector.cpp | 2 +-
.../Target/PowerPC/PPCHazardRecognizers.cpp | 6 +-
llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 6 +-
llvm/lib/Target/RISCV/RISCVInstrInfo.cpp | 4 +-
llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp | 7 ++-
.../WebAssemblySetP2AlignOperands.cpp | 2 +-
.../X86/X86AvoidStoreForwardingBlocks.cpp | 6 +-
llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll | 6 +-
.../RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll | 4 +-
50 files changed, 276 insertions(+), 284 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
index 8e456015cd3736..c73ac2c9f55b7b 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
@@ -647,15 +647,15 @@ bool GIMatchTableExecutor::executeMatchTable(
unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
- MMO->getSizeInBits() != Size) {
+ MMO->getSizeInBits().getValue() != Size) {
if (handleReject() == RejectAndGiveUp)
return false;
} else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
- MMO->getSizeInBits() >= Size) {
+ MMO->getSizeInBits().getValue() >= Size) {
if (handleReject() == RejectAndGiveUp)
return false;
} else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
- MMO->getSizeInBits() <= Size)
+ MMO->getSizeInBits().getValue() <= Size)
if (handleReject() == RejectAndGiveUp)
return false;
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
index f5a6528d10a973..1d2c45a12b268a 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
@@ -54,9 +54,9 @@ class GMemOperation : public GenericMachineInstr {
bool isUnordered() const { return getMMO().isUnordered(); }
/// Returns the size in bytes of the memory access.
- uint64_t getMemSize() const { return getMMO().getSize(); }
+ LocationSize getMemSize() const { return getMMO().getSize(); }
/// Returns the size in bits of the memory access.
- uint64_t getMemSizeInBits() const { return getMMO().getSizeInBits(); }
+ LocationSize getMemSizeInBits() const { return getMMO().getSizeInBits(); }
static bool classof(const MachineInstr *MI) {
return GenericMachineInstr::classof(MI) && MI->hasOneMemOperand();
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index bc93bef12dbc54..2f17b5d5ac71a8 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -1026,25 +1026,37 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
/// MachineMemOperands are owned by the MachineFunction and need not be
/// explicitly deallocated.
MachineMemOperand *getMachineMemOperand(
- MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s,
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy,
Align base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
-
MachineMemOperand *getMachineMemOperand(
- MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, TypeSize TS,
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LocationSize TS,
Align BaseAlignment, const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
-
MachineMemOperand *getMachineMemOperand(
- MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy,
- Align base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, uint64_t TS,
+ Align BaseAlignment, const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
- AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
+ AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic) {
+ return getMachineMemOperand(PtrInfo, F, LocationSize::precise(TS),
+ BaseAlignment, AAInfo, Ranges, SSID, Ordering,
+ FailureOrdering);
+ }
+ MachineMemOperand *getMachineMemOperand(
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, TypeSize TS,
+ Align BaseAlignment, const AAMDNodes &AAInfo = AAMDNodes(),
+ const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
+ AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
+ AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic) {
+ return getMachineMemOperand(PtrInfo, F, LocationSize::precise(TS),
+ BaseAlignment, AAInfo, Ranges, SSID, Ordering,
+ FailureOrdering);
+ }
/// getMachineMemOperand - Allocate a new MachineMemOperand by copying
/// an existing one, adjusting by an offset and using the given size.
@@ -1053,18 +1065,21 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
int64_t Offset, LLT Ty);
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
- int64_t Offset, uint64_t Size) {
+ int64_t Offset, LocationSize TS) {
return getMachineMemOperand(
- MMO, Offset, Size == ~UINT64_C(0) ? LLT() : LLT::scalar(8 * Size));
+ MMO, Offset,
+ !TS.hasValue() ? LLT()
+ : TS.isScalable()
+ ? LLT::scalable_vector(1, 8 * TS.getValue().getKnownMinValue())
+ : LLT::scalar(8 * TS.getValue().getKnownMinValue()));
}
-
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
- int64_t Offset, TypeSize TS) {
- return getMachineMemOperand(
- MMO, Offset,
- TS.getKnownMinValue() == ~UINT64_C(0) ? LLT()
- : TS.isScalable() ? LLT::scalable_vector(1, 8 * TS.getKnownMinValue())
- : LLT::scalar(8 * TS.getKnownMinValue()));
+ int64_t Offset, uint64_t Size) {
+ return getMachineMemOperand(MMO, Offset, LocationSize::precise(Size));
+ }
+ MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
+ int64_t Offset, TypeSize Size) {
+ return getMachineMemOperand(MMO, Offset, LocationSize::precise(Size));
}
/// getMachineMemOperand - Allocate a new MachineMemOperand by copying
@@ -1073,13 +1088,20 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
/// explicitly deallocated.
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
- uint64_t Size);
+ LocationSize TS);
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
- TypeSize TS);
+ LLT Ty);
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
- LLT Ty);
+ uint64_t Size) {
+ return getMachineMemOperand(MMO, PtrInfo, LocationSize::precise(Size));
+ }
+ MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
+ const MachinePointerInfo &PtrInfo,
+ TypeSize Size) {
+ return getMachineMemOperand(MMO, PtrInfo, LocationSize::precise(Size));
+ }
/// Allocate a new MachineMemOperand by copying an existing one,
/// replacing only AliasAnalysis information. MachineMemOperands are owned
diff --git a/llvm/include/llvm/CodeGen/MachineMemOperand.h b/llvm/include/llvm/CodeGen/MachineMemOperand.h
index a4f3792309399b..da4ca582cb9e4f 100644
--- a/llvm/include/llvm/CodeGen/MachineMemOperand.h
+++ b/llvm/include/llvm/CodeGen/MachineMemOperand.h
@@ -17,6 +17,7 @@
#include "llvm/ADT/BitmaskEnum.h"
#include "llvm/ADT/PointerUnion.h"
+#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/CodeGen/PseudoSourceValue.h"
#include "llvm/CodeGenTypes/LowLevelType.h"
#include "llvm/IR/DerivedTypes.h"
@@ -186,13 +187,7 @@ class MachineMemOperand {
/// and atomic ordering requirements must also be specified. For cmpxchg
/// atomic operations the atomic ordering requirements when store does not
/// occur must also be specified.
- MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, uint64_t s,
- Align a, const AAMDNodes &AAInfo = AAMDNodes(),
- const MDNode *Ranges = nullptr,
- SyncScope::ID SSID = SyncScope::System,
- AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
- AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
- MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, TypeSize TS,
+ MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, LocationSize TS,
Align a, const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr,
SyncScope::ID SSID = SyncScope::System,
@@ -241,15 +236,17 @@ class MachineMemOperand {
LLT getMemoryType() const { return MemoryType; }
/// Return the size in bytes of the memory reference.
- TypeSize getSize() const {
- return MemoryType.isValid() ? MemoryType.getSizeInBytes()
- : TypeSize::getFixed(~UINT64_C(0));
+ LocationSize getSize() const {
+ return MemoryType.isValid()
+ ? LocationSize::precise(MemoryType.getSizeInBytes())
+ : LocationSize::beforeOrAfterPointer();
}
/// Return the size in bits of the memory reference.
- TypeSize getSizeInBits() const {
- return MemoryType.isValid() ? MemoryType.getSizeInBits()
- : TypeSize::getFixed(~UINT64_C(0));
+ LocationSize getSizeInBits() const {
+ return MemoryType.isValid()
+ ? LocationSize::precise(MemoryType.getSizeInBits())
+ : LocationSize::beforeOrAfterPointer();
}
LLT getType() const {
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 25e6c525b672a1..4785e93d72d1cc 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -1299,7 +1299,7 @@ class SelectionDAG {
EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad |
MachineMemOperand::MOStore,
- uint64_t Size = 0, const AAMDNodes &AAInfo = AAMDNodes());
+ LocationSize Size = 0, const AAMDNodes &AAInfo = AAMDNodes());
inline SDValue getMemIntrinsicNode(
unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
@@ -1307,7 +1307,7 @@ class SelectionDAG {
MaybeAlign Alignment = std::nullopt,
MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad |
MachineMemOperand::MOStore,
- uint64_t Size = 0, const AAMDNodes &AAInfo = AAMDNodes()) {
+ LocationSize Size = 0, const AAMDNodes &AAInfo = AAMDNodes()) {
// Ensure that codegen never sees alignment 0
return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, PtrInfo,
Alignment.value_or(getEVTAlign(MemVT)), Flags,
diff --git a/llvm/lib/CodeGen/DFAPacketizer.cpp b/llvm/lib/CodeGen/DFAPacketizer.cpp
index 48bb4a07662e10..c16166a1d5e1c5 100644
--- a/llvm/lib/CodeGen/DFAPacketizer.cpp
+++ b/llvm/lib/CodeGen/DFAPacketizer.cpp
@@ -252,12 +252,13 @@ void VLIWPacketizerList::PacketizeMIs(MachineBasicBlock *MBB,
bool VLIWPacketizerList::alias(const MachineMemOperand &Op1,
const MachineMemOperand &Op2,
bool UseTBAA) const {
- if (!Op1.getValue() || !Op2.getValue())
+ if (!Op1.getValue() || !Op2.getValue() || !Op1.getSize().hasValue() ||
+ !Op2.getSize().hasValue())
return true;
int64_t MinOffset = std::min(Op1.getOffset(), Op2.getOffset());
- int64_t Overlapa = Op1.getSize() + Op1.getOffset() - MinOffset;
- int64_t Overlapb = Op2.getSize() + Op2.getOffset() - MinOffset;
+ int64_t Overlapa = Op1.getSize().getValue() + Op1.getOffset() - MinOffset;
+ int64_t Overlapb = Op2.getSize().getValue() + Op2.getOffset() - MinOffset;
AliasResult AAResult =
AA->alias(MemoryLocation(Op1.getValue(), Overlapa,
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index ab055b723dbb1f..79541490ce7412 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -770,12 +770,12 @@ bool CombinerHelper::matchCombineLoadWithAndMask(MachineInstr &MI,
LLT RegTy = MRI.getType(LoadReg);
Register PtrReg = LoadMI->getPointerReg();
unsigned RegSize = RegTy.getSizeInBits();
- uint64_t LoadSizeBits = LoadMI->getMemSizeInBits();
+ LocationSize LoadSizeBits = LoadMI->getMemSizeInBits();
unsigned MaskSizeBits = MaskVal.countr_one();
// The mask may not be larger than the in-memory type, as it might cover sign
// extended bits
- if (MaskSizeBits > LoadSizeBits)
+ if (MaskSizeBits > LoadSizeBits.getValue())
return false;
// If the mask covers the whole destination register, there's nothing to
@@ -795,7 +795,8 @@ bool CombinerHelper::matchCombineLoadWithAndMask(MachineInstr &MI,
// still adjust the opcode to indicate the high bit behavior.
if (LoadMI->isSimple())
MemDesc.MemoryTy = LLT::scalar(MaskSizeBits);
- else if (LoadSizeBits > MaskSizeBits || LoadSizeBits == RegSize)
+ else if (LoadSizeBits.getValue() > MaskSizeBits ||
+ LoadSizeBits.getValue() == RegSize)
return false;
// TODO: Could check if it's legal with the reduced or original memory size.
@@ -860,7 +861,8 @@ bool CombinerHelper::matchSextTruncSextLoad(MachineInstr &MI) {
if (auto *LoadMI = getOpcodeDef<GSExtLoad>(LoadUser, MRI)) {
// If truncating more than the original extended value, abort.
auto LoadSizeBits = LoadMI->getMemSizeInBits();
- if (TruncSrc && MRI.getType(TruncSrc).getSizeInBits() < LoadSizeBits)
+ if (TruncSrc &&
+ MRI.getType(TruncSrc).getSizeInBits() < LoadSizeBits.getValue())
return false;
if (LoadSizeBits == SizeInBits)
return true;
@@ -891,7 +893,7 @@ bool CombinerHelper::matchSextInRegOfLoad(
if (!LoadDef || !MRI.hasOneNonDBGUse(DstReg))
return false;
- uint64_t MemBits = LoadDef->getMemSizeInBits();
+ uint64_t MemBits = LoadDef->getMemSizeInBits().getValue();
// If the sign extend extends from a narrower width than the load's width,
// then we can narrow the load width when we combine to a G_SEXTLOAD.
diff --git a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
index 099bf45b2734cb..2e2cc9a95bd95c 100644
--- a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
@@ -415,7 +415,8 @@ void GISelKnownBits::computeKnownBitsImpl(Register R, KnownBits &Known,
if (DstTy.isVector())
break;
// Everything above the retrieved bits is zero
- Known.Zero.setBitsFrom((*MI.memoperands_begin())->getSizeInBits());
+ Known.Zero.setBitsFrom(
+ (*MI.memoperands_begin())->getSizeInBits().getValue());
break;
}
case TargetOpcode::G_ASHR: {
@@ -666,7 +667,7 @@ unsigned GISelKnownBits::computeNumSignBits(Register R,
// e.g. i16->i32 = '17' bits known.
const MachineMemOperand *MMO = *MI.memoperands_begin();
- return TyBits - MMO->getSizeInBits() + 1;
+ return TyBits - MMO->getSizeInBits().getValue() + 1;
}
case TargetOpcode::G_ZEXTLOAD: {
// FIXME: We need an in-memory type representation.
@@ -675,7 +676,7 @@ unsigned GISelKnownBits::computeNumSignBits(Register R,
// e.g. i16->i32 = '16' bits known.
const MachineMemOperand *MMO = *MI.memoperands_begin();
- return TyBits - MMO->getSizeInBits();
+ return TyBits - MMO->getSizeInBits().getValue();
}
case TargetOpcode::G_TRUNC: {
Register Src = MI.getOperand(1).getReg();
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 2ec47f72aca39a..d95f461760a5e3 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -1317,7 +1317,7 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
if (DstTy.isVector())
return UnableToLegalize;
- if (8 * LoadMI.getMemSize() != DstTy.getSizeInBits()) {
+ if (8 * LoadMI.getMemSize().getValue() != DstTy.getSizeInBits()) {
Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
MIRBuilder.buildLoad(TmpReg, LoadMI.getPointerReg(), LoadMI.getMMO());
MIRBuilder.buildAnyExt(DstReg, TmpReg);
@@ -1335,7 +1335,7 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
auto &MMO = LoadMI.getMMO();
- unsigned MemSize = MMO.getSizeInBits();
+ unsigned MemSize = MMO.getSizeInBits().getValue();
if (MemSize == NarrowSize) {
MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
@@ -1368,7 +1368,7 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
if (SrcTy.isVector() && LeftoverBits != 0)
return UnableToLegalize;
- if (8 * StoreMI.getMemSize() != SrcTy.getSizeInBits()) {
+ if (8 * StoreMI.getMemSize().getValue() != SrcTy.getSizeInBits()) {
Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
MIRBuilder.buildTrunc(TmpReg, SrcReg);
MIRBuilder.buildStore(TmpReg, StoreMI.getPointerReg(), StoreMI.getMMO());
@@ -4449,7 +4449,7 @@ LegalizerHelper::reduceLoadStoreWidth(GLoadStore &LdStMI, unsigned TypeIdx,
LLT ValTy = MRI.getType(ValReg);
// FIXME: Do we need a distinct NarrowMemory legalize action?
- if (ValTy.getSizeInBits() != 8 * LdStMI.getMemSize()) {
+ if (ValTy.getSizeInBits() != 8 * LdStMI.getMemSize().getValue()) {
LLVM_DEBUG(dbgs() << "Can't narrow extload/truncstore\n");
return UnableToLegalize;
}
diff --git a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
index 7756f6ed0d53e9..6229ef3c65204b 100644
--- a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
@@ -117,12 +117,8 @@ bool GISelAddressing::aliasIsKnownForLoadStore(const MachineInstr &MI1,
if (!BasePtr0.BaseReg.isValid() || !BasePtr1.BaseReg.isValid())
return false;
- LocationSize Size1 = LdSt1->getMemSize() != MemoryLocation::UnknownSize
- ? LdSt1->getMemSize()
- : LocationSize::beforeOrAfterPointer();
- LocationSize Size2 = LdSt2->getMemSize() != MemoryLocation::UnknownSize
- ? LdSt2->getMemSize()
- : LocationSize::beforeOrAfterPointer();
+ LocationSize Size1 = LdSt1->getMemSize();
+ LocationSize Size2 = LdSt2->getMemSize();
int64_t PtrDiff;
if (BasePtr0.BaseReg == BasePtr1.BaseReg) {
@@ -132,14 +128,14 @@ bool GISelAddressing::aliasIsKnownForLoadStore(const MachineInstr &MI1,
// vector objects on the stack.
// BasePtr1 is PtrDiff away from BasePtr0. They alias if none of the
// following situations arise:
- if (PtrDiff >= 0 && Size1.hasValue()) {
+ if (PtrDiff >= 0 && Size1.hasValue() && !Size1.isScalable()) {
// [----BasePtr0----]
// [---BasePtr1--]
// ========PtrDiff========>
IsAlias = !((int64_t)Size1.getValue() <= PtrDiff);
return true;
}
- if (PtrDiff < 0 && Size2.hasValue()) {
+ if (PtrDiff < 0 && Size2.hasValue() && !Size2.isScalable()) {
// [----BasePtr0----]
// [---BasePtr1--]
// =====(-PtrDiff)====>
@@ -214,9 +210,9 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
Offset = 0;
}
- TypeSize Size = LS->getMMO().getMemoryType().getSizeInBytes();
- return {LS->isVolatile(), LS->isAtomic(), BaseReg,
- Offset /*base offset*/, LocationSize::precise(Size), &LS->getMMO()};
+ LocationSize Size = LS->getMMO().getSize();
+ return {LS->isVolatile(), LS->isAtomic(), BaseReg,
+ Offset /*base offset*/, Size, &LS->getMMO()};
}
// FIXME: support recognizing lifetime instructions.
// Default.
@@ -254,12 +250,12 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
// If NumBytes is scalable and offset is not 0, conservatively return may
// alias
- if ((MUC0.NumBytes.isScalable() && (MUC0.Offset != 0)) ||
- (MUC1.NumBytes.isScalable() && (MUC1.Offset != 0)))
+ if ((MUC0.NumBytes.isScalable() && MUC0.Offset != 0) ||
+ (MUC1.NumBytes.isScalable() && MUC1.Offset != 0))
return true;
const bool BothNotScalable =
- !(MUC0.NumBytes.isScalable() || MUC1.NumBytes.isScalable());
+ !MUC0.NumBytes.isScalable() && !MUC1.NumBytes.isScalable();
// Try to prove that there is aliasing, or that there is no aliasing. Either
// way, we can return now. If nothing can be proved, proceed with more tests.
@@ -527,7 +523,7 @@ bool LoadStoreOpt::addStoreToCandidate(GStore &StoreMI,
return false;
// Don't allow truncating stores for now.
- if (StoreMI.getMemSizeInBits() != ValueTy.getSizeInBits())
+ if (StoreMI.getMemSizeInBits().getValue() != ValueTy.getSizeInBits())
return false;
// Avoid adding volatile or ordered stores to the candidate. We already have a
diff --git a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
index cfc8c28b99e562..43c278831c987d 100644
--- a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
+++ b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
@@ -1356,7 +1356,7 @@ InstrRefBasedLDV::findLocationForMemOperand(const MachineInstr &MI) {
// from the stack at some point. Happily the memory operand will tell us
// the size written to the stack.
auto *MemOperand = *MI.memoperands_begin();
- unsigned SizeInBits = MemOperand->getSizeInBits();
+ unsigned SizeInBits = MemOperand->getSizeInBits().getValue();
// Find that position in the stack indexes we're tracking.
auto IdxIt = MTracker->StackSlotIdxes.find({SizeInBits, 0});
diff --git a/llvm/lib/CodeGen/MIRVRegNamerUtils.cpp b/llvm/lib/CodeGen/MIRVRegNamerUtils.cpp
index 812d57984e6cae..ccfc4565d3a9bc 100644
--- a/llvm/lib/CodeGen/MIRVRegNamerUtils.cpp
+++ b/llvm/lib/CodeGen/MIRVRegNamerUtils.cpp
@@ -123,7 +123,7 @@ std::string VRegRenamer::getInstructionOpcodeHash(MachineInstr &MI) {
llvm::transform(MI.uses(), std::back_inserter(MIOperands), GetHashableMO);
for (const auto *Op : MI.memoperands()) {
- MIOperands.push_back((unsigned)Op->getSize());
+ MIOperands.push_back((unsigned)Op->getSize().getValue());
MIOperands.push_back((unsigned)Op->getFlags());
MIOperands.push_back((unsigned)Op->getOffset());
MIOperands.push_back((unsigned)Op->getSuccessOrdering());
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index 06f89badb65b96..3e16e9db57e1cb 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -484,17 +484,7 @@ void MachineFunction::deleteMachineBasicBlock(MachineBasicBlock *MBB) {
}
MachineMemOperand *MachineFunction::getMachineMemOperand(
- MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s,
- Align base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
- SyncScope::ID SSID, AtomicOrdering Ordering,
- AtomicOrdering FailureOrdering) {
- return new (Allocator)
- MachineMemOperand(PtrInfo, f, s, base_alignment, AAInfo, Ranges,
- SSID, Ordering, FailureOrdering);
-}
-
-MachineMemOperand *MachineFunction::getMachineMemOperand(
- MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, TypeSize TS,
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LocationSize TS,
Align BaseAlignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
SyncScope::ID SSID, AtomicOrdering Ordering,
AtomicOrdering FailureOrdering) {
@@ -513,18 +503,10 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
Ordering, FailureOrdering);
}
-MachineMemOperand *MachineFunction::getMachineMemOperand(
- const MachineMemOperand *MMO, const MachinePointerInfo &PtrInfo, uint64_t Size) {
- return new (Allocator)
- MachineMemOperand(PtrInfo, MMO->getFlags(), Size, MMO->getBaseAlign(),
- AAMDNodes(), nullptr, MMO->getSyncScopeID(),
- MMO->getSuccessOrdering(), MMO->getFailureOrdering());
-}
-
MachineMemOperand *
MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
- TypeSize TS) {
+ LocationSize TS) {
return new (Allocator)
MachineMemOperand(PtrInfo, MMO->getFlags(), TS, MMO->getBaseAlign(),
AAMDNodes(), nullptr, MMO->getSyncScopeID(),
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 6aebd71d92028a..8d6909abc5d6c4 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -1302,10 +1302,10 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
int64_t OffsetB = MMOb->getOffset();
int64_t MinOffset = std::min(OffsetA, OffsetB);
- TypeSize WidthA = MMOa->getSize();
- TypeSize WidthB = MMOb->getSize();
- bool KnownWidthA = WidthA.getKnownMinValue() != MemoryLocation::UnknownSize;
- bool KnownWidthB = WidthB.getKnownMinValue() != MemoryLocation::UnknownSize;
+ LocationSize WidthA = MMOa->getSize();
+ LocationSize WidthB = MMOb->getSize();
+ bool KnownWidthA = WidthA.hasValue();
+ bool KnownWidthB = WidthB.hasValue();
bool BothMMONonScalable = !WidthA.isScalable() && !WidthB.isScalable();
const Value *ValA = MMOa->getValue();
@@ -1326,8 +1326,9 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
if (!KnownWidthA || !KnownWidthB)
return true;
int64_t MaxOffset = std::max(OffsetA, OffsetB);
- int64_t LowWidth = (MinOffset == OffsetA) ? WidthA.getKnownMinValue()
- : WidthB.getKnownMinValue();
+ int64_t LowWidth = (MinOffset == OffsetA)
+ ? WidthA.getValue().getKnownMinValue()
+ : WidthB.getValue().getKnownMinValue();
return (MinOffset + LowWidth > MaxOffset);
}
@@ -1346,19 +1347,19 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
(WidthB.isScalable() && OffsetB > 0))
return true;
- int64_t OverlapA = KnownWidthA
- ? WidthA.getKnownMinValue() + OffsetA - MinOffset
- : MemoryLocation::UnknownSize;
- int64_t OverlapB = KnownWidthB
- ? WidthB.getKnownMinValue() + OffsetB - MinOffset
- : MemoryLocation::UnknownSize;
+ int64_t OverlapA =
+ KnownWidthA ? WidthA.getValue().getKnownMinValue() + OffsetA - MinOffset
+ : MemoryLocation::UnknownSize;
+ int64_t OverlapB =
+ KnownWidthB ? WidthB.getValue().getKnownMinValue() + OffsetB - MinOffset
+ : MemoryLocation::UnknownSize;
- LocationSize LocA = WidthA.isScalable() && KnownWidthA
- ? LocationSize::precise(WidthA)
- : LocationSize(OverlapA);
- LocationSize LocB = WidthB.isScalable() && KnownWidthB
- ? LocationSize::precise(WidthB)
- : LocationSize(OverlapB);
+ LocationSize LocA = (WidthA.isScalable() || !KnownWidthA)
+ ? WidthA
+ : LocationSize::precise(OverlapA);
+ LocationSize LocB = (WidthB.isScalable() || !KnownWidthB)
+ ? WidthB
+ : LocationSize::precise(OverlapB);
return !AA->isNoAlias(
MemoryLocation(ValA, LocA, UseTBAA ? MMOa->getAAInfo() : AAMDNodes()),
@@ -2373,15 +2374,16 @@ using MMOList = SmallVector<const MachineMemOperand *, 2>;
static LocationSize getSpillSlotSize(const MMOList &Accesses,
const MachineFrameInfo &MFI) {
uint64_t Size = 0;
- for (const auto *A : Accesses)
+ for (const auto *A : Accesses) {
if (MFI.isSpillSlotObjectIndex(
cast<FixedStackPseudoSourceValue>(A->getPseudoValue())
->getFrameIndex())) {
- uint64_t S = A->getSize();
- if (S == ~UINT64_C(0))
+ LocationSize S = A->getSize();
+ if (!S.hasValue())
return LocationSize::beforeOrAfterPointer();
- Size += S;
+ Size += S.getValue();
}
+ }
return Size;
}
@@ -2390,10 +2392,8 @@ MachineInstr::getSpillSize(const TargetInstrInfo *TII) const {
int FI;
if (TII->isStoreToStackSlotPostFE(*this, FI)) {
const MachineFrameInfo &MFI = getMF()->getFrameInfo();
- if (MFI.isSpillSlotObjectIndex(FI)) {
- uint64_t Size = (*memoperands_begin())->getSize();
- return Size == ~UINT64_C(0) ? LocationSize::beforeOrAfterPointer() : Size;
- }
+ if (MFI.isSpillSlotObjectIndex(FI))
+ return (*memoperands_begin())->getSize();
}
return std::nullopt;
}
@@ -2411,10 +2411,8 @@ MachineInstr::getRestoreSize(const TargetInstrInfo *TII) const {
int FI;
if (TII->isLoadFromStackSlotPostFE(*this, FI)) {
const MachineFrameInfo &MFI = getMF()->getFrameInfo();
- if (MFI.isSpillSlotObjectIndex(FI)) {
- uint64_t Size = (*memoperands_begin())->getSize();
- return Size == ~UINT64_C(0) ? LocationSize::beforeOrAfterPointer() : Size;
- }
+ if (MFI.isSpillSlotObjectIndex(FI))
+ return (*memoperands_begin())->getSize();
}
return std::nullopt;
}
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index 2823760a94b385..ace05902d5df79 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -1101,35 +1101,25 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
assert(getFailureOrdering() == FailureOrdering && "Value truncated");
}
-MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
- uint64_t s, Align a,
- const AAMDNodes &AAInfo,
- const MDNode *Ranges, SyncScope::ID SSID,
- AtomicOrdering Ordering,
- AtomicOrdering FailureOrdering)
- : MachineMemOperand(ptrinfo, f,
- s == ~UINT64_C(0) ? LLT() : LLT::scalar(8 * s), a,
- AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
-
MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags F,
- TypeSize TS, Align BaseAlignment,
+ LocationSize TS, Align BaseAlignment,
const AAMDNodes &AAInfo,
const MDNode *Ranges, SyncScope::ID SSID,
AtomicOrdering Ordering,
AtomicOrdering FailureOrdering)
: MachineMemOperand(
ptrinfo, F,
- TS.getKnownMinValue() == ~UINT64_C(0) ? LLT()
- : TS.isScalable() ? LLT::scalable_vector(1, 8 * TS.getKnownMinValue())
- : LLT::scalar(8 * TS.getKnownMinValue()),
+ !TS.hasValue() ? LLT()
+ : TS.isScalable()
+ ? LLT::scalable_vector(1, 8 * TS.getValue().getKnownMinValue())
+ : LLT::scalar(8 * TS.getValue().getKnownMinValue()),
BaseAlignment, AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
void MachineMemOperand::refineAlignment(const MachineMemOperand *MMO) {
// The Value and Offset may differ due to CSE. But the flags and size
// should be the same.
assert(MMO->getFlags() == getFlags() && "Flags mismatch!");
- assert((MMO->getSize().getKnownMinValue() == ~UINT64_C(0) ||
- getSize().getKnownMinValue() == ~UINT64_C(0) ||
+ assert((!MMO->getSize().hasValue() || !getSize().hasValue() ||
MMO->getSize() == getSize()) &&
"Size mismatch!");
if (MMO->getBaseAlign() >= getBaseAlign()) {
@@ -1253,10 +1243,8 @@ void MachineMemOperand::print(raw_ostream &OS, ModuleSlotTracker &MST,
<< "unknown-address";
}
MachineOperand::printOperandOffset(OS, getOffset());
- if (getSize().isScalable())
- OS << ", vscale ";
- if (getSize().getKnownMinValue() > 0 &&
- getAlign() != getSize().getKnownMinValue())
+ if (!getSize().hasValue() ||
+ getAlign() != getSize().getValue().getKnownMinValue())
OS << ", align " << getAlign().value();
if (getAlign() != getBaseAlign())
OS << ", basealign " << getBaseAlign().value();
diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp
index d8cb6816883394..eb42a78603d407 100644
--- a/llvm/lib/CodeGen/MachinePipeliner.cpp
+++ b/llvm/lib/CodeGen/MachinePipeliner.cpp
@@ -2732,19 +2732,20 @@ bool SwingSchedulerDAG::isLoopCarriedDep(SUnit *Source, const SDep &Dep,
if (!LoopDefS || !TII->getIncrementValue(*LoopDefS, D))
return true;
- uint64_t AccessSizeS = (*SI->memoperands_begin())->getSize();
- uint64_t AccessSizeD = (*DI->memoperands_begin())->getSize();
+ LocationSize AccessSizeS = (*SI->memoperands_begin())->getSize();
+ LocationSize AccessSizeD = (*DI->memoperands_begin())->getSize();
// This is the main test, which checks the offset values and the loop
// increment value to determine if the accesses may be loop carried.
- if (AccessSizeS == MemoryLocation::UnknownSize ||
- AccessSizeD == MemoryLocation::UnknownSize)
+ if (!AccessSizeS.hasValue() || !AccessSizeD.hasValue())
return true;
- if (DeltaS != DeltaD || DeltaS < AccessSizeS || DeltaD < AccessSizeD)
+ if (DeltaS != DeltaD || DeltaS < AccessSizeS.getValue() ||
+ DeltaD < AccessSizeD.getValue())
return true;
- return (OffsetS + (int64_t)AccessSizeS < OffsetD + (int64_t)AccessSizeD);
+ return (OffsetS + (int64_t)AccessSizeS.getValue() <
+ OffsetD + (int64_t)AccessSizeD.getValue());
}
void SwingSchedulerDAG::postProcessDAG() {
diff --git a/llvm/lib/CodeGen/MachineStableHash.cpp b/llvm/lib/CodeGen/MachineStableHash.cpp
index 1cd90474898e77..5abfbd5981fba8 100644
--- a/llvm/lib/CodeGen/MachineStableHash.cpp
+++ b/llvm/lib/CodeGen/MachineStableHash.cpp
@@ -200,7 +200,7 @@ stable_hash llvm::stableHashValue(const MachineInstr &MI, bool HashVRegs,
for (const auto *Op : MI.memoperands()) {
if (!HashMemOperands)
break;
- HashComponents.push_back(static_cast<unsigned>(Op->getSize()));
+ HashComponents.push_back(static_cast<unsigned>(Op->getSize().getValue()));
HashComponents.push_back(static_cast<unsigned>(Op->getFlags()));
HashComponents.push_back(static_cast<unsigned>(Op->getOffset()));
HashComponents.push_back(static_cast<unsigned>(Op->getSuccessOrdering()));
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index ecb3bd33bdfd49..5f9e8449fb7eea 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -1195,13 +1195,16 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
const MachineMemOperand &MMO = **MI->memoperands_begin();
if (MI->getOpcode() == TargetOpcode::G_ZEXTLOAD ||
MI->getOpcode() == TargetOpcode::G_SEXTLOAD) {
- if (MMO.getSizeInBits() >= ValTy.getSizeInBits())
+ if (TypeSize::isKnownGE(MMO.getSizeInBits().getValue(),
+ ValTy.getSizeInBits()))
report("Generic extload must have a narrower memory type", MI);
} else if (MI->getOpcode() == TargetOpcode::G_LOAD) {
- if (MMO.getSize() > ValTy.getSizeInBytes())
+ if (TypeSize::isKnownGT(MMO.getSize().getValue(),
+ ValTy.getSizeInBytes()))
report("load memory size cannot exceed result size", MI);
} else if (MI->getOpcode() == TargetOpcode::G_STORE) {
- if (ValTy.getSizeInBytes() < MMO.getSize())
+ if (TypeSize::isKnownLT(ValTy.getSizeInBytes(),
+ MMO.getSize().getValue()))
report("store memory size cannot exceed value size", MI);
}
diff --git a/llvm/lib/CodeGen/ModuloSchedule.cpp b/llvm/lib/CodeGen/ModuloSchedule.cpp
index 0bef513342ff12..bdae94c4e6f885 100644
--- a/llvm/lib/CodeGen/ModuloSchedule.cpp
+++ b/llvm/lib/CodeGen/ModuloSchedule.cpp
@@ -979,8 +979,8 @@ void ModuloScheduleExpander::updateMemOperands(MachineInstr &NewMI,
NewMMOs.push_back(
MF.getMachineMemOperand(MMO, AdjOffset, MMO->getSize()));
} else {
- NewMMOs.push_back(
- MF.getMachineMemOperand(MMO, 0, MemoryLocation::UnknownSize));
+ NewMMOs.push_back(MF.getMachineMemOperand(
+ MMO, 0, LocationSize::beforeOrAfterPointer()));
}
}
NewMI.setMemRefs(MF, NewMMOs);
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 7ba974286c67c5..c8e081b86089a1 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -27869,12 +27869,9 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
? -1 * C->getSExtValue()
: 0;
TypeSize Size = LSN->getMemoryVT().getStoreSize();
- return {LSN->isVolatile(),
- LSN->isAtomic(),
- LSN->getBasePtr(),
- Offset /*base offset*/,
- LocationSize::precise(Size),
- LSN->getMemOperand()};
+ return {LSN->isVolatile(), LSN->isAtomic(),
+ LSN->getBasePtr(), Offset /*base offset*/,
+ LocationSize::precise(Size), LSN->getMemOperand()};
}
if (const auto *LN = cast<LifetimeSDNode>(N))
return {false /*isVolatile*/,
@@ -27956,16 +27953,20 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 &&
Size0.hasValue() && Size1.hasValue() && !Size0.isScalable() &&
- !Size1.isScalable() && Size0.getValue() == Size1.getValue() && OrigAlignment0 > Size0.getValue() &&
- SrcValOffset0 % Size0.getValue() == 0 && SrcValOffset1 % Size1.getValue() == 0) {
+ !Size1.isScalable() && Size0 == Size1 &&
+ OrigAlignment0 > Size0.getValue().getKnownMinValue() &&
+ SrcValOffset0 % Size0.getValue().getKnownMinValue() == 0 &&
+ SrcValOffset1 % Size1.getValue().getKnownMinValue() == 0) {
int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.value();
int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.value();
- // There is no overlap between these relatively aligned accesses of
- // similar size. Return no alias.
- if ((OffAlign0 + Size0.getValue()) <= OffAlign1 ||
- (OffAlign1 + Size1.getValue()) <= OffAlign0)
- return false;
+ // There is no overlap between these relatively aligned accesses of
+ // similar size. Return no alias.
+ if ((OffAlign0 + static_cast<int64_t>(
+ Size0.getValue().getKnownMinValue())) <= OffAlign1 ||
+ (OffAlign1 + static_cast<int64_t>(
+ Size1.getValue().getKnownMinValue())) <= OffAlign0)
+ return false;
}
bool UseAA = CombinerGlobalAA.getNumOccurrences() > 0
@@ -27985,12 +27986,10 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
Size0.getValue().getKnownMinValue() + SrcValOffset0 - MinOffset;
int64_t Overlap1 =
Size1.getValue().getKnownMinValue() + SrcValOffset1 - MinOffset;
- LocationSize Loc0 = Size0.getValue().isScalable()
- ? LocationSize::precise(Size0.getValue())
- : LocationSize::precise(Overlap0);
- LocationSize Loc1 = Size1.isScalable()
- ? LocationSize::precise(Size1.getValue())
- : LocationSize::precise(Overlap1);
+ LocationSize Loc0 =
+ Size0.isScalable() ? Size0 : LocationSize::precise(Overlap0);
+ LocationSize Loc1 =
+ Size1.isScalable() ? Size1 : LocationSize::precise(Overlap1);
if (AA->isNoAlias(
MemoryLocation(MUC0.MMO->getValue(), Loc0,
UseTBAA ? MUC0.MMO->getAAInfo() : AAMDNodes()),
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 5fb9d8d07d1514..1f6e0097f31ab4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -1967,7 +1967,8 @@ void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
LD->getPointerInfo(), MachineMemOperand::MOLoad,
- MemoryLocation::UnknownSize, Alignment, LD->getAAInfo(), LD->getRanges());
+ LocationSize::beforeOrAfterPointer(), Alignment, LD->getAAInfo(),
+ LD->getRanges());
Lo =
DAG.getLoadVP(LD->getAddressingMode(), ExtType, LoVT, dl, Ch, Ptr, Offset,
@@ -1990,8 +1991,8 @@ void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
LoMemVT.getStoreSize().getFixedValue());
MMO = DAG.getMachineFunction().getMachineMemOperand(
- MPI, MachineMemOperand::MOLoad, MemoryLocation::UnknownSize, Alignment,
- LD->getAAInfo(), LD->getRanges());
+ MPI, MachineMemOperand::MOLoad, LocationSize::beforeOrAfterPointer(),
+ Alignment, LD->getAAInfo(), LD->getRanges());
Hi = DAG.getLoadVP(LD->getAddressingMode(), ExtType, HiVT, dl, Ch, Ptr,
Offset, MaskHi, EVLHi, HiMemVT, MMO,
@@ -2070,8 +2071,8 @@ void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(SLD->getPointerInfo().getAddrSpace()),
- MachineMemOperand::MOLoad, MemoryLocation::UnknownSize, Alignment,
- SLD->getAAInfo(), SLD->getRanges());
+ MachineMemOperand::MOLoad, LocationSize::beforeOrAfterPointer(),
+ Alignment, SLD->getAAInfo(), SLD->getRanges());
Hi = DAG.getStridedLoadVP(SLD->getAddressingMode(), SLD->getExtensionType(),
HiVT, DL, SLD->getChain(), Ptr, SLD->getOffset(),
@@ -2130,7 +2131,7 @@ void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MLD->getPointerInfo(), MachineMemOperand::MOLoad,
- MemoryLocation::UnknownSize, Alignment, MLD->getAAInfo(),
+ LocationSize::beforeOrAfterPointer(), Alignment, MLD->getAAInfo(),
MLD->getRanges());
Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT,
@@ -2154,8 +2155,8 @@ void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
LoMemVT.getStoreSize().getFixedValue());
MMO = DAG.getMachineFunction().getMachineMemOperand(
- MPI, MachineMemOperand::MOLoad, MemoryLocation::UnknownSize, Alignment,
- MLD->getAAInfo(), MLD->getRanges());
+ MPI, MachineMemOperand::MOLoad, LocationSize::beforeOrAfterPointer(),
+ Alignment, MLD->getAAInfo(), MLD->getRanges());
Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi,
HiMemVT, MMO, MLD->getAddressingMode(), ExtType,
@@ -2217,7 +2218,8 @@ void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
N->getPointerInfo(), MachineMemOperand::MOLoad,
- MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
+ LocationSize::beforeOrAfterPointer(), Alignment, N->getAAInfo(),
+ N->getRanges());
if (auto *MGT = dyn_cast<MaskedGatherSDNode>(N)) {
SDValue PassThru = MGT->getPassThru();
@@ -2884,10 +2886,10 @@ void DAGTypeLegalizer::SplitVecRes_VP_REVERSE(SDNode *N, SDValue &Lo,
auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
- PtrInfo, MachineMemOperand::MOStore, MemoryLocation::UnknownSize,
+ PtrInfo, MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer(),
Alignment);
MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
- PtrInfo, MachineMemOperand::MOLoad, MemoryLocation::UnknownSize,
+ PtrInfo, MachineMemOperand::MOLoad, LocationSize::beforeOrAfterPointer(),
Alignment);
unsigned EltWidth = VT.getScalarSizeInBits() / 8;
@@ -3478,7 +3480,8 @@ SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
SDValue Lo, Hi;
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
N->getPointerInfo(), MachineMemOperand::MOStore,
- MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
+ LocationSize::beforeOrAfterPointer(), Alignment, N->getAAInfo(),
+ N->getRanges());
Lo = DAG.getStoreVP(Ch, DL, DataLo, Ptr, Offset, MaskLo, EVLLo, LoMemVT, MMO,
N->getAddressingMode(), N->isTruncatingStore(),
@@ -3501,8 +3504,8 @@ SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
LoMemVT.getStoreSize().getFixedValue());
MMO = DAG.getMachineFunction().getMachineMemOperand(
- MPI, MachineMemOperand::MOStore, MemoryLocation::UnknownSize, Alignment,
- N->getAAInfo(), N->getRanges());
+ MPI, MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer(),
+ Alignment, N->getAAInfo(), N->getRanges());
Hi = DAG.getStoreVP(Ch, DL, DataHi, Ptr, Offset, MaskHi, EVLHi, HiMemVT, MMO,
N->getAddressingMode(), N->isTruncatingStore(),
@@ -3574,8 +3577,8 @@ SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(N->getPointerInfo().getAddrSpace()),
- MachineMemOperand::MOStore, MemoryLocation::UnknownSize, Alignment,
- N->getAAInfo(), N->getRanges());
+ MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer(),
+ Alignment, N->getAAInfo(), N->getRanges());
SDValue Hi = DAG.getStridedStoreVP(
N->getChain(), DL, HiData, Ptr, N->getOffset(), N->getStride(), HiMask,
@@ -3626,7 +3629,8 @@ SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
SDValue Lo, Hi, Res;
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
N->getPointerInfo(), MachineMemOperand::MOStore,
- MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
+ LocationSize::beforeOrAfterPointer(), Alignment, N->getAAInfo(),
+ N->getRanges());
Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO,
N->getAddressingMode(), N->isTruncatingStore(),
@@ -3651,8 +3655,8 @@ SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
LoMemVT.getStoreSize().getFixedValue());
MMO = DAG.getMachineFunction().getMachineMemOperand(
- MPI, MachineMemOperand::MOStore, MemoryLocation::UnknownSize, Alignment,
- N->getAAInfo(), N->getRanges());
+ MPI, MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer(),
+ Alignment, N->getAAInfo(), N->getRanges());
Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO,
N->getAddressingMode(), N->isTruncatingStore(),
@@ -3716,7 +3720,8 @@ SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
SDValue Lo;
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
N->getPointerInfo(), MachineMemOperand::MOStore,
- MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
+ LocationSize::beforeOrAfterPointer(), Alignment, N->getAAInfo(),
+ N->getRanges());
if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) {
SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 641a596f36eaf3..b6acbaed5661b2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -8390,11 +8390,10 @@ SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
SDValue SelectionDAG::getMemIntrinsicNode(
unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
- MachineMemOperand::Flags Flags, uint64_t Size, const AAMDNodes &AAInfo) {
- if (!Size && MemVT.isScalableVector())
- Size = MemoryLocation::UnknownSize;
- else if (!Size)
- Size = MemVT.getStoreSize();
+ MachineMemOperand::Flags Flags, LocationSize Size,
+ const AAMDNodes &AAInfo) {
+ if (Size.hasValue() && !Size.getValue())
+ Size = LocationSize::precise(MemVT.getStoreSize());
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO =
@@ -11712,9 +11711,10 @@ MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
// We check here that the size of the memory operand fits within the size of
// the MMO. This is because the MMO might indicate only a possible address
// range instead of specifying the affected memory addresses precisely.
- assert((!MMO->getType().isValid() ||
- TypeSize::isKnownLE(memvt.getStoreSize(), MMO->getSize())) &&
- "Size mismatch!");
+ assert(
+ (!MMO->getType().isValid() ||
+ TypeSize::isKnownLE(memvt.getStoreSize(), MMO->getSize().getValue())) &&
+ "Size mismatch!");
}
/// Profile - Gather unique data for the node.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp
index 9670c3ac8430eb..f2ab88851b780e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp
@@ -106,8 +106,6 @@ bool BaseIndexOffset::computeAliasing(const SDNode *Op0,
int64_t PtrDiff;
if (BasePtr0.equalBaseIndex(BasePtr1, DAG, PtrDiff)) {
// If the size of memory access is unknown, do not use it to analysis.
- // One example of unknown size memory access is to load/store scalable
- // vector objects on the stack.
// BasePtr1 is PtrDiff away from BasePtr0. They alias if none of the
// following situations arise:
if (PtrDiff >= 0 && NumBytes0.hasValue() && !NumBytes0.isScalable()) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 22e57d0d99e9b1..5e646329ee57b4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -4753,7 +4753,7 @@ void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
- MemoryLocation::UnknownSize, Alignment, I.getAAMetadata());
+ LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata());
SDValue StoreNode =
DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, VT, MMO,
ISD::UNINDEXED, false /* Truncating */, IsCompressing);
@@ -4857,9 +4857,7 @@ void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(AS), MachineMemOperand::MOStore,
- // TODO: Make MachineMemOperands aware of scalable
- // vectors.
- MemoryLocation::UnknownSize, Alignment, I.getAAMetadata());
+ LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata());
if (!UniformBase) {
Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
Index = getValue(Ptr);
@@ -4925,7 +4923,7 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
- MemoryLocation::UnknownSize, Alignment, AAInfo, Ranges);
+ LocationSize::beforeOrAfterPointer(), Alignment, AAInfo, Ranges);
SDValue Load =
DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO,
@@ -4961,9 +4959,8 @@ void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(AS), MachineMemOperand::MOLoad,
- // TODO: Make MachineMemOperands aware of scalable
- // vectors.
- MemoryLocation::UnknownSize, Alignment, I.getAAMetadata(), Ranges);
+ LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata(),
+ Ranges);
if (!UniformBase) {
Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
@@ -6904,7 +6901,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
auto MPI =
MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
- MPI, MachineMemOperand::MOStore, MemoryLocation::UnknownSize,
+ MPI, MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer(),
TempAlign);
Chain = DAG.getGetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
Res = DAG.getLoad(EnvVT, sdl, Chain, Temp, MPI);
@@ -6933,7 +6930,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
Chain = DAG.getStore(Chain, sdl, Env, Temp, MPI, TempAlign,
MachineMemOperand::MOStore);
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
- MPI, MachineMemOperand::MOLoad, MemoryLocation::UnknownSize,
+ MPI, MachineMemOperand::MOLoad, LocationSize::beforeOrAfterPointer(),
TempAlign);
Chain = DAG.getSetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
}
@@ -8087,7 +8084,7 @@ void SelectionDAGBuilder::visitVPLoad(
SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
- MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
+ LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
MMO, false /*IsExpanding */);
if (AddToChain)
@@ -8110,8 +8107,8 @@ void SelectionDAGBuilder::visitVPGather(
unsigned AS =
PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
- MachinePointerInfo(AS), MachineMemOperand::MOLoad,
- MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
+ MachinePointerInfo(AS), MachineMemOperand::MOLoad,
+ LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
SDValue Base, Index, Scale;
ISD::MemIndexType IndexType;
bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
@@ -8151,7 +8148,7 @@ void SelectionDAGBuilder::visitVPStore(
SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
- MemoryLocation::UnknownSize, *Alignment, AAInfo);
+ LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
ST = DAG.getStoreVP(getMemoryRoot(), DL, OpValues[0], Ptr, Offset,
OpValues[2], OpValues[3], VT, MMO, ISD::UNINDEXED,
/* IsTruncating */ false, /*IsCompressing*/ false);
@@ -8174,7 +8171,7 @@ void SelectionDAGBuilder::visitVPScatter(
PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(AS), MachineMemOperand::MOStore,
- MemoryLocation::UnknownSize, *Alignment, AAInfo);
+ LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
SDValue Base, Index, Scale;
ISD::MemIndexType IndexType;
bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
@@ -8217,7 +8214,7 @@ void SelectionDAGBuilder::visitVPStridedLoad(
unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(AS), MachineMemOperand::MOLoad,
- MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
+ LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
SDValue LD = DAG.getStridedLoadVP(VT, DL, InChain, OpValues[0], OpValues[1],
OpValues[2], OpValues[3], MMO,
@@ -8240,7 +8237,7 @@ void SelectionDAGBuilder::visitVPStridedStore(
unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(AS), MachineMemOperand::MOStore,
- MemoryLocation::UnknownSize, *Alignment, AAInfo);
+ LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
SDValue ST = DAG.getStridedStoreVP(
getMemoryRoot(), DL, OpValues[0], OpValues[1],
diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp
index 5b02c1bc39c0a7..c0e8d085346103 100644
--- a/llvm/lib/CodeGen/TargetInstrInfo.cpp
+++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp
@@ -1554,7 +1554,7 @@ TargetInstrInfo::describeLoadedValue(const MachineInstr &MI,
SmallVector<uint64_t, 8> Ops;
DIExpression::appendOffset(Ops, Offset);
Ops.push_back(dwarf::DW_OP_deref_size);
- Ops.push_back(MMO->getSize());
+ Ops.push_back(MMO->getSize().getValue());
Expr = DIExpression::prependOpcodes(Expr, Ops);
return ParamLoadedValue(*BaseOp, Expr);
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 5df691f35275df..6376e07060147b 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -2687,10 +2687,7 @@ bool AArch64InstrInfo::getMemOperandsWithOffsetWidth(
return false;
// The maximum vscale is 16 under AArch64, return the maximal extent for the
// vector.
- Width = WidthN.isScalable()
- ? WidthN.getKnownMinValue() * AArch64::SVEMaxBitsPerVector /
- AArch64::SVEBitsPerBlock
- : WidthN.getKnownMinValue();
+ Width = LocationSize::precise(WidthN);
BaseOps.push_back(BaseOp);
return true;
}
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 0f3c3cb96e6ce3..7dee214d7358be 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -2852,8 +2852,8 @@ bool AArch64InstructionSelector::select(MachineInstr &I) {
return false;
}
- uint64_t MemSizeInBytes = LdSt.getMemSize();
- unsigned MemSizeInBits = LdSt.getMemSizeInBits();
+ uint64_t MemSizeInBytes = LdSt.getMemSize().getValue();
+ unsigned MemSizeInBits = LdSt.getMemSizeInBits().getValue();
AtomicOrdering Order = LdSt.getMMO().getSuccessOrdering();
// Need special instructions for atomics that affect ordering.
@@ -3276,7 +3276,7 @@ bool AArch64InstructionSelector::select(MachineInstr &I) {
RBI.getRegBank(SrcReg, MRI, TRI)->getID() == AArch64::GPRRegBankID;
if (LoadMI && IsGPR) {
const MachineMemOperand *MemOp = *LoadMI->memoperands_begin();
- unsigned BytesLoaded = MemOp->getSize();
+ unsigned BytesLoaded = MemOp->getSize().getValue();
if (BytesLoaded < 4 && SrcTy.getSizeInBytes() == BytesLoaded)
return selectCopy(I, TII, MRI, TRI, RBI);
}
@@ -7688,7 +7688,7 @@ bool AArch64InstructionSelector::isLoadStoreOfNumBytes(
return false;
assert(MI.hasOneMemOperand() &&
"Expected load/store to have only one mem op!");
- return (*MI.memoperands_begin())->getSize() == NumBytes;
+ return (*MI.memoperands_begin())->getSize().getValue() == NumBytes;
}
bool AArch64InstructionSelector::isDef32(const MachineInstr &MI) const {
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
index 51c52aad359497..967ac79744b4a8 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
@@ -311,7 +311,7 @@ bool matchSplitStoreZero128(MachineInstr &MI, MachineRegisterInfo &MRI) {
LLT ValTy = MRI.getType(Store.getValueReg());
if (!ValTy.isVector() || ValTy.getSizeInBits() != 128)
return false;
- if (ValTy.getSizeInBits() != Store.getMemSizeInBits())
+ if (ValTy.getSizeInBits() != Store.getMemSizeInBits().getValue())
return false; // Don't split truncating stores.
if (!MRI.hasOneNonDBGUse(Store.getValueReg()))
return false;
@@ -658,7 +658,7 @@ bool AArch64PostLegalizerCombiner::optimizeConsecutiveMemOpAddressing(
APInt Offset;
LLT StoredValTy = MRI.getType(St->getValueReg());
unsigned ValSize = StoredValTy.getSizeInBits();
- if (ValSize < 32 || ValSize != St->getMMO().getSizeInBits())
+ if (ValSize < 32 || ValSize != St->getMMO().getSizeInBits().getValue())
continue;
Register PtrReg = St->getPointerReg();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 0e8ff46c66b0fb..0095609f55bc2e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -3550,7 +3550,7 @@ bool AMDGPUDAGToDAGISel::isUniformLoad(const SDNode *N) const {
if (N->isDivergent() && !AMDGPUInstrInfo::isUniformMMO(MMO))
return false;
- return Ld->getAlign() >= Align(std::min(MMO->getSize().getKnownMinValue(), uint64_t(4))) &&
+ return Ld->getAlign() >= Align(std::min(MMO->getSize().getValue().getKnownMinValue(), uint64_t(4))) &&
((Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) ||
(Subtarget->getScalarizeGlobalBehavior() &&
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 0029c51231f286..90872516dd6db1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -5661,7 +5661,7 @@ bool AMDGPULegalizerInfo::legalizeBufferStore(MachineInstr &MI,
Register RSrc = MI.getOperand(2).getReg();
MachineMemOperand *MMO = *MI.memoperands_begin();
- const int MemSize = MMO->getSize();
+ const int MemSize = MMO->getSize().getValue();
unsigned ImmOffset;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index b174d57bd57656..0037825ce08938 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -449,7 +449,7 @@ bool AMDGPURegisterBankInfo::isScalarLoadLegal(const MachineInstr &MI) const {
const unsigned AS = MMO->getAddrSpace();
const bool IsConst = AS == AMDGPUAS::CONSTANT_ADDRESS ||
AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
- const unsigned MemSize = 8 * MMO->getSize();
+ const unsigned MemSize = 8 * MMO->getSize().getValue();
// Require 4-byte alignment.
return (MMO->getAlign() >= Align(4) ||
@@ -1070,7 +1070,7 @@ bool AMDGPURegisterBankInfo::applyMappingLoad(
return false;
MachineMemOperand *MMO = *MI.memoperands_begin();
- const unsigned MemSize = 8 * MMO->getSize();
+ const unsigned MemSize = 8 * MMO->getSize().getValue();
// Scalar loads of size 8 or 16 bit with proper alignment may be widened to
// 32 bit. Check to see if we need to widen the memory access, 8 or 16 bit
// scalar loads should have a load size of 32 but memory access size of less
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index ca2c2d87009f2f..e0ebcf0b0b19c5 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -422,9 +422,10 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
if (DataOpIdx == -1) {
DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
- Width = getOpSize(LdSt, DataOpIdx);
+ unsigned Width2 = getOpSize(LdSt, DataOpIdx);
DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data1);
- Width = Width.getValue() + getOpSize(LdSt, DataOpIdx);
+ Width2 += getOpSize(LdSt, DataOpIdx);
+ Width = LocationSize::precise(Width2);
} else {
Width = getOpSize(LdSt, DataOpIdx);
}
@@ -3635,12 +3636,13 @@ memOpsHaveSameBaseOperands(ArrayRef<const MachineOperand *> BaseOps1,
return true;
}
-static bool offsetsDoNotOverlap(int WidthA, int OffsetA,
- int WidthB, int OffsetB) {
+static bool offsetsDoNotOverlap(LocationSize WidthA, int OffsetA,
+ LocationSize WidthB, int OffsetB) {
int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
- int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
- return LowOffset + LowWidth <= HighOffset;
+ LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
+ return LowWidth.hasValue() &&
+ LowOffset + (int)LowWidth.getValue() <= HighOffset;
}
bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
@@ -3662,8 +3664,8 @@ bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
// FIXME: Handle ds_read2 / ds_write2.
return false;
}
- unsigned Width0 = MIa.memoperands().front()->getSize();
- unsigned Width1 = MIb.memoperands().front()->getSize();
+ LocationSize Width0 = MIa.memoperands().front()->getSize();
+ LocationSize Width1 = MIb.memoperands().front()->getSize();
return offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1);
}
diff --git a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
index 9c85ff3c43e22f..4ddee2f6d5befa 100644
--- a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
@@ -862,7 +862,7 @@ SILoadStoreOptimizer::combineKnownAdjacentMMOs(const CombineInfo &CI,
const MachineMemOperand *MMOa = *CI.I->memoperands_begin();
const MachineMemOperand *MMOb = *Paired.I->memoperands_begin();
- unsigned Size = MMOa->getSize() + MMOb->getSize();
+ unsigned Size = MMOa->getSize().getValue() + MMOb->getSize().getValue();
// A base pointer for the combined operation is the same as the leading
// operation's pointer.
diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
index dd63ca17e5b9f1..5d0468948dfb61 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -3710,7 +3710,7 @@ unsigned ARMBaseInstrInfo::getNumLDMAddresses(const MachineInstr &MI) const {
for (MachineInstr::mmo_iterator I = MI.memoperands_begin(),
E = MI.memoperands_end();
I != E; ++I) {
- Size += (*I)->getSize();
+ Size += (*I)->getSize().getValue();
}
// FIXME: The scheduler currently can't handle values larger than 16. But
// the values can actually go up to 32 for floating-point load/store
diff --git a/llvm/lib/Target/ARM/ARMHazardRecognizer.cpp b/llvm/lib/Target/ARM/ARMHazardRecognizer.cpp
index 9b26aac6c0b71e..34b6f0575f7276 100644
--- a/llvm/lib/Target/ARM/ARMHazardRecognizer.cpp
+++ b/llvm/lib/Target/ARM/ARMHazardRecognizer.cpp
@@ -191,7 +191,7 @@ ARMBankConflictHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
auto BasePseudoVal0 = MO0->getPseudoValue();
int64_t Offset0 = 0;
- if (MO0->getSize() > 4)
+ if (!MO0->getSize().hasValue() || MO0->getSize().getValue() > 4)
return NoHazard;
bool SPvalid = false;
@@ -259,8 +259,8 @@ void ARMBankConflictHazardRecognizer::EmitInstruction(SUnit *SU) {
return;
auto MO = *MI.memoperands().begin();
- uint64_t Size1 = MO->getSize();
- if (Size1 > 4)
+ LocationSize Size1 = MO->getSize();
+ if (Size1.hasValue() && Size1.getValue() > 4)
return;
Accesses.push_back(&MI);
}
diff --git a/llvm/lib/Target/BPF/BPFISelDAGToDAG.cpp b/llvm/lib/Target/BPF/BPFISelDAGToDAG.cpp
index d8139958e9fcf2..7b8bcb2c5866da 100644
--- a/llvm/lib/Target/BPF/BPFISelDAGToDAG.cpp
+++ b/llvm/lib/Target/BPF/BPFISelDAGToDAG.cpp
@@ -242,7 +242,9 @@ void BPFDAGToDAGISel::PreprocessLoad(SDNode *Node,
bool to_replace = false;
SDLoc DL(Node);
const LoadSDNode *LD = cast<LoadSDNode>(Node);
- uint64_t size = LD->getMemOperand()->getSize();
+ if (!LD->getMemOperand()->getSize().hasValue())
+ return;
+ uint64_t size = LD->getMemOperand()->getSize().getValue();
if (!size || size > 8 || (size & (size - 1)) || !LD->isSimple())
return;
diff --git a/llvm/lib/Target/Hexagon/HexagonStoreWidening.cpp b/llvm/lib/Target/Hexagon/HexagonStoreWidening.cpp
index f3facbe511a2f7..9d8e5c53b8227a 100644
--- a/llvm/lib/Target/Hexagon/HexagonStoreWidening.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonStoreWidening.cpp
@@ -172,16 +172,14 @@ bool HexagonStoreWidening::instrAliased(InstrGroup &Stores,
if (!MMO.getValue())
return true;
- MemoryLocation L(MMO.getValue(), MMO.getSize().getFixedValue(),
- MMO.getAAInfo());
+ MemoryLocation L(MMO.getValue(), MMO.getSize(), MMO.getAAInfo());
for (auto *SI : Stores) {
const MachineMemOperand &SMO = getStoreTarget(SI);
if (!SMO.getValue())
return true;
- MemoryLocation SL(SMO.getValue(), SMO.getSize().getFixedValue(),
- SMO.getAAInfo());
+ MemoryLocation SL(SMO.getValue(), SMO.getSize(), SMO.getAAInfo());
if (!AA->isNoAlias(L, SL))
return true;
}
@@ -294,8 +292,8 @@ bool HexagonStoreWidening::storesAreAdjacent(const MachineInstr *S1,
int Off1 = S1->getOperand(1).getImm();
int Off2 = S2->getOperand(1).getImm();
- return (Off1 >= 0) ? Off1+S1MO.getSize() == unsigned(Off2)
- : int(Off1+S1MO.getSize()) == Off2;
+ return (Off1 >= 0) ? Off1 + S1MO.getSize().getValue() == unsigned(Off2)
+ : int(Off1 + S1MO.getSize().getValue()) == Off2;
}
/// Given a sequence of adjacent stores, and a maximum size of a single wide
@@ -317,7 +315,7 @@ bool HexagonStoreWidening::selectStores(InstrGroup::iterator Begin,
assert(!FirstMI->memoperands_empty() && "Expecting some memory operands");
const MachineMemOperand &FirstMMO = getStoreTarget(FirstMI);
unsigned Alignment = FirstMMO.getAlign().value();
- unsigned SizeAccum = FirstMMO.getSize();
+ unsigned SizeAccum = FirstMMO.getSize().getValue();
unsigned FirstOffset = getStoreOffset(FirstMI);
// The initial value of SizeAccum should always be a power of 2.
@@ -359,7 +357,7 @@ bool HexagonStoreWidening::selectStores(InstrGroup::iterator Begin,
if (!storesAreAdjacent(S1, S2))
break;
- unsigned S2Size = getStoreTarget(S2).getSize();
+ unsigned S2Size = getStoreTarget(S2).getSize().getValue();
if (SizeAccum + S2Size > std::min(MaxSize, Alignment))
break;
@@ -407,7 +405,7 @@ bool HexagonStoreWidening::createWideStores(InstrGroup &OG, InstrGroup &NG,
MachineOperand &SO = MI->getOperand(2); // Source.
assert(SO.isImm() && "Expecting an immediate operand");
- unsigned NBits = MMO.getSize()*8;
+ unsigned NBits = MMO.getSize().getValue() * 8;
unsigned Mask = (0xFFFFFFFFU >> (32-NBits));
unsigned Val = (SO.getImm() & Mask) << Shift;
Acc |= Val;
diff --git a/llvm/lib/Target/Mips/MipsInstructionSelector.cpp b/llvm/lib/Target/Mips/MipsInstructionSelector.cpp
index 654f29d08af0b5..ccc36f9c30e717 100644
--- a/llvm/lib/Target/Mips/MipsInstructionSelector.cpp
+++ b/llvm/lib/Target/Mips/MipsInstructionSelector.cpp
@@ -184,7 +184,8 @@ MipsInstructionSelector::selectLoadStoreOpCode(MachineInstr &I,
const Register ValueReg = I.getOperand(0).getReg();
const LLT Ty = MRI.getType(ValueReg);
const unsigned TySize = Ty.getSizeInBits();
- const unsigned MemSizeInBytes = (*I.memoperands_begin())->getSize();
+ const unsigned MemSizeInBytes =
+ (*I.memoperands_begin())->getSize().getValue();
unsigned Opc = I.getOpcode();
const bool isStore = Opc == TargetOpcode::G_STORE;
@@ -455,7 +456,7 @@ bool MipsInstructionSelector::select(MachineInstr &I) {
}
// Unaligned memory access
- if (MMO->getAlign() < MMO->getSize() &&
+ if ((!MMO->getSize().hasValue() || MMO->getAlign() < MMO->getSize().getValue()) &&
!STI.systemSupportsUnalignedAccess()) {
if (MMO->getSize() != 4 || !isRegInGprb(I.getOperand(0).getReg(), MRI))
return false;
diff --git a/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp b/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp
index f5e94235859a06..3307e840a2afd6 100644
--- a/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp
+++ b/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp
@@ -344,7 +344,7 @@ bool MipsLegalizerInfo::legalizeCustom(
switch (MI.getOpcode()) {
case G_LOAD:
case G_STORE: {
- unsigned MemSize = (**MI.memoperands_begin()).getSize();
+ unsigned MemSize = (**MI.memoperands_begin()).getSize().getValue();
Register Val = MI.getOperand(0).getReg();
unsigned Size = MRI.getType(Val).getSizeInBits();
diff --git a/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp b/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
index acf0d6312ef51b..26926d38b290c7 100644
--- a/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
@@ -70,9 +70,9 @@ class MipsPreLegalizerCombinerImpl : public Combiner {
// subtarget doesn't support them.
auto MMO = *MI.memoperands_begin();
const MipsSubtarget &STI = MI.getMF()->getSubtarget<MipsSubtarget>();
- if (!isPowerOf2_64(MMO->getSize()))
+ if (!MMO->getSize().hasValue() || !isPowerOf2_64(MMO->getSize().getValue()))
return false;
- bool isUnaligned = MMO->getAlign() < MMO->getSize();
+ bool isUnaligned = MMO->getAlign() < MMO->getSize().getValue();
if (!STI.systemSupportsUnalignedAccess() && isUnaligned)
return false;
diff --git a/llvm/lib/Target/Mips/MipsRegisterBankInfo.cpp b/llvm/lib/Target/Mips/MipsRegisterBankInfo.cpp
index db7afc3c86c57e..6af1fd8c88e570 100644
--- a/llvm/lib/Target/Mips/MipsRegisterBankInfo.cpp
+++ b/llvm/lib/Target/Mips/MipsRegisterBankInfo.cpp
@@ -155,7 +155,8 @@ static bool isGprbTwoInstrUnalignedLoadOrStore(const MachineInstr *MI) {
auto MMO = *MI->memoperands_begin();
const MipsSubtarget &STI = MI->getMF()->getSubtarget<MipsSubtarget>();
if (MMO->getSize() == 4 && (!STI.systemSupportsUnalignedAccess() &&
- MMO->getAlign() < MMO->getSize()))
+ (!MMO->getSize().hasValue() ||
+ MMO->getAlign() < MMO->getSize().getValue())))
return true;
}
return false;
diff --git a/llvm/lib/Target/PowerPC/GISel/PPCInstructionSelector.cpp b/llvm/lib/Target/PowerPC/GISel/PPCInstructionSelector.cpp
index 3fd7a1ad9efa59..fdffbcd71d0490 100644
--- a/llvm/lib/Target/PowerPC/GISel/PPCInstructionSelector.cpp
+++ b/llvm/lib/Target/PowerPC/GISel/PPCInstructionSelector.cpp
@@ -739,7 +739,7 @@ bool PPCInstructionSelector::select(MachineInstr &I) {
auto SelectLoadStoreAddressingMode = [&]() -> MachineInstr * {
const unsigned NewOpc = selectLoadStoreOp(
I.getOpcode(), RBI.getRegBank(LdSt.getReg(0), MRI, TRI)->getID(),
- LdSt.getMemSizeInBits());
+ LdSt.getMemSizeInBits().getValue());
if (NewOpc == I.getOpcode())
return nullptr;
diff --git a/llvm/lib/Target/PowerPC/PPCHazardRecognizers.cpp b/llvm/lib/Target/PowerPC/PPCHazardRecognizers.cpp
index ffaa3e05c84790..44549d08ec8825 100644
--- a/llvm/lib/Target/PowerPC/PPCHazardRecognizers.cpp
+++ b/llvm/lib/Target/PowerPC/PPCHazardRecognizers.cpp
@@ -374,8 +374,8 @@ getHazardType(SUnit *SU, int Stalls) {
// overlapping address.
if (isLoad && NumStores && !MI->memoperands_empty()) {
MachineMemOperand *MO = *MI->memoperands_begin();
- if (isLoadOfStoredAddress(MO->getSize(),
- MO->getOffset(), MO->getValue()))
+ if (isLoadOfStoredAddress(MO->getSize().getValue(), MO->getOffset(),
+ MO->getValue()))
return NoopHazard;
}
@@ -401,7 +401,7 @@ void PPCHazardRecognizer970::EmitInstruction(SUnit *SU) {
// Track the address stored to.
if (isStore && NumStores < 4 && !MI->memoperands_empty()) {
MachineMemOperand *MO = *MI->memoperands_begin();
- StoreSize[NumStores] = MO->getSize();
+ StoreSize[NumStores] = MO->getSize().getValue();
StoreOffset[NumStores] = MO->getOffset();
StoreValue[NumStores] = MO->getValue();
++NumStores;
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 68c80dd9aa5c76..10dd3d731fe01a 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -8520,7 +8520,7 @@ bool PPCTargetLowering::directMoveIsProfitable(const SDValue &Op) const {
// If there is no LXSIBZX/LXSIHZX, like Power8,
// prefer direct move if the memory size is 1 or 2 bytes.
MachineMemOperand *MMO = cast<LoadSDNode>(Origin)->getMemOperand();
- if (!Subtarget.hasP9Vector() && MMO->getSize() <= 2)
+ if (!Subtarget.hasP9Vector() && MMO->getSize().getValue() <= 2)
return true;
for (SDNode::use_iterator UI = Origin->use_begin(),
@@ -15115,7 +15115,7 @@ SDValue PPCTargetLowering::expandVSXLoadForLE(SDNode *N,
// If the MMO suggests this isn't a load of a full vector, leave
// things alone. For a built-in, we have to make the change for
// correctness, so if there is a size problem that will be a bug.
- if (MMO->getSize() < 16)
+ if (MMO->getSize().getValue() < 16)
return SDValue();
break;
}
@@ -15183,7 +15183,7 @@ SDValue PPCTargetLowering::expandVSXStoreForLE(SDNode *N,
// If the MMO suggests this isn't a store of a full vector, leave
// things alone. For a built-in, we have to make the change for
// correctness, so if there is a size problem that will be a bug.
- if (MMO->getSize() < 16)
+ if (MMO->getSize().getValue() < 16)
return SDValue();
break;
}
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 37a8079dcbf10d..4ca300f9151e1a 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -656,7 +656,7 @@ void RISCVInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
if (IsScalableVector) {
MachineMemOperand *MMO = MF->getMachineMemOperand(
MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore,
- MemoryLocation::UnknownSize, MFI.getObjectAlign(FI));
+ LocationSize::beforeOrAfterPointer(), MFI.getObjectAlign(FI));
MFI.setStackID(FI, TargetStackID::ScalableVector);
BuildMI(MBB, I, DebugLoc(), get(Opcode))
@@ -739,7 +739,7 @@ void RISCVInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
if (IsScalableVector) {
MachineMemOperand *MMO = MF->getMachineMemOperand(
MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOLoad,
- MemoryLocation::UnknownSize, MFI.getObjectAlign(FI));
+ LocationSize::beforeOrAfterPointer(), MFI.getObjectAlign(FI));
MFI.setStackID(FI, TargetStackID::ScalableVector);
BuildMI(MBB, I, DebugLoc(), get(Opcode), DstReg)
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
index 046a12208467b4..72e5882cc61f06 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
@@ -2020,11 +2020,12 @@ areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
}
if (SameVal) {
int OffsetA = MMOa->getOffset(), OffsetB = MMOb->getOffset();
- int WidthA = MMOa->getSize(), WidthB = MMOb->getSize();
+ LocationSize WidthA = MMOa->getSize(), WidthB = MMOb->getSize();
int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
- int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
- if (LowOffset + LowWidth <= HighOffset)
+ LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
+ if (LowWidth.hasValue() &&
+ LowOffset + (int)LowWidth.getValue() <= HighOffset)
return true;
}
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblySetP2AlignOperands.cpp b/llvm/lib/Target/WebAssembly/WebAssemblySetP2AlignOperands.cpp
index a453e7388e274f..5f5f3a58da3341 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblySetP2AlignOperands.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblySetP2AlignOperands.cpp
@@ -59,7 +59,7 @@ static void rewriteP2Align(MachineInstr &MI, unsigned OperandNo) {
"ISel should set p2align operands to 0");
assert(MI.hasOneMemOperand() &&
"Load and store instructions have exactly one mem operand");
- assert((*MI.memoperands_begin())->getSize() ==
+ assert((*MI.memoperands_begin())->getSize().getValue() ==
(UINT64_C(1) << WebAssembly::GetDefaultP2Align(MI.getOpcode())) &&
"Default p2align value should be natural");
assert(MI.getDesc().operands()[OperandNo].OperandType ==
diff --git a/llvm/lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp b/llvm/lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp
index 04931afdec51c3..a72eeb53915d65 100644
--- a/llvm/lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp
+++ b/llvm/lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp
@@ -521,8 +521,8 @@ bool X86AvoidSFBPass::alias(const MachineMemOperand &Op1,
return true;
int64_t MinOffset = std::min(Op1.getOffset(), Op2.getOffset());
- int64_t Overlapa = Op1.getSize() + Op1.getOffset() - MinOffset;
- int64_t Overlapb = Op2.getSize() + Op2.getOffset() - MinOffset;
+ int64_t Overlapa = Op1.getSize().getValue() + Op1.getOffset() - MinOffset;
+ int64_t Overlapb = Op2.getSize().getValue() + Op2.getOffset() - MinOffset;
return !AA->isNoAlias(
MemoryLocation(Op1.getValue(), Overlapa, Op1.getAAInfo()),
@@ -688,7 +688,7 @@ bool X86AvoidSFBPass::runOnMachineFunction(MachineFunction &MF) {
!isRelevantAddressingMode(PBInst) || !PBInst->hasOneMemOperand())
continue;
int64_t PBstDispImm = getDispOperand(PBInst).getImm();
- unsigned PBstSize = (*PBInst->memoperands_begin())->getSize();
+ unsigned PBstSize = (*PBInst->memoperands_begin())->getSize().getValue();
// This check doesn't cover all cases, but it will suffice for now.
// TODO: take branch probability into consideration, if the blocking
// store is in an unreached block, breaking the memcopy could lose
diff --git a/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll b/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
index 39c546e59afbf5..d4d803a91cfa14 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
@@ -3,7 +3,7 @@
define void @UphPNR(target("aarch64.svcount") %predcnt) {
entry:
; CHECK: %0:ppr = COPY $p0
-; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr, vscale )
+; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr)
; CHECK: %1:pnr_p8to15 = COPY %0
; CHECK: INLINEASM &"ld1w {z0.s,z1.s,z2.s,z3.s}, $0/z, [x10]", 1 /* sideeffect attdialect */, {{[0-9]+}} /* reguse:PNR_p8to15 */, %1
; CHECK: RET_ReallyLR
@@ -17,7 +17,7 @@ entry:
define void @UpaPNR(target("aarch64.svcount") %predcnt) {
entry:
; CHECK: %0:ppr = COPY $p0
-; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr, vscale )
+; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr)
; CHECK: %1:pnr = COPY %0
; CHECK: INLINEASM &"ld1w {z0.s,z1.s,z2.s,z3.s}, $0/z, [x10]", 1 /* sideeffect attdialect */, {{[0-9]+}} /* reguse:PNR */, %1
; CHECK: RET_ReallyLR
@@ -31,7 +31,7 @@ entry:
define void @UplPNR(target("aarch64.svcount") %predcnt) {
entry:
; CHECK: %0:ppr = COPY $p0
-; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr, vscale )
+; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr)
; CHECK: %1:pnr_3b = COPY %0
; CHECK: INLINEASM &"fadd z0.h, $0/m, z0.h, #0.5", 1 /* sideeffect attdialect */, {{[0-9]+}} /* reguse:PNR_3b */, %1
; CHECK: RET_ReallyLR
diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
index 05814a0b3f5e70..52bd15742ef4bb 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
@@ -17,7 +17,7 @@ define void @vpmerge_vpload_store(<vscale x 2 x i32> %passthru, ptr %p, <vscale
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vrnov0 = COPY $v8
; CHECK-NEXT: $v0 = COPY [[COPY1]]
; CHECK-NEXT: [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], $v0, [[COPY]], 5 /* e32 */, 0 /* tu, mu */ :: (load unknown-size from %ir.p, align 8)
- ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store (<vscale x 1 x s64>) into %ir.p, vscale )
+ ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store (<vscale x 1 x s64>) into %ir.p)
; CHECK-NEXT: PseudoRET
%splat = insertelement <vscale x 2 x i1> poison, i1 -1, i32 0
%mask = shufflevector <vscale x 2 x i1> %splat, <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
@@ -38,7 +38,7 @@ define void @vpselect_vpload_store(<vscale x 2 x i32> %passthru, ptr %p, <vscale
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vrnov0 = COPY $v8
; CHECK-NEXT: $v0 = COPY [[COPY1]]
; CHECK-NEXT: [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], $v0, [[COPY]], 5 /* e32 */, 1 /* ta, mu */ :: (load unknown-size from %ir.p, align 8)
- ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store (<vscale x 1 x s64>) into %ir.p, vscale )
+ ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store (<vscale x 1 x s64>) into %ir.p)
; CHECK-NEXT: PseudoRET
%splat = insertelement <vscale x 2 x i1> poison, i1 -1, i32 0
%mask = shufflevector <vscale x 2 x i1> %splat, <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
>From 71f3861546a60379e5e84752d7db937150a2d8b9 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Fri, 8 Mar 2024 10:58:45 +0000
Subject: [PATCH 4/4] Formatting and fixups
---
llvm/include/llvm/CodeGen/MachineFunction.h | 37 +++++--------------
llvm/lib/CodeGen/MachineFunction.cpp | 16 ++++++--
llvm/lib/CodeGen/MachineInstr.cpp | 4 +-
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 11 +++---
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 4 +-
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 12 +++---
.../SelectionDAG/SelectionDAGBuilder.cpp | 24 +++++++-----
.../AArch64/AArch64LoadStoreOptimizer.cpp | 2 +-
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 4 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 5 +--
.../Target/Mips/MipsInstructionSelector.cpp | 3 +-
.../Target/Mips/MipsPreLegalizerCombiner.cpp | 3 +-
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 22 +++++++++--
.../alloca-load-store-scalable-array.ll | 36 +++++++++---------
.../alloca-load-store-scalable-struct.ll | 12 +++---
.../rvv/alloca-load-store-scalable-array.ll | 12 +++---
.../rvv/alloca-load-store-scalable-struct.ll | 8 ++--
17 files changed, 114 insertions(+), 101 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index 2f17b5d5ac71a8..c2bff279449398 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -1032,28 +1032,18 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
MachineMemOperand *getMachineMemOperand(
- MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LocationSize TS,
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LocationSize Size,
Align BaseAlignment, const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
MachineMemOperand *getMachineMemOperand(
- MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, uint64_t TS,
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, uint64_t Size,
Align BaseAlignment, const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic) {
- return getMachineMemOperand(PtrInfo, F, LocationSize::precise(TS),
- BaseAlignment, AAInfo, Ranges, SSID, Ordering,
- FailureOrdering);
- }
- MachineMemOperand *getMachineMemOperand(
- MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, TypeSize TS,
- Align BaseAlignment, const AAMDNodes &AAInfo = AAMDNodes(),
- const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
- AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
- AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic) {
- return getMachineMemOperand(PtrInfo, F, LocationSize::precise(TS),
+ return getMachineMemOperand(PtrInfo, F, LocationSize::precise(Size),
BaseAlignment, AAInfo, Ranges, SSID, Ordering,
FailureOrdering);
}
@@ -1065,22 +1055,18 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
int64_t Offset, LLT Ty);
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
- int64_t Offset, LocationSize TS) {
+ int64_t Offset, LocationSize Size) {
return getMachineMemOperand(
MMO, Offset,
- !TS.hasValue() ? LLT()
- : TS.isScalable()
- ? LLT::scalable_vector(1, 8 * TS.getValue().getKnownMinValue())
- : LLT::scalar(8 * TS.getValue().getKnownMinValue()));
+ !Size.hasValue() ? LLT()
+ : Size.isScalable()
+ ? LLT::scalable_vector(1, 8 * Size.getValue().getKnownMinValue())
+ : LLT::scalar(8 * Size.getValue().getKnownMinValue()));
}
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
int64_t Offset, uint64_t Size) {
return getMachineMemOperand(MMO, Offset, LocationSize::precise(Size));
}
- MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
- int64_t Offset, TypeSize Size) {
- return getMachineMemOperand(MMO, Offset, LocationSize::precise(Size));
- }
/// getMachineMemOperand - Allocate a new MachineMemOperand by copying
/// an existing one, replacing only the MachinePointerInfo and size.
@@ -1088,7 +1074,7 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
/// explicitly deallocated.
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
- LocationSize TS);
+ LocationSize Size);
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
LLT Ty);
@@ -1097,11 +1083,6 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
uint64_t Size) {
return getMachineMemOperand(MMO, PtrInfo, LocationSize::precise(Size));
}
- MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
- const MachinePointerInfo &PtrInfo,
- TypeSize Size) {
- return getMachineMemOperand(MMO, PtrInfo, LocationSize::precise(Size));
- }
/// Allocate a new MachineMemOperand by copying an existing one,
/// replacing only AliasAnalysis information. MachineMemOperands are owned
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index 3e16e9db57e1cb..ad532149926670 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -484,12 +484,16 @@ void MachineFunction::deleteMachineBasicBlock(MachineBasicBlock *MBB) {
}
MachineMemOperand *MachineFunction::getMachineMemOperand(
- MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LocationSize TS,
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LocationSize Size,
Align BaseAlignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
SyncScope::ID SSID, AtomicOrdering Ordering,
AtomicOrdering FailureOrdering) {
+ assert((!Size.hasValue() ||
+ Size.getValue().getKnownMinValue() != ~UINT64_C(0)) &&
+ "Unexpected an unknown size to be represented using "
+ "LocationSize::beforeOrAfter()");
return new (Allocator)
- MachineMemOperand(PtrInfo, F, TS, BaseAlignment, AAInfo, Ranges, SSID,
+ MachineMemOperand(PtrInfo, F, Size, BaseAlignment, AAInfo, Ranges, SSID,
Ordering, FailureOrdering);
}
@@ -506,9 +510,13 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
MachineMemOperand *
MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
- LocationSize TS) {
+ LocationSize Size) {
+ assert((!Size.hasValue() ||
+ Size.getValue().getKnownMinValue() != ~UINT64_C(0)) &&
+ "Unexpected an unknown size to be represented using "
+ "LocationSize::beforeOrAfter()");
return new (Allocator)
- MachineMemOperand(PtrInfo, MMO->getFlags(), TS, MMO->getBaseAlign(),
+ MachineMemOperand(PtrInfo, MMO->getFlags(), Size, MMO->getBaseAlign(),
AAMDNodes(), nullptr, MMO->getSyncScopeID(),
MMO->getSuccessOrdering(), MMO->getFailureOrdering());
}
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 8d6909abc5d6c4..8102bb971ba66e 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -1341,8 +1341,8 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
assert((OffsetA >= 0) && "Negative MachineMemOperand offset");
assert((OffsetB >= 0) && "Negative MachineMemOperand offset");
- // If Scalable Location Size has non-zero offset,
- // Width + Offset does not work at the moment
+ // If Scalable Location Size has non-zero offset, Width + Offset does not work
+ // at the moment
if ((WidthA.isScalable() && OffsetA > 0) ||
(WidthB.isScalable() && OffsetB > 0))
return true;
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index c8e081b86089a1..91a842be8ee6d9 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -24222,7 +24222,7 @@ static SDValue narrowExtractedVectorLoad(SDNode *Extract, SelectionDAG &DAG) {
// TODO: Use "BaseIndexOffset" to make this more effective.
SDValue NewAddr = DAG.getMemBasePlusOffset(Ld->getBasePtr(), Offset, DL);
- TypeSize StoreSize = VT.getStoreSize();
+ LocationSize StoreSize = LocationSize::precise(VT.getStoreSize());
MachineFunction &MF = DAG.getMachineFunction();
MachineMemOperand *MMO;
if (Offset.isScalable()) {
@@ -27863,11 +27863,10 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
if (const auto *LSN = dyn_cast<LSBaseSDNode>(N)) {
int64_t Offset = 0;
if (auto *C = dyn_cast<ConstantSDNode>(LSN->getOffset()))
- Offset = (LSN->getAddressingMode() == ISD::PRE_INC)
- ? C->getSExtValue()
- : (LSN->getAddressingMode() == ISD::PRE_DEC)
- ? -1 * C->getSExtValue()
- : 0;
+ Offset = (LSN->getAddressingMode() == ISD::PRE_INC) ? C->getSExtValue()
+ : (LSN->getAddressingMode() == ISD::PRE_DEC)
+ ? -1 * C->getSExtValue()
+ : 0;
TypeSize Size = LSN->getMemoryVT().getStoreSize();
return {LSN->isVolatile(), LSN->isAtomic(),
LSN->getBasePtr(), Offset /*base offset*/,
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 2dccc45c803a05..808e3c622033e0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -267,7 +267,9 @@ static MachineMemOperand *getStackAlignedMMO(SDValue StackPtr,
auto &MFI = MF.getFrameInfo();
int FI = cast<FrameIndexSDNode>(StackPtr)->getIndex();
MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
- uint64_t ObjectSize = isObjectScalable ? ~UINT64_C(0) : MFI.getObjectSize(FI);
+ LocationSize ObjectSize = isObjectScalable
+ ? LocationSize::beforeOrAfterPointer()
+ : LocationSize::precise(MFI.getObjectSize(FI));
return MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,
ObjectSize, MFI.getObjectAlign(FI));
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index b6acbaed5661b2..b0d3068d406d09 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -8555,7 +8555,7 @@ SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
if (PtrInfo.V.isNull())
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
- TypeSize Size = MemVT.getStoreSize();
+ LocationSize Size = LocationSize::precise(MemVT.getStoreSize());
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
Alignment, AAInfo, Ranges);
@@ -8676,7 +8676,7 @@ SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
MachineFunction &MF = getMachineFunction();
- TypeSize Size = Val.getValueType().getStoreSize();
+ LocationSize Size = LocationSize::precise(Val.getValueType().getStoreSize());
MachineMemOperand *MMO =
MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo);
return getStore(Chain, dl, Val, Ptr, MMO);
@@ -8729,7 +8729,8 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
- PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment, AAInfo);
+ PtrInfo, MMOFlags, LocationSize::precise(SVT.getStoreSize()), Alignment,
+ AAInfo);
return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
}
@@ -8823,7 +8824,7 @@ SDValue SelectionDAG::getLoadVP(
if (PtrInfo.V.isNull())
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
- TypeSize Size = MemVT.getStoreSize();
+ LocationSize Size = LocationSize::precise(MemVT.getStoreSize());
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
Alignment, AAInfo, Ranges);
@@ -8976,7 +8977,8 @@ SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
- PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment, AAInfo);
+ PtrInfo, MMOFlags, LocationSize::precise(SVT.getStoreSize()), Alignment,
+ AAInfo);
return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO,
IsCompressing);
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 5e646329ee57b4..50995a5e2483c8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3037,7 +3037,8 @@ static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,
auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
MachineMemOperand::MODereferenceable;
MachineMemOperand *MemRef = MF.getMachineMemOperand(
- MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy));
+ MPInfo, Flags, LocationSize::precise(PtrTy.getSizeInBits() / 8),
+ DAG.getEVTAlign(PtrTy));
DAG.setNodeMemRefs(Node, {MemRef});
}
if (PtrTy != PtrMemTy)
@@ -5000,9 +5001,9 @@ void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
MachineFunction &MF = DAG.getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
- MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
- DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, SuccessOrdering,
- FailureOrdering);
+ MachinePointerInfo(I.getPointerOperand()), Flags,
+ LocationSize::precise(MemVT.getStoreSize()), DAG.getEVTAlign(MemVT),
+ AAMDNodes(), nullptr, SSID, SuccessOrdering, FailureOrdering);
SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
dl, MemVT, VTs, InChain,
@@ -5054,8 +5055,9 @@ void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
MachineFunction &MF = DAG.getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
- MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
- DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, Ordering);
+ MachinePointerInfo(I.getPointerOperand()), Flags,
+ LocationSize::precise(MemVT.getStoreSize()), DAG.getEVTAlign(MemVT),
+ AAMDNodes(), nullptr, SSID, Ordering);
SDValue L =
DAG.getAtomic(NT, dl, MemVT, InChain,
@@ -5100,8 +5102,9 @@ void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
auto Flags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo);
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
- MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
- I.getAlign(), AAMDNodes(), nullptr, SSID, Order);
+ MachinePointerInfo(I.getPointerOperand()), Flags,
+ LocationSize::precise(MemVT.getStoreSize()), I.getAlign(), AAMDNodes(),
+ nullptr, SSID, Order);
InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
@@ -5137,8 +5140,9 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
MachineFunction &MF = DAG.getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
- MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
- I.getAlign(), AAMDNodes(), nullptr, SSID, Ordering);
+ MachinePointerInfo(I.getPointerOperand()), Flags,
+ LocationSize::precise(MemVT.getStoreSize()), I.getAlign(), AAMDNodes(),
+ nullptr, SSID, Ordering);
SDValue Val = getValue(I.getValueOperand());
if (Val.getValueType() != MemVT)
diff --git a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
index 0ab2c401b1749d..d0adb78b231a76 100644
--- a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
@@ -2362,7 +2362,7 @@ bool AArch64LoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) {
// Get the needed alignments to check them if
// ldp-aligned-only/stp-aligned-only features are opted.
uint64_t MemAlignment = MemOp->getAlign().value();
- uint64_t TypeAlignment = Align(MemOp->getSize()).value();
+ uint64_t TypeAlignment = Align(MemOp->getSize().getValue()).value();
if (MemAlignment < 2 * TypeAlignment) {
NumFailedAlignmentCheck++;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 0095609f55bc2e..0d43977f408298 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -3550,7 +3550,9 @@ bool AMDGPUDAGToDAGISel::isUniformLoad(const SDNode *N) const {
if (N->isDivergent() && !AMDGPUInstrInfo::isUniformMMO(MMO))
return false;
- return Ld->getAlign() >= Align(std::min(MMO->getSize().getValue().getKnownMinValue(), uint64_t(4))) &&
+ return Ld->getAlign() >=
+ Align(std::min(MMO->getSize().getValue().getKnownMinValue(),
+ uint64_t(4))) &&
((Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) ||
(Subtarget->getScalarizeGlobalBehavior() &&
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index e0ebcf0b0b19c5..295fa49f1fc56f 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -422,10 +422,9 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
if (DataOpIdx == -1) {
DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
- unsigned Width2 = getOpSize(LdSt, DataOpIdx);
+ Width = getOpSize(LdSt, DataOpIdx);
DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data1);
- Width2 += getOpSize(LdSt, DataOpIdx);
- Width = LocationSize::precise(Width2);
+ Width = Width.getValue() + getOpSize(LdSt, DataOpIdx);
} else {
Width = getOpSize(LdSt, DataOpIdx);
}
diff --git a/llvm/lib/Target/Mips/MipsInstructionSelector.cpp b/llvm/lib/Target/Mips/MipsInstructionSelector.cpp
index ccc36f9c30e717..4e1e27088cce42 100644
--- a/llvm/lib/Target/Mips/MipsInstructionSelector.cpp
+++ b/llvm/lib/Target/Mips/MipsInstructionSelector.cpp
@@ -456,7 +456,8 @@ bool MipsInstructionSelector::select(MachineInstr &I) {
}
// Unaligned memory access
- if ((!MMO->getSize().hasValue() || MMO->getAlign() < MMO->getSize().getValue()) &&
+ if ((!MMO->getSize().hasValue() ||
+ MMO->getAlign() < MMO->getSize().getValue()) &&
!STI.systemSupportsUnalignedAccess()) {
if (MMO->getSize() != 4 || !isRegInGprb(I.getOperand(0).getReg(), MRI))
return false;
diff --git a/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp b/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
index 26926d38b290c7..639e623954ffc6 100644
--- a/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
@@ -70,7 +70,8 @@ class MipsPreLegalizerCombinerImpl : public Combiner {
// subtarget doesn't support them.
auto MMO = *MI.memoperands_begin();
const MipsSubtarget &STI = MI.getMF()->getSubtarget<MipsSubtarget>();
- if (!MMO->getSize().hasValue() || !isPowerOf2_64(MMO->getSize().getValue()))
+ if (!MMO->getSize().hasValue() ||
+ !isPowerOf2_64(MMO->getSize().getValue()))
return false;
bool isUnaligned = MMO->getAlign() < MMO->getSize().getValue();
if (!STI.systemSupportsUnalignedAccess() && isUnaligned)
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 9b748cdcf74511..83c3551ffc02c0 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -10309,9 +10309,15 @@ RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(SDValue Op,
RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget);
if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() &&
getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
+ MachineMemOperand *MMO = Load->getMemOperand();
+ MachineFunction &MF = DAG.getMachineFunction();
+ MMO = MF.getMachineMemOperand(
+ MMO, MMO->getPointerInfo(),
+ MMO->getMemoryType().isValid()
+ ? LLT::scalable_vector(1, MMO->getMemoryType().getSizeInBits())
+ : MMO->getMemoryType());
SDValue NewLoad =
- DAG.getLoad(ContainerVT, DL, Load->getChain(), Load->getBasePtr(),
- Load->getMemOperand());
+ DAG.getLoad(ContainerVT, DL, Load->getChain(), Load->getBasePtr(), MMO);
SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
return DAG.getMergeValues({Result, NewLoad.getValue(1)}, DL);
}
@@ -10369,9 +10375,17 @@ RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(SDValue Op,
const auto [MinVLMAX, MaxVLMAX] =
RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget);
if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() &&
- getLMUL1VT(ContainerVT).bitsLE(ContainerVT))
+ getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
+ MachineMemOperand *MMO = Store->getMemOperand();
+ MachineFunction &MF = DAG.getMachineFunction();
+ MMO = MF.getMachineMemOperand(
+ MMO, MMO->getPointerInfo(),
+ MMO->getMemoryType().isValid()
+ ? LLT::scalable_vector(1, MMO->getMemoryType().getSizeInBits())
+ : MMO->getMemoryType());
return DAG.getStore(Store->getChain(), DL, NewValue, Store->getBasePtr(),
- Store->getMemOperand());
+ MMO);
+ }
SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG,
Subtarget);
diff --git a/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-array.ll b/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-array.ll
index 9a4e01a29ecb6d..7244ac949ab88c 100644
--- a/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-array.ll
+++ b/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-array.ll
@@ -14,12 +14,12 @@ define void @array_1D(ptr %addr) #0 {
; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #1, mul vl]
-; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0]
-; CHECK-NEXT: st1d { z0.d }, p0, [sp, #2, mul vl]
-; CHECK-NEXT: st1d { z1.d }, p0, [sp, #1, mul vl]
-; CHECK-NEXT: st1d { z2.d }, p0, [sp]
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #2, mul vl]
+; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT: st1d { z0.d }, p0, [sp]
+; CHECK-NEXT: st1d { z1.d }, p0, [sp, #2, mul vl]
+; CHECK-NEXT: st1d { z2.d }, p0, [sp, #1, mul vl]
; CHECK-NEXT: addvl sp, sp, #3
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
@@ -81,18 +81,18 @@ define void @array_2D(ptr %addr) #0 {
; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x30, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 48 * VG
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, #5, mul vl]
-; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #4, mul vl]
-; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0]
-; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0, #3, mul vl]
-; CHECK-NEXT: ld1d { z4.d }, p0/z, [x0, #1, mul vl]
-; CHECK-NEXT: ld1d { z5.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: st1d { z0.d }, p0, [sp, #5, mul vl]
-; CHECK-NEXT: st1d { z1.d }, p0, [sp, #4, mul vl]
-; CHECK-NEXT: st1d { z3.d }, p0, [sp, #3, mul vl]
-; CHECK-NEXT: st1d { z5.d }, p0, [sp, #2, mul vl]
-; CHECK-NEXT: st1d { z4.d }, p0, [sp, #1, mul vl]
-; CHECK-NEXT: st1d { z2.d }, p0, [sp]
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #5, mul vl]
+; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0, #4, mul vl]
+; CHECK-NEXT: ld1d { z4.d }, p0/z, [x0, #2, mul vl]
+; CHECK-NEXT: ld1d { z5.d }, p0/z, [x0, #3, mul vl]
+; CHECK-NEXT: st1d { z0.d }, p0, [sp]
+; CHECK-NEXT: st1d { z1.d }, p0, [sp, #5, mul vl]
+; CHECK-NEXT: st1d { z3.d }, p0, [sp, #4, mul vl]
+; CHECK-NEXT: st1d { z5.d }, p0, [sp, #3, mul vl]
+; CHECK-NEXT: st1d { z4.d }, p0, [sp, #2, mul vl]
+; CHECK-NEXT: st1d { z2.d }, p0, [sp, #1, mul vl]
; CHECK-NEXT: addvl sp, sp, #6
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-struct.ll b/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-struct.ll
index 7292d52aaf4765..f03a6f018d34d0 100644
--- a/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-struct.ll
+++ b/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-struct.ll
@@ -13,12 +13,12 @@ define void @test(ptr %addr) #0 {
; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #1, mul vl]
-; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0]
-; CHECK-NEXT: st1d { z0.d }, p0, [sp, #2, mul vl]
-; CHECK-NEXT: st1d { z1.d }, p0, [sp, #1, mul vl]
-; CHECK-NEXT: st1d { z2.d }, p0, [sp]
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #2, mul vl]
+; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT: st1d { z0.d }, p0, [sp]
+; CHECK-NEXT: st1d { z1.d }, p0, [sp, #2, mul vl]
+; CHECK-NEXT: st1d { z2.d }, p0, [sp, #1, mul vl]
; CHECK-NEXT: addvl sp, sp, #3
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-array.ll b/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-array.ll
index 1d025a2f776f82..1fe91c721f4dd2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-array.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-array.ll
@@ -18,15 +18,15 @@ define void @test(ptr %addr) {
; CHECK-NEXT: add a2, a0, a1
; CHECK-NEXT: vl1re64.v v8, (a2)
; CHECK-NEXT: slli a2, a1, 1
-; CHECK-NEXT: add a3, a0, a2
-; CHECK-NEXT: vl1re64.v v9, (a3)
+; CHECK-NEXT: vl1re64.v v9, (a0)
+; CHECK-NEXT: add a0, a0, a2
; CHECK-NEXT: vl1re64.v v10, (a0)
; CHECK-NEXT: addi a0, sp, 16
+; CHECK-NEXT: vs1r.v v9, (a0)
; CHECK-NEXT: add a2, a0, a2
-; CHECK-NEXT: vs1r.v v9, (a2)
-; CHECK-NEXT: add a1, a0, a1
-; CHECK-NEXT: vs1r.v v8, (a1)
-; CHECK-NEXT: vs1r.v v10, (a0)
+; CHECK-NEXT: vs1r.v v10, (a2)
+; CHECK-NEXT: add a0, a0, a1
+; CHECK-NEXT: vs1r.v v8, (a0)
; CHECK-NEXT: csrrs a0, vlenb, zero
; CHECK-NEXT: slli a0, a0, 2
; CHECK-NEXT: add sp, sp, a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-struct.ll b/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-struct.ll
index 64031f8a93598f..a9a680d54d5897 100644
--- a/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-struct.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-struct.ll
@@ -16,13 +16,13 @@ define <vscale x 1 x double> @test(ptr %addr, i64 %vl) {
; CHECK-NEXT: sub sp, sp, a2
; CHECK-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb
; CHECK-NEXT: csrrs a2, vlenb, zero
-; CHECK-NEXT: add a3, a0, a2
-; CHECK-NEXT: vl1re64.v v8, (a3)
+; CHECK-NEXT: vl1re64.v v8, (a0)
+; CHECK-NEXT: add a0, a0, a2
; CHECK-NEXT: vl1re64.v v9, (a0)
; CHECK-NEXT: addi a0, sp, 16
+; CHECK-NEXT: vs1r.v v8, (a0)
; CHECK-NEXT: add a2, a0, a2
-; CHECK-NEXT: vs1r.v v8, (a2)
-; CHECK-NEXT: vs1r.v v9, (a0)
+; CHECK-NEXT: vs1r.v v9, (a2)
; CHECK-NEXT: vl1re64.v v8, (a2)
; CHECK-NEXT: vl1re64.v v9, (a0)
; CHECK-NEXT: vsetvli zero, a1, e64, m1, ta, ma
More information about the llvm-commits
mailing list