[llvm] [CodeGen] Update for scalable MemoryType in MMO (PR #70452)
David Green via llvm-commits
llvm-commits at lists.llvm.org
Thu Nov 23 14:01:16 PST 2023
https://github.com/davemgreen updated https://github.com/llvm/llvm-project/pull/70452
>From a1490b4c8155a35008d8b0add41e8456cc5aa7f9 Mon Sep 17 00:00:00 2001
From: Harvin Iriawan <harvin.iriawan at arm.com>
Date: Wed, 25 Oct 2023 09:27:03 +0000
Subject: [PATCH 1/3] [SelectionDAG] Update for scalable MemoryType in MMO
Remove getSizeOrUnknown call when MachineMemOperand is created. For Scalable
TypeSize, the MemoryType created becomes a scalable_vector.
2 MMOs that have scalable memory access can then use the updated BasicAA that
understands scalable LocationSize
---
llvm/include/llvm/Analysis/MemoryLocation.h | 9 ++-
llvm/include/llvm/CodeGen/MachineFunction.h | 21 ++++++
llvm/include/llvm/CodeGen/MachineMemOperand.h | 16 +++--
llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp | 39 +++++++----
llvm/lib/CodeGen/MachineFunction.cpp | 20 ++++++
llvm/lib/CodeGen/MachineInstr.cpp | 43 ++++++++----
llvm/lib/CodeGen/MachineOperand.cpp | 24 ++++++-
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 70 ++++++++++++-------
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 19 +++--
llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll | 6 +-
.../RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll | 8 +--
11 files changed, 198 insertions(+), 77 deletions(-)
diff --git a/llvm/include/llvm/Analysis/MemoryLocation.h b/llvm/include/llvm/Analysis/MemoryLocation.h
index b72a27cab86b349..bea96827a7c9e39 100644
--- a/llvm/include/llvm/Analysis/MemoryLocation.h
+++ b/llvm/include/llvm/Analysis/MemoryLocation.h
@@ -291,8 +291,9 @@ class MemoryLocation {
return MemoryLocation(Ptr, LocationSize::beforeOrAfterPointer(), AATags);
}
- // Return the exact size if the exact size is known at compiletime,
- // otherwise return MemoryLocation::UnknownSize.
+ // TODO: Remove getSizeOrUnknown
+ // interface once llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest is
+ // updated
static uint64_t getSizeOrUnknown(const TypeSize &T) {
return T.isScalable() ? UnknownSize : T.getFixedValue();
}
@@ -303,6 +304,10 @@ class MemoryLocation {
const AAMDNodes &AATags = AAMDNodes())
: Ptr(Ptr), Size(Size), AATags(AATags) {}
+ explicit MemoryLocation(const Value *Ptr, TypeSize Size,
+ const AAMDNodes &AATags = AAMDNodes())
+ : Ptr(Ptr), Size(LocationSize::precise(Size)), AATags(AATags) {}
+
MemoryLocation getWithNewPtr(const Value *NewPtr) const {
MemoryLocation Copy(*this);
Copy.Ptr = NewPtr;
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index 05c9b14a423cda1..e1ba0e29d6ace81 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -1032,6 +1032,13 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
+ MachineMemOperand *getMachineMemOperand(
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, TypeSize ts,
+ Align base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
+ const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
+ AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
+ AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
+
MachineMemOperand *getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy,
Align base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
@@ -1051,6 +1058,17 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
MMO, Offset, Size == ~UINT64_C(0) ? LLT() : LLT::scalar(8 * Size));
}
+ MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
+ int64_t Offset, TypeSize ts) {
+ return getMachineMemOperand(
+ MMO, Offset,
+ ts.getKnownMinValue() == ~UINT64_C(0)
+ ? LLT()
+ : ts.isScalable()
+ ? LLT::scalable_vector(1, 8 * ts.getKnownMinValue())
+ : LLT::scalar(8 * ts.getKnownMinValue()));
+ }
+
/// getMachineMemOperand - Allocate a new MachineMemOperand by copying
/// an existing one, replacing only the MachinePointerInfo and size.
/// MachineMemOperands are owned by the MachineFunction and need not be
@@ -1058,6 +1076,9 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
uint64_t Size);
+ MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
+ const MachinePointerInfo &PtrInfo,
+ TypeSize ts);
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
LLT Ty);
diff --git a/llvm/include/llvm/CodeGen/MachineMemOperand.h b/llvm/include/llvm/CodeGen/MachineMemOperand.h
index da7fd7cdf02958d..4a575b2d9b32da9 100644
--- a/llvm/include/llvm/CodeGen/MachineMemOperand.h
+++ b/llvm/include/llvm/CodeGen/MachineMemOperand.h
@@ -190,6 +190,12 @@ class MachineMemOperand {
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
+ MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, TypeSize ts,
+ Align a, const AAMDNodes &AAInfo = AAMDNodes(),
+ const MDNode *Ranges = nullptr,
+ SyncScope::ID SSID = SyncScope::System,
+ AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
+ AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, LLT type, Align a,
const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr,
@@ -233,13 +239,15 @@ class MachineMemOperand {
LLT getMemoryType() const { return MemoryType; }
/// Return the size in bytes of the memory reference.
- uint64_t getSize() const {
- return MemoryType.isValid() ? MemoryType.getSizeInBytes() : ~UINT64_C(0);
+ TypeSize getSize() const {
+ return MemoryType.isValid() ? MemoryType.getSizeInBytes()
+ : TypeSize::Fixed(~UINT64_C(0));
}
/// Return the size in bits of the memory reference.
- uint64_t getSizeInBits() const {
- return MemoryType.isValid() ? MemoryType.getSizeInBits() : ~UINT64_C(0);
+ TypeSize getSizeInBits() const {
+ return MemoryType.isValid() ? MemoryType.getSizeInBits()
+ : TypeSize::Fixed(~UINT64_C(0));
}
LLT getType() const {
diff --git a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
index 246aa88b09acf61..b55904665f4e3f3 100644
--- a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
@@ -196,7 +196,7 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
bool IsAtomic;
Register BasePtr;
int64_t Offset;
- uint64_t NumBytes;
+ TypeSize NumBytes;
MachineMemOperand *MMO;
};
@@ -212,8 +212,7 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
Offset = 0;
}
- uint64_t Size = MemoryLocation::getSizeOrUnknown(
- LS->getMMO().getMemoryType().getSizeInBytes());
+ TypeSize Size = LS->getMMO().getMemoryType().getSizeInBytes();
return {LS->isVolatile(), LS->isAtomic(), BaseReg,
Offset /*base offset*/, Size, &LS->getMMO()};
}
@@ -221,7 +220,7 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
// Default.
return {false /*isvolatile*/,
/*isAtomic*/ false, Register(),
- (int64_t)0 /*offset*/, 0 /*size*/,
+ (int64_t)0 /*offset*/, TypeSize::getFixed(0) /*size*/,
(MachineMemOperand *)nullptr};
};
MemUseCharacteristics MUC0 = getCharacteristics(&MI),
@@ -249,10 +248,20 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
return false;
}
+ // if NumBytes is scalable and offset is not 0, conservatively return may
+ // alias
+ if ((MUC0.NumBytes.isScalable() && (MUC0.Offset != 0)) ||
+ (MUC1.NumBytes.isScalable() && (MUC1.Offset != 0)))
+ return true;
+
+ const bool BothNotScalable =
+ !(MUC0.NumBytes.isScalable() || MUC1.NumBytes.isScalable());
+
// Try to prove that there is aliasing, or that there is no aliasing. Either
// way, we can return now. If nothing can be proved, proceed with more tests.
bool IsAlias;
- if (GISelAddressing::aliasIsKnownForLoadStore(MI, Other, IsAlias, MRI))
+ if (BothNotScalable &&
+ GISelAddressing::aliasIsKnownForLoadStore(MI, Other, IsAlias, MRI))
return IsAlias;
// The following all rely on MMO0 and MMO1 being valid.
@@ -262,19 +271,23 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
// FIXME: port the alignment based alias analysis from SDAG's isAlias().
int64_t SrcValOffset0 = MUC0.MMO->getOffset();
int64_t SrcValOffset1 = MUC1.MMO->getOffset();
- uint64_t Size0 = MUC0.NumBytes;
- uint64_t Size1 = MUC1.NumBytes;
+ TypeSize Size0 = MUC0.NumBytes;
+ TypeSize Size1 = MUC1.NumBytes;
if (AA && MUC0.MMO->getValue() && MUC1.MMO->getValue() &&
Size0 != MemoryLocation::UnknownSize &&
Size1 != MemoryLocation::UnknownSize) {
// Use alias analysis information.
int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1);
- int64_t Overlap0 = Size0 + SrcValOffset0 - MinOffset;
- int64_t Overlap1 = Size1 + SrcValOffset1 - MinOffset;
- if (AA->isNoAlias(MemoryLocation(MUC0.MMO->getValue(), Overlap0,
- MUC0.MMO->getAAInfo()),
- MemoryLocation(MUC1.MMO->getValue(), Overlap1,
- MUC1.MMO->getAAInfo())))
+ int64_t Overlap0 = Size0.getKnownMinValue() + SrcValOffset0 - MinOffset;
+ int64_t Overlap1 = Size1.getKnownMinValue() + SrcValOffset1 - MinOffset;
+ LocationSize Loc0 = Size0.isScalable() ? LocationSize::precise(Size0)
+ : LocationSize::precise(Overlap0);
+ LocationSize Loc1 = Size1.isScalable() ? LocationSize::precise(Size1)
+ : LocationSize::precise(Overlap1);
+
+ if (AA->isNoAlias(
+ MemoryLocation(MUC0.MMO->getValue(), Loc0, MUC0.MMO->getAAInfo()),
+ MemoryLocation(MUC1.MMO->getValue(), Loc1, MUC1.MMO->getAAInfo())))
return false;
}
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index 07eb0ba7f45c2e3..7be09eb53ec1668 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -491,6 +491,16 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
SSID, Ordering, FailureOrdering);
}
+MachineMemOperand *MachineFunction::getMachineMemOperand(
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, TypeSize ts,
+ Align base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
+ SyncScope::ID SSID, AtomicOrdering Ordering,
+ AtomicOrdering FailureOrdering) {
+ return new (Allocator)
+ MachineMemOperand(PtrInfo, f, ts, base_alignment, AAInfo, Ranges, SSID,
+ Ordering, FailureOrdering);
+}
+
MachineMemOperand *MachineFunction::getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy,
Align base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
@@ -509,6 +519,16 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
MMO->getSuccessOrdering(), MMO->getFailureOrdering());
}
+MachineMemOperand *
+MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
+ const MachinePointerInfo &PtrInfo,
+ TypeSize ts) {
+ return new (Allocator)
+ MachineMemOperand(PtrInfo, MMO->getFlags(), ts, MMO->getBaseAlign(),
+ AAMDNodes(), nullptr, MMO->getSyncScopeID(),
+ MMO->getSuccessOrdering(), MMO->getFailureOrdering());
+}
+
MachineMemOperand *MachineFunction::getMachineMemOperand(
const MachineMemOperand *MMO, const MachinePointerInfo &PtrInfo, LLT Ty) {
return new (Allocator)
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 9e7b4df2576feee..b9a385ee993f70f 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -1306,10 +1306,11 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
int64_t OffsetB = MMOb->getOffset();
int64_t MinOffset = std::min(OffsetA, OffsetB);
- uint64_t WidthA = MMOa->getSize();
- uint64_t WidthB = MMOb->getSize();
- bool KnownWidthA = WidthA != MemoryLocation::UnknownSize;
- bool KnownWidthB = WidthB != MemoryLocation::UnknownSize;
+ TypeSize WidthA = MMOa->getSize();
+ TypeSize WidthB = MMOb->getSize();
+ bool KnownWidthA = WidthA.getKnownMinValue() != MemoryLocation::UnknownSize;
+ bool KnownWidthB = WidthB.getKnownMinValue() != MemoryLocation::UnknownSize;
+ bool BothMMONonScalable = !WidthA.isScalable() && !WidthB.isScalable();
const Value *ValA = MMOa->getValue();
const Value *ValB = MMOb->getValue();
@@ -1325,11 +1326,12 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
SameVal = true;
}
- if (SameVal) {
+ if (SameVal && BothMMONonScalable) {
if (!KnownWidthA || !KnownWidthB)
return true;
int64_t MaxOffset = std::max(OffsetA, OffsetB);
- int64_t LowWidth = (MinOffset == OffsetA) ? WidthA : WidthB;
+ int64_t LowWidth = (MinOffset == OffsetA) ? WidthA.getKnownMinValue()
+ : WidthB.getKnownMinValue();
return (MinOffset + LowWidth > MaxOffset);
}
@@ -1342,15 +1344,30 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
assert((OffsetA >= 0) && "Negative MachineMemOperand offset");
assert((OffsetB >= 0) && "Negative MachineMemOperand offset");
- int64_t OverlapA =
- KnownWidthA ? WidthA + OffsetA - MinOffset : MemoryLocation::UnknownSize;
- int64_t OverlapB =
- KnownWidthB ? WidthB + OffsetB - MinOffset : MemoryLocation::UnknownSize;
+ // If Scalable Location Size has non-zero offset,
+ // Width + Offset does not work at the moment
+ if ((WidthA.isScalable() && (OffsetA > 0)) ||
+ (WidthB.isScalable() && (OffsetB > 0))) {
+ return true;
+ }
+
+ int64_t OverlapA = KnownWidthA
+ ? WidthA.getKnownMinValue() + OffsetA - MinOffset
+ : MemoryLocation::UnknownSize;
+ int64_t OverlapB = KnownWidthB
+ ? WidthB.getKnownMinValue() + OffsetB - MinOffset
+ : MemoryLocation::UnknownSize;
+
+ LocationSize LocA = WidthA.isScalable() && KnownWidthA
+ ? LocationSize::precise(WidthA)
+ : LocationSize(OverlapA);
+ LocationSize LocB = WidthB.isScalable() && KnownWidthB
+ ? LocationSize::precise(WidthB)
+ : LocationSize(OverlapB);
return !AA->isNoAlias(
- MemoryLocation(ValA, OverlapA, UseTBAA ? MMOa->getAAInfo() : AAMDNodes()),
- MemoryLocation(ValB, OverlapB,
- UseTBAA ? MMOb->getAAInfo() : AAMDNodes()));
+ MemoryLocation(ValA, LocA, UseTBAA ? MMOa->getAAInfo() : AAMDNodes()),
+ MemoryLocation(ValB, LocB, UseTBAA ? MMOb->getAAInfo() : AAMDNodes()));
}
bool MachineInstr::mayAlias(AAResults *AA, const MachineInstr &Other,
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index b6d6a7532d34074..02a58263ff27d24 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -1097,14 +1097,29 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
s == ~UINT64_C(0) ? LLT() : LLT::scalar(8 * s), a,
AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
+MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
+ TypeSize ts, Align a,
+ const AAMDNodes &AAInfo,
+ const MDNode *Ranges, SyncScope::ID SSID,
+ AtomicOrdering Ordering,
+ AtomicOrdering FailureOrdering)
+ : MachineMemOperand(
+ ptrinfo, f,
+ ts.getKnownMinValue() == ~UINT64_C(0)
+ ? LLT()
+ : ts.isScalable()
+ ? LLT::scalable_vector(1, 8 * ts.getKnownMinValue())
+ : LLT::scalar(8 * ts.getKnownMinValue()),
+ a, AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
+
void MachineMemOperand::refineAlignment(const MachineMemOperand *MMO) {
// The Value and Offset may differ due to CSE. But the flags and size
// should be the same.
assert(MMO->getFlags() == getFlags() && "Flags mismatch!");
- assert((MMO->getSize() == ~UINT64_C(0) || getSize() == ~UINT64_C(0) ||
+ assert((MMO->getSize().getKnownMinValue() == ~UINT64_C(0) ||
+ getSize().getKnownMinValue() == ~UINT64_C(0) ||
MMO->getSize() == getSize()) &&
"Size mismatch!");
-
if (MMO->getBaseAlign() >= getBaseAlign()) {
// Update the alignment value.
BaseAlign = MMO->getBaseAlign();
@@ -1226,7 +1241,10 @@ void MachineMemOperand::print(raw_ostream &OS, ModuleSlotTracker &MST,
<< "unknown-address";
}
MachineOperand::printOperandOffset(OS, getOffset());
- if (getSize() > 0 && getAlign() != getSize())
+ if (getSize().isScalable())
+ OS << ", vscale ";
+ if (getSize().getKnownMinValue() > 0 &&
+ getAlign() != getSize().getKnownMinValue())
OS << ", align " << getAlign().value();
if (getAlign() != getBaseAlign())
OS << ", basealign " << getBaseAlign().value();
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 41d36e7d16d2e14..e23394c1851d1b2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -24188,7 +24188,7 @@ static SDValue narrowExtractedVectorLoad(SDNode *Extract, SelectionDAG &DAG) {
// TODO: Use "BaseIndexOffset" to make this more effective.
SDValue NewAddr = DAG.getMemBasePlusOffset(Ld->getBasePtr(), Offset, DL);
- uint64_t StoreSize = MemoryLocation::getSizeOrUnknown(VT.getStoreSize());
+ TypeSize StoreSize = VT.getStoreSize();
MachineFunction &MF = DAG.getMachineFunction();
MachineMemOperand *MMO;
if (Offset.isScalable()) {
@@ -27807,7 +27807,7 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
bool IsAtomic;
SDValue BasePtr;
int64_t Offset;
- std::optional<int64_t> NumBytes;
+ std::optional<TypeSize> NumBytes;
MachineMemOperand *MMO;
};
@@ -27820,13 +27820,12 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
: (LSN->getAddressingMode() == ISD::PRE_DEC)
? -1 * C->getSExtValue()
: 0;
- uint64_t Size =
- MemoryLocation::getSizeOrUnknown(LSN->getMemoryVT().getStoreSize());
+ TypeSize Size = LSN->getMemoryVT().getStoreSize();
return {LSN->isVolatile(),
LSN->isAtomic(),
LSN->getBasePtr(),
Offset /*base offset*/,
- std::optional<int64_t>(Size),
+ std::optional<TypeSize>(Size),
LSN->getMemOperand()};
}
if (const auto *LN = cast<LifetimeSDNode>(N))
@@ -27834,13 +27833,14 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
/*isAtomic*/ false,
LN->getOperand(1),
(LN->hasOffset()) ? LN->getOffset() : 0,
- (LN->hasOffset()) ? std::optional<int64_t>(LN->getSize())
- : std::optional<int64_t>(),
+ (LN->hasOffset())
+ ? std::optional<TypeSize>(TypeSize::getFixed(LN->getSize()))
+ : std::optional<TypeSize>(),
(MachineMemOperand *)nullptr};
// Default.
return {false /*isvolatile*/,
/*isAtomic*/ false, SDValue(),
- (int64_t)0 /*offset*/, std::optional<int64_t>() /*size*/,
+ (int64_t)0 /*offset*/, std::optional<TypeSize>() /*size*/,
(MachineMemOperand *)nullptr};
};
@@ -27867,10 +27867,20 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
return false;
}
+ // if NumBytes is scalable and offset is not 0, conservatively return may
+ // alias
+ if ((MUC0.NumBytes && MUC0.NumBytes.value().isScalable() &&
+ (MUC0.Offset != 0)) ||
+ (MUC1.NumBytes && MUC1.NumBytes.value().isScalable() &&
+ (MUC1.Offset != 0)))
+ return true;
// Try to prove that there is aliasing, or that there is no aliasing. Either
// way, we can return now. If nothing can be proved, proceed with more tests.
+ const bool BothNotScalable = !(MUC0.NumBytes.value().isScalable() ||
+ MUC1.NumBytes.value().isScalable());
bool IsAlias;
- if (BaseIndexOffset::computeAliasing(Op0, MUC0.NumBytes, Op1, MUC1.NumBytes,
+ if (BothNotScalable &&
+ BaseIndexOffset::computeAliasing(Op0, MUC0.NumBytes, Op1, MUC1.NumBytes,
DAG, IsAlias))
return IsAlias;
@@ -27897,17 +27907,21 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
Align OrigAlignment1 = MUC1.MMO->getBaseAlign();
auto &Size0 = MUC0.NumBytes;
auto &Size1 = MUC1.NumBytes;
- if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 &&
- Size0.has_value() && Size1.has_value() && *Size0 == *Size1 &&
- OrigAlignment0 > *Size0 && SrcValOffset0 % *Size0 == 0 &&
- SrcValOffset1 % *Size1 == 0) {
- int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.value();
- int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.value();
-
- // There is no overlap between these relatively aligned accesses of
- // similar size. Return no alias.
- if ((OffAlign0 + *Size0) <= OffAlign1 || (OffAlign1 + *Size1) <= OffAlign0)
- return false;
+
+ if (BothNotScalable) {
+ if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 &&
+ Size0.has_value() && Size1.has_value() && *Size0 == *Size1 &&
+ OrigAlignment0 > *Size0 && SrcValOffset0 % *Size0 == 0 &&
+ SrcValOffset1 % *Size1 == 0) {
+ int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.value();
+ int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.value();
+
+ // There is no overlap between these relatively aligned accesses of
+ // similar size. Return no alias.
+ if ((OffAlign0 + *Size0) <= OffAlign1 ||
+ (OffAlign1 + *Size1) <= OffAlign0)
+ return false;
+ }
}
bool UseAA = CombinerGlobalAA.getNumOccurrences() > 0
@@ -27923,12 +27937,20 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
Size1) {
// Use alias analysis information.
int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1);
- int64_t Overlap0 = *Size0 + SrcValOffset0 - MinOffset;
- int64_t Overlap1 = *Size1 + SrcValOffset1 - MinOffset;
+ int64_t Overlap0 =
+ Size0.value().getKnownMinValue() + SrcValOffset0 - MinOffset;
+ int64_t Overlap1 =
+ Size1.value().getKnownMinValue() + SrcValOffset1 - MinOffset;
+ LocationSize Loc0 = Size0.value().isScalable()
+ ? LocationSize::precise(Size0.value())
+ : LocationSize::precise(Overlap0);
+ LocationSize Loc1 = Size1.value().isScalable()
+ ? LocationSize::precise(Size1.value())
+ : LocationSize::precise(Overlap1);
if (AA->isNoAlias(
- MemoryLocation(MUC0.MMO->getValue(), Overlap0,
+ MemoryLocation(MUC0.MMO->getValue(), Loc0,
UseTBAA ? MUC0.MMO->getAAInfo() : AAMDNodes()),
- MemoryLocation(MUC1.MMO->getValue(), Overlap1,
+ MemoryLocation(MUC1.MMO->getValue(), Loc1,
UseTBAA ? MUC1.MMO->getAAInfo() : AAMDNodes())))
return false;
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 8a9e74d6de2bde1..dc69171a6a98f6d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -8501,7 +8501,7 @@ SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
if (PtrInfo.V.isNull())
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
- uint64_t Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize());
+ TypeSize Size = MemVT.getStoreSize();
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
Alignment, AAInfo, Ranges);
@@ -8622,8 +8622,7 @@ SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
MachineFunction &MF = getMachineFunction();
- uint64_t Size =
- MemoryLocation::getSizeOrUnknown(Val.getValueType().getStoreSize());
+ TypeSize Size = Val.getValueType().getStoreSize();
MachineMemOperand *MMO =
MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo);
return getStore(Chain, dl, Val, Ptr, MMO);
@@ -8676,8 +8675,7 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
- PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()),
- Alignment, AAInfo);
+ PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment, AAInfo);
return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
}
@@ -8771,7 +8769,7 @@ SDValue SelectionDAG::getLoadVP(
if (PtrInfo.V.isNull())
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
- uint64_t Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize());
+ TypeSize Size = MemVT.getStoreSize();
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
Alignment, AAInfo, Ranges);
@@ -8924,8 +8922,7 @@ SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
- PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()),
- Alignment, AAInfo);
+ PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment, AAInfo);
return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO,
IsCompressing);
}
@@ -11776,9 +11773,9 @@ MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
// We check here that the size of the memory operand fits within the size of
// the MMO. This is because the MMO might indicate only a possible address
// range instead of specifying the affected memory addresses precisely.
- // TODO: Make MachineMemOperands aware of scalable vectors.
- assert(memvt.getStoreSize().getKnownMinValue() <= MMO->getSize() &&
- "Size mismatch!");
+ if (MMO->getType().isValid())
+ assert(TypeSize::isKnownLE(memvt.getStoreSize(), MMO->getSize()) &&
+ "Size mismatch!");
}
/// Profile - Gather unique data for the node.
diff --git a/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll b/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
index 8ed7059d2e754c4..431796fac1d29ab 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
@@ -3,7 +3,7 @@
define void @UphPNR(target("aarch64.svcount") %predcnt) {
entry:
; CHECK: %0:ppr = COPY $p0
-; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2)
+; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr, vscale )
; CHECK: %1:pnr_p8to15 = COPY %0
; CHECK: INLINEASM &"ld1w {z0.s,z1.s,z2.s,z3.s}, $0/z, [x10]", 1 /* sideeffect attdialect */, 458761 /* reguse:PNR_p8to15 */, %1
; CHECK: RET_ReallyLR
@@ -17,7 +17,7 @@ entry:
define void @UpaPNR(target("aarch64.svcount") %predcnt) {
entry:
; CHECK: %0:ppr = COPY $p0
-; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2)
+; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr, vscale )
; CHECK: %1:pnr = COPY %0
; CHECK: INLINEASM &"ld1w {z0.s,z1.s,z2.s,z3.s}, $0/z, [x10]", 1 /* sideeffect attdialect */, 262153 /* reguse:PNR */, %1
; CHECK: RET_ReallyLR
@@ -31,7 +31,7 @@ entry:
define void @UplPNR(target("aarch64.svcount") %predcnt) {
entry:
; CHECK: %0:ppr = COPY $p0
-; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2)
+; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr, vscale )
; CHECK: %1:pnr_3b = COPY %0
; CHECK: INLINEASM &"fadd z0.h, $0/m, z0.h, #0.5", 1 /* sideeffect attdialect */, 393225 /* reguse:PNR_3b */, %1
; CHECK: RET_ReallyLR
diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
index d612298bf50e737..2391e9fea6e3fa5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
@@ -16,8 +16,8 @@ define void @vpmerge_vpload_store(<vscale x 2 x i32> %passthru, <vscale x 2 x i3
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr = COPY $x10
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vrnov0 = COPY $v8
; CHECK-NEXT: $v0 = COPY [[COPY1]]
- ; CHECK-NEXT: [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], $v0, [[COPY]], 5 /* e32 */, 0 /* tu, mu */
- ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store unknown-size into %ir.p, align 8)
+ ; CHECK-NEXT: [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], $v0, [[COPY]], 5 /* e32 */, 0 /* tu, mu */ :: (load unknown-size from %ir.p, align 8)
+ ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store (<vscale x 1 x s64>) into %ir.p, vscale )
; CHECK-NEXT: PseudoRET
%splat = insertelement <vscale x 2 x i1> poison, i1 -1, i32 0
%mask = shufflevector <vscale x 2 x i1> %splat, <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
@@ -37,8 +37,8 @@ define void @vpselect_vpload_store(<vscale x 2 x i32> %passthru, <vscale x 2 x i
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr = COPY $x10
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vrnov0 = COPY $v8
; CHECK-NEXT: $v0 = COPY [[COPY1]]
- ; CHECK-NEXT: [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], $v0, [[COPY]], 5 /* e32 */, 1 /* ta, mu */
- ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store unknown-size into %ir.p, align 8)
+ ; CHECK-NEXT: [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], $v0, [[COPY]], 5 /* e32 */, 1 /* ta, mu */ :: (load unknown-size from %ir.p, align 8)
+ ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store (<vscale x 1 x s64>) into %ir.p, vscale )
; CHECK-NEXT: PseudoRET
%splat = insertelement <vscale x 2 x i1> poison, i1 -1, i32 0
%mask = shufflevector <vscale x 2 x i1> %splat, <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
>From 1550ba8fccbf090acb67562efbdfce0649b2681c Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Tue, 21 Nov 2023 08:43:31 +0000
Subject: [PATCH 2/3] Cleanup
---
llvm/include/llvm/Analysis/MemoryLocation.h | 11 ----
llvm/include/llvm/CodeGen/MachineFunction.h | 16 +++---
llvm/include/llvm/CodeGen/MachineMemOperand.h | 2 +-
.../CodeGen/SelectionDAGAddressAnalysis.h | 4 +-
llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp | 4 +-
llvm/lib/CodeGen/MachineFunction.cpp | 10 ++--
llvm/lib/CodeGen/MachineInstr.cpp | 5 +-
llvm/lib/CodeGen/MachineOperand.cpp | 16 +++---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 51 ++++++++-----------
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 6 +--
.../SelectionDAGAddressAnalysis.cpp | 17 +++----
.../Target/Hexagon/HexagonStoreWidening.cpp | 6 ++-
.../SelectionDAGAddressAnalysisTest.cpp | 44 ++++++----------
13 files changed, 77 insertions(+), 115 deletions(-)
diff --git a/llvm/include/llvm/Analysis/MemoryLocation.h b/llvm/include/llvm/Analysis/MemoryLocation.h
index bea96827a7c9e39..4680b6854abdbec 100644
--- a/llvm/include/llvm/Analysis/MemoryLocation.h
+++ b/llvm/include/llvm/Analysis/MemoryLocation.h
@@ -291,23 +291,12 @@ class MemoryLocation {
return MemoryLocation(Ptr, LocationSize::beforeOrAfterPointer(), AATags);
}
- // TODO: Remove getSizeOrUnknown
- // interface once llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest is
- // updated
- static uint64_t getSizeOrUnknown(const TypeSize &T) {
- return T.isScalable() ? UnknownSize : T.getFixedValue();
- }
-
MemoryLocation() : Ptr(nullptr), Size(LocationSize::beforeOrAfterPointer()) {}
explicit MemoryLocation(const Value *Ptr, LocationSize Size,
const AAMDNodes &AATags = AAMDNodes())
: Ptr(Ptr), Size(Size), AATags(AATags) {}
- explicit MemoryLocation(const Value *Ptr, TypeSize Size,
- const AAMDNodes &AATags = AAMDNodes())
- : Ptr(Ptr), Size(LocationSize::precise(Size)), AATags(AATags) {}
-
MemoryLocation getWithNewPtr(const Value *NewPtr) const {
MemoryLocation Copy(*this);
Copy.Ptr = NewPtr;
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index e1ba0e29d6ace81..5493f84fda35ddf 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -1033,8 +1033,8 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
MachineMemOperand *getMachineMemOperand(
- MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, TypeSize ts,
- Align base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, TypeSize TS,
+ Align BaseAlignment, const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
@@ -1059,14 +1059,12 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
}
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
- int64_t Offset, TypeSize ts) {
+ int64_t Offset, TypeSize TS) {
return getMachineMemOperand(
MMO, Offset,
- ts.getKnownMinValue() == ~UINT64_C(0)
- ? LLT()
- : ts.isScalable()
- ? LLT::scalable_vector(1, 8 * ts.getKnownMinValue())
- : LLT::scalar(8 * ts.getKnownMinValue()));
+ TS.getKnownMinValue() == ~UINT64_C(0) ? LLT()
+ : TS.isScalable() ? LLT::scalable_vector(1, 8 * TS.getKnownMinValue())
+ : LLT::scalar(8 * TS.getKnownMinValue()));
}
/// getMachineMemOperand - Allocate a new MachineMemOperand by copying
@@ -1078,7 +1076,7 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
uint64_t Size);
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
- TypeSize ts);
+ TypeSize TS);
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
LLT Ty);
diff --git a/llvm/include/llvm/CodeGen/MachineMemOperand.h b/llvm/include/llvm/CodeGen/MachineMemOperand.h
index 4a575b2d9b32da9..eff50fb272db74b 100644
--- a/llvm/include/llvm/CodeGen/MachineMemOperand.h
+++ b/llvm/include/llvm/CodeGen/MachineMemOperand.h
@@ -190,7 +190,7 @@ class MachineMemOperand {
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
- MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, TypeSize ts,
+ MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, TypeSize TS,
Align a, const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr,
SyncScope::ID SSID = SyncScope::System,
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h b/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h
index 3d0f836b0c75784..b7828d890018b5a 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h
@@ -82,9 +82,9 @@ class BaseIndexOffset {
// Returns true `Op0` and `Op1` can be proven to alias/not alias, in
// which case `IsAlias` is set to true/false.
static bool computeAliasing(const SDNode *Op0,
- const std::optional<int64_t> NumBytes0,
+ const std::optional<TypeSize> NumBytes0,
const SDNode *Op1,
- const std::optional<int64_t> NumBytes1,
+ const std::optional<TypeSize> NumBytes1,
const SelectionDAG &DAG, bool &IsAlias);
/// Parses tree in N for base, index, offset addresses.
diff --git a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
index b55904665f4e3f3..5474c2965726e5c 100644
--- a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
@@ -220,7 +220,7 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
// Default.
return {false /*isvolatile*/,
/*isAtomic*/ false, Register(),
- (int64_t)0 /*offset*/, TypeSize::getFixed(0) /*size*/,
+ (int64_t)0 /*offset*/, TypeSize::Fixed(0) /*size*/,
(MachineMemOperand *)nullptr};
};
MemUseCharacteristics MUC0 = getCharacteristics(&MI),
@@ -248,7 +248,7 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
return false;
}
- // if NumBytes is scalable and offset is not 0, conservatively return may
+ // If NumBytes is scalable and offset is not 0, conservatively return may
// alias
if ((MUC0.NumBytes.isScalable() && (MUC0.Offset != 0)) ||
(MUC1.NumBytes.isScalable() && (MUC1.Offset != 0)))
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index 7be09eb53ec1668..70c12db29a519df 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -492,12 +492,12 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
}
MachineMemOperand *MachineFunction::getMachineMemOperand(
- MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, TypeSize ts,
- Align base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, TypeSize TS,
+ Align BaseAlignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
SyncScope::ID SSID, AtomicOrdering Ordering,
AtomicOrdering FailureOrdering) {
return new (Allocator)
- MachineMemOperand(PtrInfo, f, ts, base_alignment, AAInfo, Ranges, SSID,
+ MachineMemOperand(PtrInfo, F, TS, BaseAlignment, AAInfo, Ranges, SSID,
Ordering, FailureOrdering);
}
@@ -522,9 +522,9 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
MachineMemOperand *
MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
- TypeSize ts) {
+ TypeSize TS) {
return new (Allocator)
- MachineMemOperand(PtrInfo, MMO->getFlags(), ts, MMO->getBaseAlign(),
+ MachineMemOperand(PtrInfo, MMO->getFlags(), TS, MMO->getBaseAlign(),
AAMDNodes(), nullptr, MMO->getSyncScopeID(),
MMO->getSuccessOrdering(), MMO->getFailureOrdering());
}
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index b9a385ee993f70f..15a77b5a753e047 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -1346,10 +1346,9 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
// If Scalable Location Size has non-zero offset,
// Width + Offset does not work at the moment
- if ((WidthA.isScalable() && (OffsetA > 0)) ||
- (WidthB.isScalable() && (OffsetB > 0))) {
+ if ((WidthA.isScalable() && OffsetA > 0) ||
+ (WidthB.isScalable() && OffsetB > 0))
return true;
- }
int64_t OverlapA = KnownWidthA
? WidthA.getKnownMinValue() + OffsetA - MinOffset
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index 02a58263ff27d24..14fda69b2eb3447 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -1097,20 +1097,18 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
s == ~UINT64_C(0) ? LLT() : LLT::scalar(8 * s), a,
AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
-MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
- TypeSize ts, Align a,
+MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags F,
+ TypeSize TS, Align BaseAlignment,
const AAMDNodes &AAInfo,
const MDNode *Ranges, SyncScope::ID SSID,
AtomicOrdering Ordering,
AtomicOrdering FailureOrdering)
: MachineMemOperand(
- ptrinfo, f,
- ts.getKnownMinValue() == ~UINT64_C(0)
- ? LLT()
- : ts.isScalable()
- ? LLT::scalable_vector(1, 8 * ts.getKnownMinValue())
- : LLT::scalar(8 * ts.getKnownMinValue()),
- a, AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
+ ptrinfo, F,
+ TS.getKnownMinValue() == ~UINT64_C(0) ? LLT()
+ : TS.isScalable() ? LLT::scalable_vector(1, 8 * TS.getKnownMinValue())
+ : LLT::scalar(8 * TS.getKnownMinValue()),
+ BaseAlignment, AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
void MachineMemOperand::refineAlignment(const MachineMemOperand *MMO) {
// The Value and Offset may differ due to CSE. But the flags and size
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index e23394c1851d1b2..b17b3edad38621b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -27834,7 +27834,7 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
LN->getOperand(1),
(LN->hasOffset()) ? LN->getOffset() : 0,
(LN->hasOffset())
- ? std::optional<TypeSize>(TypeSize::getFixed(LN->getSize()))
+ ? std::optional<TypeSize>(TypeSize::Fixed(LN->getSize()))
: std::optional<TypeSize>(),
(MachineMemOperand *)nullptr};
// Default.
@@ -27867,20 +27867,15 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
return false;
}
- // if NumBytes is scalable and offset is not 0, conservatively return may
+ // If NumBytes is scalable and offset is not 0, conservatively return may
// alias
- if ((MUC0.NumBytes && MUC0.NumBytes.value().isScalable() &&
- (MUC0.Offset != 0)) ||
- (MUC1.NumBytes && MUC1.NumBytes.value().isScalable() &&
- (MUC1.Offset != 0)))
+ if ((MUC0.NumBytes && MUC0.NumBytes->isScalable() && MUC0.Offset != 0) ||
+ (MUC1.NumBytes && MUC1.NumBytes->isScalable() && MUC1.Offset != 0))
return true;
// Try to prove that there is aliasing, or that there is no aliasing. Either
// way, we can return now. If nothing can be proved, proceed with more tests.
- const bool BothNotScalable = !(MUC0.NumBytes.value().isScalable() ||
- MUC1.NumBytes.value().isScalable());
bool IsAlias;
- if (BothNotScalable &&
- BaseIndexOffset::computeAliasing(Op0, MUC0.NumBytes, Op1, MUC1.NumBytes,
+ if (BaseIndexOffset::computeAliasing(Op0, MUC0.NumBytes, Op1, MUC1.NumBytes,
DAG, IsAlias))
return IsAlias;
@@ -27908,20 +27903,18 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
auto &Size0 = MUC0.NumBytes;
auto &Size1 = MUC1.NumBytes;
- if (BothNotScalable) {
- if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 &&
- Size0.has_value() && Size1.has_value() && *Size0 == *Size1 &&
- OrigAlignment0 > *Size0 && SrcValOffset0 % *Size0 == 0 &&
- SrcValOffset1 % *Size1 == 0) {
- int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.value();
- int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.value();
-
- // There is no overlap between these relatively aligned accesses of
- // similar size. Return no alias.
- if ((OffAlign0 + *Size0) <= OffAlign1 ||
- (OffAlign1 + *Size1) <= OffAlign0)
- return false;
- }
+ if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 &&
+ Size0.has_value() && Size1.has_value() && !Size0->isScalable() &&
+ !Size1->isScalable() && *Size0 == *Size1 && OrigAlignment0 > *Size0 &&
+ SrcValOffset0 % *Size0 == 0 && SrcValOffset1 % *Size1 == 0) {
+ int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.value();
+ int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.value();
+
+ // There is no overlap between these relatively aligned accesses of
+ // similar size. Return no alias.
+ if ((OffAlign0 + static_cast<int64_t>(*Size0)) <= OffAlign1 ||
+ (OffAlign1 + static_cast<int64_t>(*Size1)) <= OffAlign0)
+ return false;
}
bool UseAA = CombinerGlobalAA.getNumOccurrences() > 0
@@ -27937,14 +27930,12 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
Size1) {
// Use alias analysis information.
int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1);
- int64_t Overlap0 =
- Size0.value().getKnownMinValue() + SrcValOffset0 - MinOffset;
- int64_t Overlap1 =
- Size1.value().getKnownMinValue() + SrcValOffset1 - MinOffset;
- LocationSize Loc0 = Size0.value().isScalable()
+ int64_t Overlap0 = Size0->getKnownMinValue() + SrcValOffset0 - MinOffset;
+ int64_t Overlap1 = Size1->getKnownMinValue() + SrcValOffset1 - MinOffset;
+ LocationSize Loc0 = Size0->isScalable()
? LocationSize::precise(Size0.value())
: LocationSize::precise(Overlap0);
- LocationSize Loc1 = Size1.value().isScalable()
+ LocationSize Loc1 = Size1->isScalable()
? LocationSize::precise(Size1.value())
: LocationSize::precise(Overlap1);
if (AA->isNoAlias(
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index dc69171a6a98f6d..71519c1898064e3 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -11773,9 +11773,9 @@ MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
// We check here that the size of the memory operand fits within the size of
// the MMO. This is because the MMO might indicate only a possible address
// range instead of specifying the affected memory addresses precisely.
- if (MMO->getType().isValid())
- assert(TypeSize::isKnownLE(memvt.getStoreSize(), MMO->getSize()) &&
- "Size mismatch!");
+ assert((!MMO->getType().isValid() ||
+ TypeSize::isKnownLE(memvt.getStoreSize(), MMO->getSize())) &&
+ "Size mismatch!");
}
/// Profile - Gather unique data for the node.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp
index 39a1e09e83c5940..3572da19f1d35f3 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp
@@ -85,11 +85,10 @@ bool BaseIndexOffset::equalBaseIndex(const BaseIndexOffset &Other,
}
bool BaseIndexOffset::computeAliasing(const SDNode *Op0,
- const std::optional<int64_t> NumBytes0,
+ const std::optional<TypeSize> NumBytes0,
const SDNode *Op1,
- const std::optional<int64_t> NumBytes1,
+ const std::optional<TypeSize> NumBytes1,
const SelectionDAG &DAG, bool &IsAlias) {
-
BaseIndexOffset BasePtr0 = match(Op0, DAG);
BaseIndexOffset BasePtr1 = match(Op1, DAG);
@@ -103,20 +102,20 @@ bool BaseIndexOffset::computeAliasing(const SDNode *Op0,
// vector objects on the stack.
// BasePtr1 is PtrDiff away from BasePtr0. They alias if none of the
// following situations arise:
- if (PtrDiff >= 0 &&
- *NumBytes0 != static_cast<int64_t>(MemoryLocation::UnknownSize)) {
+ if (PtrDiff >= 0 && !NumBytes0->isScalable() &&
+ NumBytes0->getFixedValue() != MemoryLocation::UnknownSize) {
// [----BasePtr0----]
// [---BasePtr1--]
// ========PtrDiff========>
- IsAlias = !(*NumBytes0 <= PtrDiff);
+ IsAlias = !((int64_t)NumBytes0->getFixedValue() <= PtrDiff);
return true;
}
- if (PtrDiff < 0 &&
- *NumBytes1 != static_cast<int64_t>(MemoryLocation::UnknownSize)) {
+ if (PtrDiff < 0 && !NumBytes1->isScalable() &&
+ NumBytes1->getFixedValue() != MemoryLocation::UnknownSize) {
// [----BasePtr0----]
// [---BasePtr1--]
// =====(-PtrDiff)====>
- IsAlias = !((PtrDiff + *NumBytes1) <= 0);
+ IsAlias = !((PtrDiff + (int64_t)NumBytes1->getFixedValue()) <= 0);
return true;
}
return false;
diff --git a/llvm/lib/Target/Hexagon/HexagonStoreWidening.cpp b/llvm/lib/Target/Hexagon/HexagonStoreWidening.cpp
index a647e699a8f07a4..f3facbe511a2f7c 100644
--- a/llvm/lib/Target/Hexagon/HexagonStoreWidening.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonStoreWidening.cpp
@@ -172,14 +172,16 @@ bool HexagonStoreWidening::instrAliased(InstrGroup &Stores,
if (!MMO.getValue())
return true;
- MemoryLocation L(MMO.getValue(), MMO.getSize(), MMO.getAAInfo());
+ MemoryLocation L(MMO.getValue(), MMO.getSize().getFixedValue(),
+ MMO.getAAInfo());
for (auto *SI : Stores) {
const MachineMemOperand &SMO = getStoreTarget(SI);
if (!SMO.getValue())
return true;
- MemoryLocation SL(SMO.getValue(), SMO.getSize(), SMO.getAAInfo());
+ MemoryLocation SL(SMO.getValue(), SMO.getSize().getFixedValue(),
+ SMO.getAAInfo());
if (!AA->isNoAlias(L, SL))
return true;
}
diff --git a/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp b/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp
index 7426884217a08e3..f711f85e1248f23 100644
--- a/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp
+++ b/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp
@@ -110,8 +110,7 @@ TEST_F(SelectionDAGAddressAnalysisTest, sameFrameObject) {
SDValue Index = DAG->getMemBasePlusOffset(FIPtr, Offset, Loc);
SDValue Store = DAG->getStore(DAG->getEntryNode(), Loc, Value, Index,
PtrInfo.getWithOffset(Offset));
- std::optional<int64_t> NumBytes = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(Store)->getMemoryVT().getStoreSize());
+ TypeSize NumBytes = cast<StoreSDNode>(Store)->getMemoryVT().getStoreSize();
bool IsAlias;
bool IsValid = BaseIndexOffset::computeAliasing(
@@ -137,7 +136,7 @@ TEST_F(SelectionDAGAddressAnalysisTest, sameFrameObjectUnknownSize) {
// Maybe unlikely that BaseIndexOffset::computeAliasing is used with the
// optional NumBytes being unset like in this test, but it would be confusing
// if that function determined IsAlias=false here.
- std::optional<int64_t> NumBytes;
+ std::optional<TypeSize> NumBytes;
bool IsAlias;
bool IsValid = BaseIndexOffset::computeAliasing(
@@ -165,10 +164,8 @@ TEST_F(SelectionDAGAddressAnalysisTest, noAliasingFrameObjects) {
PtrInfo.getWithOffset(Offset0));
SDValue Store1 = DAG->getStore(DAG->getEntryNode(), Loc, Value, Index1,
PtrInfo.getWithOffset(Offset1));
- std::optional<int64_t> NumBytes0 = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(Store0)->getMemoryVT().getStoreSize());
- std::optional<int64_t> NumBytes1 = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(Store1)->getMemoryVT().getStoreSize());
+ TypeSize NumBytes0 = cast<StoreSDNode>(Store0)->getMemoryVT().getStoreSize();
+ TypeSize NumBytes1 = cast<StoreSDNode>(Store1)->getMemoryVT().getStoreSize();
bool IsAlias;
bool IsValid = BaseIndexOffset::computeAliasing(
@@ -195,10 +192,8 @@ TEST_F(SelectionDAGAddressAnalysisTest, unknownSizeFrameObjects) {
DAG->getStore(DAG->getEntryNode(), Loc, Value, FIPtr, PtrInfo);
SDValue Store1 = DAG->getStore(DAG->getEntryNode(), Loc, Value, Index1,
MachinePointerInfo(PtrInfo.getAddrSpace()));
- std::optional<int64_t> NumBytes0 = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(Store0)->getMemoryVT().getStoreSize());
- std::optional<int64_t> NumBytes1 = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(Store1)->getMemoryVT().getStoreSize());
+ TypeSize NumBytes0 = cast<StoreSDNode>(Store0)->getMemoryVT().getStoreSize();
+ TypeSize NumBytes1 = cast<StoreSDNode>(Store1)->getMemoryVT().getStoreSize();
bool IsAlias;
bool IsValid = BaseIndexOffset::computeAliasing(
@@ -220,16 +215,14 @@ TEST_F(SelectionDAGAddressAnalysisTest, globalWithFrameObject) {
SDValue Index = DAG->getMemBasePlusOffset(FIPtr, Offset, Loc);
SDValue Store = DAG->getStore(DAG->getEntryNode(), Loc, Value, Index,
PtrInfo.getWithOffset(Offset));
- std::optional<int64_t> NumBytes = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(Store)->getMemoryVT().getStoreSize());
+ TypeSize NumBytes = cast<StoreSDNode>(Store)->getMemoryVT().getStoreSize();
EVT GTy = DAG->getTargetLoweringInfo().getValueType(DAG->getDataLayout(),
G->getType());
SDValue GValue = DAG->getConstant(0, Loc, GTy);
SDValue GAddr = DAG->getGlobalAddress(G, Loc, GTy);
SDValue GStore = DAG->getStore(DAG->getEntryNode(), Loc, GValue, GAddr,
MachinePointerInfo(G, 0));
- std::optional<int64_t> GNumBytes = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(GStore)->getMemoryVT().getStoreSize());
+ TypeSize GNumBytes = cast<StoreSDNode>(GStore)->getMemoryVT().getStoreSize();
bool IsAlias;
bool IsValid = BaseIndexOffset::computeAliasing(
@@ -248,8 +241,7 @@ TEST_F(SelectionDAGAddressAnalysisTest, globalWithAliasedGlobal) {
SDValue GAddr = DAG->getGlobalAddress(G, Loc, GTy);
SDValue GStore = DAG->getStore(DAG->getEntryNode(), Loc, GValue, GAddr,
MachinePointerInfo(G, 0));
- std::optional<int64_t> GNumBytes = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(GStore)->getMemoryVT().getStoreSize());
+ TypeSize GNumBytes = cast<StoreSDNode>(GStore)->getMemoryVT().getStoreSize();
SDValue AliasedGValue = DAG->getConstant(1, Loc, GTy);
SDValue AliasedGAddr = DAG->getGlobalAddress(AliasedG, Loc, GTy);
@@ -290,10 +282,8 @@ TEST_F(SelectionDAGAddressAnalysisTest, fixedSizeFrameObjectsWithinDiff) {
PtrInfo.getWithOffset(Offset0));
SDValue Store1 = DAG->getStore(DAG->getEntryNode(), Loc, Value1, Index1,
PtrInfo.getWithOffset(Offset1));
- std::optional<int64_t> NumBytes0 = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(Store0)->getMemoryVT().getStoreSize());
- std::optional<int64_t> NumBytes1 = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(Store1)->getMemoryVT().getStoreSize());
+ TypeSize NumBytes0 = cast<StoreSDNode>(Store0)->getMemoryVT().getStoreSize();
+ TypeSize NumBytes1 = cast<StoreSDNode>(Store1)->getMemoryVT().getStoreSize();
bool IsAlias;
bool IsValid = BaseIndexOffset::computeAliasing(
@@ -331,10 +321,8 @@ TEST_F(SelectionDAGAddressAnalysisTest, fixedSizeFrameObjectsOutOfDiff) {
PtrInfo.getWithOffset(Offset0));
SDValue Store1 = DAG->getStore(DAG->getEntryNode(), Loc, Value1, Index1,
PtrInfo.getWithOffset(Offset1));
- std::optional<int64_t> NumBytes0 = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(Store0)->getMemoryVT().getStoreSize());
- std::optional<int64_t> NumBytes1 = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(Store1)->getMemoryVT().getStoreSize());
+ TypeSize NumBytes0 = cast<StoreSDNode>(Store0)->getMemoryVT().getStoreSize();
+ TypeSize NumBytes1 = cast<StoreSDNode>(Store1)->getMemoryVT().getStoreSize();
bool IsAlias;
bool IsValid = BaseIndexOffset::computeAliasing(
@@ -365,10 +353,8 @@ TEST_F(SelectionDAGAddressAnalysisTest, twoFixedStackObjects) {
PtrInfo0.getWithOffset(Offset0));
SDValue Store1 = DAG->getStore(DAG->getEntryNode(), Loc, Value1, Index1,
PtrInfo1.getWithOffset(Offset0));
- std::optional<int64_t> NumBytes0 = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(Store0)->getMemoryVT().getStoreSize());
- std::optional<int64_t> NumBytes1 = MemoryLocation::getSizeOrUnknown(
- cast<StoreSDNode>(Store1)->getMemoryVT().getStoreSize());
+ TypeSize NumBytes0 = cast<StoreSDNode>(Store0)->getMemoryVT().getStoreSize();
+ TypeSize NumBytes1 = cast<StoreSDNode>(Store1)->getMemoryVT().getStoreSize();
bool IsAlias;
bool IsValid = BaseIndexOffset::computeAliasing(
>From a1437459bfee8ddd24c155cf8be6b0b87aafbc1e Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Thu, 23 Nov 2023 21:59:17 +0000
Subject: [PATCH 3/3] More fixups and rebasing
---
.../CodeGen/GlobalISel/GenericMachineInstrs.h | 4 +--
llvm/include/llvm/CodeGen/MachineFunction.h | 32 ++++++++++---------
llvm/include/llvm/CodeGen/MachineMemOperand.h | 10 ++----
llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp | 16 ++++------
llvm/lib/CodeGen/MachineFunction.cpp | 18 -----------
llvm/lib/CodeGen/MachineOperand.cpp | 12 -------
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 2 +-
llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll | 6 ++--
.../RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll | 4 +--
9 files changed, 34 insertions(+), 70 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
index 6ab1d4550c51ca9..ef1fc1115cc3b14 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
@@ -54,9 +54,9 @@ class GMemOperation : public GenericMachineInstr {
bool isUnordered() const { return getMMO().isUnordered(); }
/// Returns the size in bytes of the memory access.
- uint64_t getMemSize() const { return getMMO().getSize(); }
+ TypeSize getMemSize() const { return getMMO().getSize(); }
/// Returns the size in bits of the memory access.
- uint64_t getMemSizeInBits() const { return getMMO().getSizeInBits(); }
+ TypeSize getMemSizeInBits() const { return getMMO().getSizeInBits(); }
static bool classof(const MachineInstr *MI) {
return GenericMachineInstr::classof(MI) && MI->hasOneMemOperand();
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index 5493f84fda35ddf..bc0c355c6f2e2d8 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -1026,25 +1026,27 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
/// MachineMemOperands are owned by the MachineFunction and need not be
/// explicitly deallocated.
MachineMemOperand *getMachineMemOperand(
- MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s,
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy,
Align base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
-
MachineMemOperand *getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, TypeSize TS,
Align BaseAlignment, const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
-
MachineMemOperand *getMachineMemOperand(
- MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy,
- Align base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, uint64_t TS,
+ Align BaseAlignment, const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
- AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
+ AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic) {
+ return getMachineMemOperand(PtrInfo, F, TypeSize::getFixed(TS),
+ BaseAlignment, AAInfo, Ranges, SSID, Ordering,
+ FailureOrdering);
+ }
/// getMachineMemOperand - Allocate a new MachineMemOperand by copying
/// an existing one, adjusting by an offset and using the given size.
@@ -1052,12 +1054,6 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
/// explicitly deallocated.
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
int64_t Offset, LLT Ty);
- MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
- int64_t Offset, uint64_t Size) {
- return getMachineMemOperand(
- MMO, Offset, Size == ~UINT64_C(0) ? LLT() : LLT::scalar(8 * Size));
- }
-
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
int64_t Offset, TypeSize TS) {
return getMachineMemOperand(
@@ -1066,20 +1062,26 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
: TS.isScalable() ? LLT::scalable_vector(1, 8 * TS.getKnownMinValue())
: LLT::scalar(8 * TS.getKnownMinValue()));
}
+ MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
+ int64_t Offset, uint64_t Size) {
+ return getMachineMemOperand(MMO, Offset, TypeSize::getFixed(Size));
+ }
/// getMachineMemOperand - Allocate a new MachineMemOperand by copying
/// an existing one, replacing only the MachinePointerInfo and size.
/// MachineMemOperands are owned by the MachineFunction and need not be
/// explicitly deallocated.
- MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
- const MachinePointerInfo &PtrInfo,
- uint64_t Size);
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
TypeSize TS);
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
LLT Ty);
+ MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
+ const MachinePointerInfo &PtrInfo,
+ uint64_t Size) {
+ return getMachineMemOperand(MMO, PtrInfo, TypeSize::getFixed(Size));
+ }
/// Allocate a new MachineMemOperand by copying an existing one,
/// replacing only AliasAnalysis information. MachineMemOperands are owned
diff --git a/llvm/include/llvm/CodeGen/MachineMemOperand.h b/llvm/include/llvm/CodeGen/MachineMemOperand.h
index eff50fb272db74b..85f950c1f74b615 100644
--- a/llvm/include/llvm/CodeGen/MachineMemOperand.h
+++ b/llvm/include/llvm/CodeGen/MachineMemOperand.h
@@ -184,12 +184,6 @@ class MachineMemOperand {
/// and atomic ordering requirements must also be specified. For cmpxchg
/// atomic operations the atomic ordering requirements when store does not
/// occur must also be specified.
- MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, uint64_t s,
- Align a, const AAMDNodes &AAInfo = AAMDNodes(),
- const MDNode *Ranges = nullptr,
- SyncScope::ID SSID = SyncScope::System,
- AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
- AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, TypeSize TS,
Align a, const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr,
@@ -241,13 +235,13 @@ class MachineMemOperand {
/// Return the size in bytes of the memory reference.
TypeSize getSize() const {
return MemoryType.isValid() ? MemoryType.getSizeInBytes()
- : TypeSize::Fixed(~UINT64_C(0));
+ : TypeSize::getFixed(~UINT64_C(0));
}
/// Return the size in bits of the memory reference.
TypeSize getSizeInBits() const {
return MemoryType.isValid() ? MemoryType.getSizeInBits()
- : TypeSize::Fixed(~UINT64_C(0));
+ : TypeSize::getFixed(~UINT64_C(0));
}
LLT getType() const {
diff --git a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
index 5474c2965726e5c..f724825ecf80be5 100644
--- a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
@@ -117,8 +117,8 @@ bool GISelAddressing::aliasIsKnownForLoadStore(const MachineInstr &MI1,
if (!BasePtr0.BaseReg.isValid() || !BasePtr1.BaseReg.isValid())
return false;
- int64_t Size1 = LdSt1->getMemSize();
- int64_t Size2 = LdSt2->getMemSize();
+ TypeSize Size1 = LdSt1->getMemSize();
+ TypeSize Size2 = LdSt2->getMemSize();
int64_t PtrDiff;
if (BasePtr0.BaseReg == BasePtr1.BaseReg) {
@@ -128,20 +128,18 @@ bool GISelAddressing::aliasIsKnownForLoadStore(const MachineInstr &MI1,
// vector objects on the stack.
// BasePtr1 is PtrDiff away from BasePtr0. They alias if none of the
// following situations arise:
- if (PtrDiff >= 0 &&
- Size1 != static_cast<int64_t>(MemoryLocation::UnknownSize)) {
+ if (PtrDiff >= 0 && !Size1.isScalable() && Size1 != ~UINT64_C(0)) {
// [----BasePtr0----]
// [---BasePtr1--]
// ========PtrDiff========>
- IsAlias = !(Size1 <= PtrDiff);
+ IsAlias = !((int64_t)Size1 <= PtrDiff);
return true;
}
- if (PtrDiff < 0 &&
- Size2 != static_cast<int64_t>(MemoryLocation::UnknownSize)) {
+ if (PtrDiff < 0 && !Size2.isScalable() && Size2 != ~UINT64_C(0)) {
// [----BasePtr0----]
// [---BasePtr1--]
// =====(-PtrDiff)====>
- IsAlias = !((PtrDiff + Size2) <= 0);
+ IsAlias = !((PtrDiff + (int64_t)Size2) <= 0);
return true;
}
return false;
@@ -220,7 +218,7 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
// Default.
return {false /*isvolatile*/,
/*isAtomic*/ false, Register(),
- (int64_t)0 /*offset*/, TypeSize::Fixed(0) /*size*/,
+ (int64_t)0 /*offset*/, TypeSize::getFixed(0) /*size*/,
(MachineMemOperand *)nullptr};
};
MemUseCharacteristics MUC0 = getCharacteristics(&MI),
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index 70c12db29a519df..142774623868dc0 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -481,16 +481,6 @@ void MachineFunction::deleteMachineBasicBlock(MachineBasicBlock *MBB) {
BasicBlockRecycler.Deallocate(Allocator, MBB);
}
-MachineMemOperand *MachineFunction::getMachineMemOperand(
- MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s,
- Align base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
- SyncScope::ID SSID, AtomicOrdering Ordering,
- AtomicOrdering FailureOrdering) {
- return new (Allocator)
- MachineMemOperand(PtrInfo, f, s, base_alignment, AAInfo, Ranges,
- SSID, Ordering, FailureOrdering);
-}
-
MachineMemOperand *MachineFunction::getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, TypeSize TS,
Align BaseAlignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
@@ -511,14 +501,6 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
Ordering, FailureOrdering);
}
-MachineMemOperand *MachineFunction::getMachineMemOperand(
- const MachineMemOperand *MMO, const MachinePointerInfo &PtrInfo, uint64_t Size) {
- return new (Allocator)
- MachineMemOperand(PtrInfo, MMO->getFlags(), Size, MMO->getBaseAlign(),
- AAMDNodes(), nullptr, MMO->getSyncScopeID(),
- MMO->getSuccessOrdering(), MMO->getFailureOrdering());
-}
-
MachineMemOperand *
MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index 14fda69b2eb3447..226c25a68e2a453 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -1087,16 +1087,6 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
assert(getFailureOrdering() == FailureOrdering && "Value truncated");
}
-MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
- uint64_t s, Align a,
- const AAMDNodes &AAInfo,
- const MDNode *Ranges, SyncScope::ID SSID,
- AtomicOrdering Ordering,
- AtomicOrdering FailureOrdering)
- : MachineMemOperand(ptrinfo, f,
- s == ~UINT64_C(0) ? LLT() : LLT::scalar(8 * s), a,
- AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
-
MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags F,
TypeSize TS, Align BaseAlignment,
const AAMDNodes &AAInfo,
@@ -1239,8 +1229,6 @@ void MachineMemOperand::print(raw_ostream &OS, ModuleSlotTracker &MST,
<< "unknown-address";
}
MachineOperand::printOperandOffset(OS, getOffset());
- if (getSize().isScalable())
- OS << ", vscale ";
if (getSize().getKnownMinValue() > 0 &&
getAlign() != getSize().getKnownMinValue())
OS << ", align " << getAlign().value();
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index b17b3edad38621b..fdc02d5ab23b743 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -27834,7 +27834,7 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
LN->getOperand(1),
(LN->hasOffset()) ? LN->getOffset() : 0,
(LN->hasOffset())
- ? std::optional<TypeSize>(TypeSize::Fixed(LN->getSize()))
+ ? std::optional<TypeSize>(TypeSize::getFixed(LN->getSize()))
: std::optional<TypeSize>(),
(MachineMemOperand *)nullptr};
// Default.
diff --git a/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll b/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
index 431796fac1d29ab..a3bd78e1cc0b2d6 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
@@ -3,7 +3,7 @@
define void @UphPNR(target("aarch64.svcount") %predcnt) {
entry:
; CHECK: %0:ppr = COPY $p0
-; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr, vscale )
+; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr)
; CHECK: %1:pnr_p8to15 = COPY %0
; CHECK: INLINEASM &"ld1w {z0.s,z1.s,z2.s,z3.s}, $0/z, [x10]", 1 /* sideeffect attdialect */, 458761 /* reguse:PNR_p8to15 */, %1
; CHECK: RET_ReallyLR
@@ -17,7 +17,7 @@ entry:
define void @UpaPNR(target("aarch64.svcount") %predcnt) {
entry:
; CHECK: %0:ppr = COPY $p0
-; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr, vscale )
+; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr)
; CHECK: %1:pnr = COPY %0
; CHECK: INLINEASM &"ld1w {z0.s,z1.s,z2.s,z3.s}, $0/z, [x10]", 1 /* sideeffect attdialect */, 262153 /* reguse:PNR */, %1
; CHECK: RET_ReallyLR
@@ -31,7 +31,7 @@ entry:
define void @UplPNR(target("aarch64.svcount") %predcnt) {
entry:
; CHECK: %0:ppr = COPY $p0
-; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr, vscale )
+; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr)
; CHECK: %1:pnr_3b = COPY %0
; CHECK: INLINEASM &"fadd z0.h, $0/m, z0.h, #0.5", 1 /* sideeffect attdialect */, 393225 /* reguse:PNR_3b */, %1
; CHECK: RET_ReallyLR
diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
index 2391e9fea6e3fa5..f3251974b40a322 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
@@ -17,7 +17,7 @@ define void @vpmerge_vpload_store(<vscale x 2 x i32> %passthru, <vscale x 2 x i3
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vrnov0 = COPY $v8
; CHECK-NEXT: $v0 = COPY [[COPY1]]
; CHECK-NEXT: [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], $v0, [[COPY]], 5 /* e32 */, 0 /* tu, mu */ :: (load unknown-size from %ir.p, align 8)
- ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store (<vscale x 1 x s64>) into %ir.p, vscale )
+ ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store (<vscale x 1 x s64>) into %ir.p)
; CHECK-NEXT: PseudoRET
%splat = insertelement <vscale x 2 x i1> poison, i1 -1, i32 0
%mask = shufflevector <vscale x 2 x i1> %splat, <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
@@ -38,7 +38,7 @@ define void @vpselect_vpload_store(<vscale x 2 x i32> %passthru, <vscale x 2 x i
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vrnov0 = COPY $v8
; CHECK-NEXT: $v0 = COPY [[COPY1]]
; CHECK-NEXT: [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], $v0, [[COPY]], 5 /* e32 */, 1 /* ta, mu */ :: (load unknown-size from %ir.p, align 8)
- ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store (<vscale x 1 x s64>) into %ir.p, vscale )
+ ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store (<vscale x 1 x s64>) into %ir.p)
; CHECK-NEXT: PseudoRET
%splat = insertelement <vscale x 2 x i1> poison, i1 -1, i32 0
%mask = shufflevector <vscale x 2 x i1> %splat, <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
More information about the llvm-commits
mailing list