[llvm] [SelectionDAG] Update for scalable MemoryType in MMO (PR #70452)
Harvin Iriawan via llvm-commits
llvm-commits at lists.llvm.org
Fri Oct 27 06:06:03 PDT 2023
https://github.com/harviniriawan created https://github.com/llvm/llvm-project/pull/70452
This is a continuation of #69152.
Remove getSizeOrUnknown call when MachineMemOperand is created.
For Scalable TypeSize, the MemoryType created becomes a
scalable_vector.
2 MMOs that have scalable memory access can then use the updated
BasicAA that understands scalable LocationSize
>From d14d0146c350c48f3fa8e78152176d84c568b3cd Mon Sep 17 00:00:00 2001
From: Harvin Iriawan <harvin.iriawan at arm.com>
Date: Mon, 9 Oct 2023 16:07:20 +0100
Subject: [PATCH 1/2] [BasicAA] Add Vscale GEP decomposition on variable index
Enable BasicAA to be done on Scalable GEP & LocationSize
Scalable GEP expression such as @llvm.vscale and GEP of scalable type
are attached to the VariableGEPIndex, with Val representing Vscale.
VScale AA works if there's only one variable index (the vscale) and
constant offsets in the GEP for now
---
llvm/lib/Analysis/BasicAliasAnalysis.cpp | 201 ++++++++++++++-----
llvm/test/Analysis/AliasSet/memloc-vscale.ll | 3 +-
llvm/test/Analysis/BasicAA/vscale.ll | 111 ++++++----
llvm/test/Transforms/GVN/vscale.ll | 11 +-
4 files changed, 228 insertions(+), 98 deletions(-)
diff --git a/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
index 2acd5c47b7681e8..74fb52386ee1dcd 100644
--- a/llvm/lib/Analysis/BasicAliasAnalysis.cpp
+++ b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
@@ -44,6 +44,7 @@
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Operator.h"
+#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
@@ -63,6 +64,7 @@
#define DEBUG_TYPE "basicaa"
using namespace llvm;
+using namespace llvm::PatternMatch;
/// Enable analysis of recursive PHI nodes.
static cl::opt<bool> EnableRecPhiAnalysis("basic-aa-recphi", cl::Hidden,
@@ -344,13 +346,20 @@ struct LinearExpression {
/// Analyzes the specified value as a linear expression: "A*V + B", where A and
/// B are constant integers.
-static LinearExpression GetLinearExpression(
- const CastedValue &Val, const DataLayout &DL, unsigned Depth,
- AssumptionCache *AC, DominatorTree *DT) {
+static LinearExpression GetLinearExpression(const CastedValue &Val,
+ const DataLayout &DL,
+ unsigned Depth, AssumptionCache *AC,
+ DominatorTree *DT) {
// Limit our recursion depth.
if (Depth == 6)
return Val;
+ // If llvm.vscale is matched, set linear expression with scale 1 and offset 0
+ if (match(Val.V, m_VScale())) {
+ return LinearExpression(Val, APInt(Val.getBitWidth(), 1),
+ APInt(Val.getBitWidth(), 0), true);
+ }
+
if (const ConstantInt *Const = dyn_cast<ConstantInt>(Val.V))
return LinearExpression(Val, APInt(Val.getBitWidth(), 0),
Val.evaluateWith(Const->getValue()), true);
@@ -457,6 +466,9 @@ struct VariableGEPIndex {
CastedValue Val;
APInt Scale;
+ // A value representing vscale quantity in a GEP expression
+ bool IsVScale;
+
// Context instruction to use when querying information about this index.
const Instruction *CxtI;
@@ -479,13 +491,10 @@ struct VariableGEPIndex {
dbgs() << "\n";
}
void print(raw_ostream &OS) const {
- OS << "(V=" << Val.V->getName()
- << ", zextbits=" << Val.ZExtBits
- << ", sextbits=" << Val.SExtBits
- << ", truncbits=" << Val.TruncBits
- << ", scale=" << Scale
- << ", nsw=" << IsNSW
- << ", negated=" << IsNegated << ")";
+ OS << "(V=" << Val.V->getName() << " IsVScale=" << IsVScale
+ << ", zextbits=" << Val.ZExtBits << ", sextbits=" << Val.SExtBits
+ << ", truncbits=" << Val.TruncBits << ", scale=" << Scale
+ << ", nsw=" << IsNSW << ", negated=" << IsNegated << ")";
}
};
}
@@ -606,6 +615,7 @@ BasicAAResult::DecomposeGEPExpression(const Value *V, const DataLayout &DL,
for (User::const_op_iterator I = GEPOp->op_begin() + 1, E = GEPOp->op_end();
I != E; ++I, ++GTI) {
const Value *Index = *I;
+ const bool ScalableGEP = isa<ScalableVectorType>(GTI.getIndexedType());
// Compute the (potentially symbolic) offset in bytes for this index.
if (StructType *STy = GTI.getStructTypeOrNull()) {
// For a struct, add the member offset.
@@ -617,27 +627,18 @@ BasicAAResult::DecomposeGEPExpression(const Value *V, const DataLayout &DL,
continue;
}
+ TypeSize AllocTypeSize = DL.getTypeAllocSize(GTI.getIndexedType());
// For an array/pointer, add the element offset, explicitly scaled.
+ // Skip adding to constant offset if GEP index is marked as scalable
+ // they are handled below as variable offset
if (const ConstantInt *CIdx = dyn_cast<ConstantInt>(Index)) {
if (CIdx->isZero())
continue;
-
- // Don't attempt to analyze GEPs if the scalable index is not zero.
- TypeSize AllocTypeSize = DL.getTypeAllocSize(GTI.getIndexedType());
- if (AllocTypeSize.isScalable()) {
- Decomposed.Base = V;
- return Decomposed;
+ if (!ScalableGEP) {
+ Decomposed.Offset += AllocTypeSize.getFixedValue() *
+ CIdx->getValue().sextOrTrunc(MaxIndexSize);
+ continue;
}
-
- Decomposed.Offset += AllocTypeSize.getFixedValue() *
- CIdx->getValue().sextOrTrunc(MaxIndexSize);
- continue;
- }
-
- TypeSize AllocTypeSize = DL.getTypeAllocSize(GTI.getIndexedType());
- if (AllocTypeSize.isScalable()) {
- Decomposed.Base = V;
- return Decomposed;
}
GepHasConstantOffset = false;
@@ -647,22 +648,55 @@ BasicAAResult::DecomposeGEPExpression(const Value *V, const DataLayout &DL,
unsigned Width = Index->getType()->getIntegerBitWidth();
unsigned SExtBits = IndexSize > Width ? IndexSize - Width : 0;
unsigned TruncBits = IndexSize < Width ? Width - IndexSize : 0;
- LinearExpression LE = GetLinearExpression(
- CastedValue(Index, 0, SExtBits, TruncBits), DL, 0, AC, DT);
+ // Scalable GEP decomposition
+ // Allow Scalable GEP to be decomposed in the case of
+ // 1. getelementptr <4 x vscale x i32> with 1st index as a constant
+ // 2. Index which have a leaf of @llvm.vscale
+ // In both cases, essentially CastedValue of VariableGEPIndex is Vscale,
+ // however in the 1st case, CastedValue is of type constant, hence another
+ // flag in VariableGEPIndex is created in this case, IsVScale If GEP is
+ // Scalable type, e.g. <4 x vscale x i32>, the first index will have
+ // vscale as a variable index, create a LE in this case
+ LinearExpression LE(CastedValue(Index, 0, SExtBits, TruncBits));
+ if (ScalableGEP) {
+ if (const ConstantInt *CIdx = dyn_cast<ConstantInt>(Index)) {
+ LE = LinearExpression(
+ CastedValue(Index, 0, SExtBits, TruncBits),
+ CastedValue(Index, 0, SExtBits, TruncBits)
+ .evaluateWith(CIdx->getValue()),
+ APInt(CastedValue(Index, 0, SExtBits, TruncBits).getBitWidth(),
+ 0),
+ true);
+ assert(LE.Offset.isZero() && "For Scalable GEP constant first index, "
+ "the offset of LE should be 0");
+ } else {
+ // if first index is not a constant, a single variable gep will
+ // contain 2 variables, bail in this case
+ Decomposed.Base = V;
+ return Decomposed;
+ }
+ } else
+ LE = GetLinearExpression(CastedValue(Index, 0, SExtBits, TruncBits), DL,
+ 0, AC, DT);
// Scale by the type size.
- unsigned TypeSize = AllocTypeSize.getFixedValue();
+ unsigned TypeSize = AllocTypeSize.getKnownMinValue();
LE = LE.mul(APInt(IndexSize, TypeSize), GEPOp->isInBounds());
Decomposed.Offset += LE.Offset.sext(MaxIndexSize);
APInt Scale = LE.Scale.sext(MaxIndexSize);
+ bool LEhasVscale = match(LE.Val.V, m_VScale());
// If we already had an occurrence of this index variable, merge this
// scale into it. For example, we want to handle:
// A[x][x] -> x*16 + x*4 -> x*20
// This also ensures that 'x' only appears in the index list once.
+ // Only add to IsVScale VariableGEPIndex if it's @llvm.vscale or gep
+ // vscale index
for (unsigned i = 0, e = Decomposed.VarIndices.size(); i != e; ++i) {
- if (Decomposed.VarIndices[i].Val.V == LE.Val.V &&
- Decomposed.VarIndices[i].Val.hasSameCastsAs(LE.Val)) {
+ if (Decomposed.VarIndices[i].Val.hasSameCastsAs(LE.Val) &&
+ ((Decomposed.VarIndices[i].IsVScale &&
+ (ScalableGEP || LEhasVscale)) ||
+ Decomposed.VarIndices[i].Val.V == LE.Val.V)) {
Scale += Decomposed.VarIndices[i].Scale;
LE.IsNSW = false; // We cannot guarantee nsw for the merge.
Decomposed.VarIndices.erase(Decomposed.VarIndices.begin() + i);
@@ -672,10 +706,21 @@ BasicAAResult::DecomposeGEPExpression(const Value *V, const DataLayout &DL,
// Make sure that we have a scale that makes sense for this target's
// index size.
+ // Only allow variableGEP decomposition for constants, in the case of
+ // vscale
Scale = adjustToIndexSize(Scale, IndexSize);
+ bool InvalidVarVScale = (ScalableGEP && LEhasVscale) ||
+ (ScalableGEP && !isa<ConstantInt>(LE.Val.V));
+
+ assert(!InvalidVarVScale &&
+ "Variable GEP index contains VScale and another variable");
if (!!Scale) {
- VariableGEPIndex Entry = {LE.Val, Scale, CxtI, LE.IsNSW,
+ VariableGEPIndex Entry = {LE.Val,
+ Scale,
+ ScalableGEP || LEhasVscale,
+ CxtI,
+ LE.IsNSW,
/* IsNegated */ false};
Decomposed.VarIndices.push_back(Entry);
}
@@ -1058,19 +1103,17 @@ AliasResult BasicAAResult::aliasGEP(
// If an inbounds GEP would have to start from an out of bounds address
// for the two to alias, then we can assume noalias.
- // TODO: Remove !isScalable() once BasicAA fully support scalable location
- // size
if (*DecompGEP1.InBounds && DecompGEP1.VarIndices.empty() &&
- V2Size.hasValue() && !V2Size.isScalable() &&
- DecompGEP1.Offset.sge(V2Size.getValue()) &&
+ V2Size.hasValue() &&
+ DecompGEP1.Offset.sge(V2Size.getValue().getKnownMinValue()) &&
isBaseOfObject(DecompGEP2.Base))
return AliasResult::NoAlias;
if (isa<GEPOperator>(V2)) {
// Symmetric case to above.
if (*DecompGEP2.InBounds && DecompGEP1.VarIndices.empty() &&
- V1Size.hasValue() && !V1Size.isScalable() &&
- DecompGEP1.Offset.sle(-V1Size.getValue()) &&
+ V1Size.hasValue() &&
+ DecompGEP1.Offset.sle(-V1Size.getValue().getKnownMinValue()) &&
isBaseOfObject(DecompGEP1.Base))
return AliasResult::NoAlias;
}
@@ -1094,10 +1137,6 @@ AliasResult BasicAAResult::aliasGEP(
return BaseAlias;
}
- // Bail on analysing scalable LocationSize
- if (V1Size.isScalable() || V2Size.isScalable())
- return AliasResult::MayAlias;
-
// If there is a constant difference between the pointers, but the difference
// is less than the size of the associated memory object, then we know
// that the objects are partially overlapping. If the difference is
@@ -1124,16 +1163,16 @@ AliasResult BasicAAResult::aliasGEP(
Off = -Off;
}
- if (!VLeftSize.hasValue())
+ if (!VLeftSize.hasValue() || VLeftSize.isScalable())
return AliasResult::MayAlias;
- const uint64_t LSize = VLeftSize.getValue();
+ const uint64_t LSize = VLeftSize.getValue().getKnownMinValue();
if (Off.ult(LSize)) {
// Conservatively drop processing if a phi was visited and/or offset is
// too big.
AliasResult AR = AliasResult::PartialAlias;
if (VRightSize.hasValue() && Off.ule(INT32_MAX) &&
- (Off + VRightSize.getValue()).ule(LSize)) {
+ (Off + VRightSize.getValue().getKnownMinValue()).ule(LSize)) {
// Memory referenced by right pointer is nested. Save the offset in
// cache. Note that originally offset estimated as GEP1-V2, but
// AliasResult contains the shift that represents GEP1+Offset=V2.
@@ -1149,12 +1188,71 @@ AliasResult BasicAAResult::aliasGEP(
if (!V1Size.hasValue() || !V2Size.hasValue())
return AliasResult::MayAlias;
+ // VScale Alias Analysis
+ // GEPs with Vscale will have the expression A*Vscale + B (1 variable index
+ // and constant offset) The difference between two GEPs and Scalable
+ // LocationSize can then be analysed as they have the form of
+ // LSize SubtractDecomposedGEP output
+ // A * Vscale B * Vscale + C
+ // Since VScale is strictly a positive number (Vscale >= 1), the larger GEP
+ // can be known
+ // TODO: Use knowledge of vscale_range to make the analysis more accurate
+ if (DecompGEP1.VarIndices.size() == 1 && DecompGEP1.VarIndices[0].IsVScale &&
+ (V1Size.isScalable() || V2Size.isScalable())) {
+ const VariableGEPIndex &ScalableVar = DecompGEP1.VarIndices[0];
+ bool StrictlyPos = false, StrictlyNeg = false;
+ APInt &Off = DecompGEP1.Offset;
+ if (!ScalableVar.IsNegated) {
+ if (Off.isNegative())
+ StrictlyPos = ScalableVar.Scale.ugt(Off.abs());
+ else
+ StrictlyPos = true;
+ } else
+ StrictlyPos = Off.isNonNegative();
+
+ if (ScalableVar.IsNegated) {
+ if (Off.isNonNegative())
+ StrictlyNeg = Off.ult(ScalableVar.Scale.abs());
+ else
+ StrictlyNeg = true;
+ } else
+ StrictlyNeg = Off.isNegative();
+
+ if (StrictlyPos || StrictlyNeg) {
+ LocationSize VLeftSize = V2Size;
+ LocationSize VRightSize = V1Size;
+ const bool Swapped = StrictlyNeg;
+
+ if (Swapped) {
+ std::swap(VLeftSize, VRightSize);
+ Off = -Off;
+ }
+
+ const uint64_t LSize = VLeftSize.getValue().getKnownMinValue();
+ if (VLeftSize.isScalable() && ScalableVar.Scale.ult(LSize) &&
+ (ScalableVar.Scale + DecompGEP1.Offset).ult(LSize))
+ return AliasResult::PartialAlias;
+
+ if ((ScalableVar.Scale.uge(LSize) && VLeftSize.isScalable()) ||
+ ((ScalableVar.Scale + DecompGEP1.Offset).uge(LSize) &&
+ !VLeftSize.isScalable()))
+ return AliasResult::NoAlias;
+ }
+ }
+
+ // Bail on Scalable location size from now onwards
+ if (V1Size.isScalable() || V2Size.isScalable())
+ return AliasResult::MayAlias;
+
APInt GCD;
ConstantRange OffsetRange = ConstantRange(DecompGEP1.Offset);
for (unsigned i = 0, e = DecompGEP1.VarIndices.size(); i != e; ++i) {
const VariableGEPIndex &Index = DecompGEP1.VarIndices[i];
const APInt &Scale = Index.Scale;
APInt ScaleForGCD = Scale;
+ assert((!Index.IsVScale || match(Index.Val.V, m_VScale()) ||
+ isa<ConstantInt>(Index.Val.V)) &&
+ "Not allowed to have non-constant values if IsVScale is set");
if (!Index.IsNSW)
ScaleForGCD =
APInt::getOneBitSet(Scale.getBitWidth(), Scale.countr_zero());
@@ -1727,7 +1825,12 @@ void BasicAAResult::subtractDecomposedGEPs(DecomposedGEP &DestGEP,
bool Found = false;
for (auto I : enumerate(DestGEP.VarIndices)) {
VariableGEPIndex &Dest = I.value();
- if (!isValueEqualInPotentialCycles(Dest.Val.V, Src.Val.V, AAQI) ||
+ if (Dest.IsVScale != Src.IsVScale)
+ continue;
+ const bool SrcDestAreVScale = Dest.IsVScale && Src.IsVScale;
+ // Suppress base value checks if Src and Dst are of constant VScale
+ if ((!SrcDestAreVScale &&
+ !isValueEqualInPotentialCycles(Dest.Val.V, Src.Val.V, AAQI)) ||
!Dest.Val.hasSameCastsAs(Src.Val))
continue;
@@ -1752,7 +1855,11 @@ void BasicAAResult::subtractDecomposedGEPs(DecomposedGEP &DestGEP,
// If we didn't consume this entry, add it to the end of the Dest list.
if (!Found) {
- VariableGEPIndex Entry = {Src.Val, Src.Scale, Src.CxtI, Src.IsNSW,
+ VariableGEPIndex Entry = {Src.Val,
+ Src.Scale,
+ Src.IsVScale,
+ Src.CxtI,
+ Src.IsNSW,
/* IsNegated */ true};
DestGEP.VarIndices.push_back(Entry);
}
diff --git a/llvm/test/Analysis/AliasSet/memloc-vscale.ll b/llvm/test/Analysis/AliasSet/memloc-vscale.ll
index 8a83645ddaf9a87..ee67f7c15fb41b5 100644
--- a/llvm/test/Analysis/AliasSet/memloc-vscale.ll
+++ b/llvm/test/Analysis/AliasSet/memloc-vscale.ll
@@ -34,7 +34,8 @@ define void @ss2(ptr %p) {
ret void
}
; CHECK-LABEL: Alias sets for function 'son':
-; CHECK: AliasSet[{{.*}}, 2] may alias, Mod Pointers: (ptr %g, LocationSize::precise(vscale x 16)), (ptr %p, LocationSize::precise(8))
+; CHECK: AliasSet[{{.*}}, 1] must alias, Mod Pointers: (ptr %g, LocationSize::precise(vscale x 16))
+; CHECK: AliasSet[{{.*}}, 1] must alias, Mod Pointers: (ptr %p, LocationSize::precise(8))
define void @son(ptr %p) {
%g = getelementptr i8, ptr %p, i64 8
store <vscale x 2 x i64> zeroinitializer, ptr %g, align 2
diff --git a/llvm/test/Analysis/BasicAA/vscale.ll b/llvm/test/Analysis/BasicAA/vscale.ll
index 0d6d8fea392bbfc..2d277eacdcb8aeb 100644
--- a/llvm/test/Analysis/BasicAA/vscale.ll
+++ b/llvm/test/Analysis/BasicAA/vscale.ll
@@ -4,8 +4,8 @@
; CHECK-LABEL: gep_alloca_const_offset_1
; CHECK-DAG: MustAlias: <vscale x 4 x i32>* %alloc, <vscale x 4 x i32>* %gep1
-; CHECK-DAG: MayAlias: <vscale x 4 x i32>* %alloc, <vscale x 4 x i32>* %gep2
-; CHECK-DAG: MayAlias: <vscale x 4 x i32>* %gep1, <vscale x 4 x i32>* %gep2
+; CHECK-DAG: NoAlias: <vscale x 4 x i32>* %alloc, <vscale x 4 x i32>* %gep2
+; CHECK-DAG: NoAlias: <vscale x 4 x i32>* %gep1, <vscale x 4 x i32>* %gep2
define void @gep_alloca_const_offset_1() {
%alloc = alloca <vscale x 4 x i32>
%gep1 = getelementptr <vscale x 4 x i32>, ptr %alloc, i64 0
@@ -17,10 +17,9 @@ define void @gep_alloca_const_offset_1() {
}
; CHECK-LABEL: gep_alloca_const_offset_2
-; CHECK-DAG: MayAlias: <vscale x 4 x i32>* %alloc, <vscale x 4 x i32>* %gep1
-; CHECK-DAG: MayAlias: <vscale x 4 x i32>* %alloc, <vscale x 4 x i32>* %gep2
-; TODO: AliasResult for gep1,gep2 can be improved as MustAlias
-; CHECK-DAG: MayAlias: <vscale x 4 x i32>* %gep1, <vscale x 4 x i32>* %gep2
+; CHECK-DAG: NoAlias: <vscale x 4 x i32>* %alloc, <vscale x 4 x i32>* %gep1
+; CHECK-DAG: NoAlias: <vscale x 4 x i32>* %alloc, <vscale x 4 x i32>* %gep2
+; CHECK-DAG: MustAlias: <vscale x 4 x i32>* %gep1, <vscale x 4 x i32>* %gep2
define void @gep_alloca_const_offset_2() {
%alloc = alloca <vscale x 4 x i32>
%gep1 = getelementptr <vscale x 4 x i32>, ptr %alloc, i64 1
@@ -33,8 +32,8 @@ define void @gep_alloca_const_offset_2() {
; CHECK-LABEL: gep_alloca_const_offset_3
; CHECK-DAG: MustAlias: <vscale x 4 x i32>* %alloc, <vscale x 4 x i32>* %gep1
-; CHECK-DAG: MayAlias: <vscale x 4 x i32>* %alloc, i32* %gep2
-; CHECK-DAG: MayAlias: <vscale x 4 x i32>* %gep1, i32* %gep2
+; CHECK-DAG: MayAlias: <vscale x 4 x i32>* %alloc, i32* %gep2
+; CHECK-DAG: MayAlias: <vscale x 4 x i32>* %gep1, i32* %gep2
define void @gep_alloca_const_offset_3() {
%alloc = alloca <vscale x 4 x i32>
%gep1 = getelementptr <vscale x 4 x i32>, ptr %alloc, i64 0
@@ -74,10 +73,9 @@ define void @gep_alloca_symbolic_offset(i64 %idx1, i64 %idx2) {
}
; CHECK-LABEL: gep_same_base_const_offset
-; CHECK-DAG: MayAlias: i32* %gep1, <vscale x 4 x i32>* %p
-; CHECK-DAG: MayAlias: i32* %gep2, <vscale x 4 x i32>* %p
-; TODO: AliasResult for gep1,gep2 can be improved as NoAlias
-; CHECK-DAG: MayAlias: i32* %gep1, i32* %gep2
+; CHECK-DAG: NoAlias: i32* %gep1, <vscale x 4 x i32>* %p
+; CHECK-DAG: NoAlias: i32* %gep2, <vscale x 4 x i32>* %p
+; CHECK-DAG: NoAlias: i32* %gep1, i32* %gep2
define void @gep_same_base_const_offset(ptr %p) {
%gep1 = getelementptr <vscale x 4 x i32>, ptr %p, i64 1, i64 0
%gep2 = getelementptr <vscale x 4 x i32>, ptr %p, i64 1, i64 1
@@ -101,8 +99,8 @@ define void @gep_same_base_symbolic_offset(ptr %p, i64 %idx1, i64 %idx2) {
}
; CHECK-LABEL: gep_different_base_const_offset
-; CHECK-DAG: MayAlias: <vscale x 4 x i32>* %gep1, <vscale x 4 x i32>* %p1
-; CHECK-DAG: MayAlias: <vscale x 4 x i32>* %gep2, <vscale x 4 x i32>* %p2
+; CHECK-DAG: NoAlias: <vscale x 4 x i32>* %gep1, <vscale x 4 x i32>* %p1
+; CHECK-DAG: NoAlias: <vscale x 4 x i32>* %gep2, <vscale x 4 x i32>* %p2
; CHECK-DAG: NoAlias: <vscale x 4 x i32>* %p1, <vscale x 4 x i32>* %p2
; CHECK-DAG: NoAlias: <vscale x 4 x i32>* %gep1, <vscale x 4 x i32>* %p2
; CHECK-DAG: NoAlias: <vscale x 4 x i32>* %gep2, <vscale x 4 x i32>* %p1
@@ -117,12 +115,41 @@ define void @gep_different_base_const_offset(ptr noalias %p1, ptr noalias %p2) {
ret void
}
+; getelementptr @llvm.vscale tests
+; CHECK-LABEL: gep_llvm_vscale_no_alias
+; CHECK-DAG: NoAlias: <vscale x 4 x i32>* %gep1, <vscale x 4 x i32>* %gep2
+; CHECK-DAG: MustAlias: <vscale x 4 x i32>* %gep1, <vscale x 4 x i32>* %gep3
+; CHECK-DAG: NoAlias: <vscale x 4 x i32>* %gep2, <vscale x 4 x i32>* %gep3
+
+declare i64 @llvm.vscale.i64()
+define void @gep_llvm_vscale_no_alias(ptr %p) {
+ %t1 = tail call i64 @llvm.vscale.i64()
+ %t2 = shl nuw nsw i64 %t1, 3
+ %gep1 = getelementptr i32, ptr %p, i64 %t2
+ %gep2 = getelementptr <vscale x 4 x i32>, ptr %p, i64 1
+ %gep3 = getelementptr <vscale x 4 x i32>, ptr %p, i64 2
+ load <vscale x 4 x i32>, ptr %gep1
+ load <vscale x 4 x i32>, ptr %gep2
+ load <vscale x 4 x i32>, ptr %gep3
+ ret void
+}
+
+; CHECK-LABEL: gep_llvm_vscale_squared_may_alias
+; CHECK-DAG: MayAlias: <vscale x 4 x i32>* %gep1, <vscale x 4 x i32>* %gep2
+define void @gep_llvm_vscale_squared_may_alias(ptr %p) {
+ %t1 = tail call i64 @llvm.vscale.i64()
+ %gep1 = getelementptr <vscale x 4 x i32>, ptr %p, i64 %t1
+ %gep2 = getelementptr i32, ptr %p, i64 1
+ load <vscale x 4 x i32>, ptr %gep1
+ load <vscale x 4 x i32>, ptr %gep2
+ ret void
+}
; getelementptr + bitcast
; CHECK-LABEL: gep_bitcast_1
; CHECK-DAG: MustAlias: i32* %p, <vscale x 4 x i32>* %p
-; CHECK-DAG: MayAlias: i32* %gep1, <vscale x 4 x i32>* %p
-; CHECK-DAG: MayAlias: i32* %gep1, i32* %p
+; CHECK-DAG: NoAlias: i32* %gep1, <vscale x 4 x i32>* %p
+; CHECK-DAG: NoAlias: i32* %gep1, i32* %p
; CHECK-DAG: MayAlias: i32* %gep2, <vscale x 4 x i32>* %p
; CHECK-DAG: MayAlias: i32* %gep1, i32* %gep2
; CHECK-DAG: NoAlias: i32* %gep2, i32* %p
@@ -138,11 +165,11 @@ define void @gep_bitcast_1(ptr %p) {
; CHECK-LABEL: gep_bitcast_2
; CHECK-DAG: MustAlias: <vscale x 4 x float>* %p, <vscale x 4 x i32>* %p
-; CHECK-DAG: MayAlias: i32* %gep1, <vscale x 4 x i32>* %p
-; CHECK-DAG: MayAlias: i32* %gep1, <vscale x 4 x float>* %p
-; CHECK-DAG: MayAlias: float* %gep2, <vscale x 4 x i32>* %p
-; CHECK-DAG: MayAlias: i32* %gep1, float* %gep2
-; CHECK-DAG: MayAlias: float* %gep2, <vscale x 4 x float>* %p
+; CHECK-DAG: NoAlias: i32* %gep1, <vscale x 4 x i32>* %p
+; CHECK-DAG: NoAlias: i32* %gep1, <vscale x 4 x float>* %p
+; CHECK-DAG: NoAlias: float* %gep2, <vscale x 4 x i32>* %p
+; CHECK-DAG: MustAlias: i32* %gep1, float* %gep2
+; CHECK-DAG: NoAlias: float* %gep2, <vscale x 4 x float>* %p
define void @gep_bitcast_2(ptr %p) {
%gep1 = getelementptr <vscale x 4 x i32>, ptr %p, i64 1, i64 0
%gep2 = getelementptr <vscale x 4 x float>, ptr %p, i64 1, i64 0
@@ -159,8 +186,8 @@ define void @gep_bitcast_2(ptr %p) {
; CHECK-DAG: MayAlias: i32* %a, <vscale x 4 x i32>* %p
; CHECK-DAG: MayAlias: i32* %a, i32* %gep
; CHECK-DAG: MayAlias: i32* %a, i32* %gep_rec_1
-; CHECK-DAG: MayAlias: i32* %gep, <vscale x 4 x i32>* %p
-; CHECK-DAG: MayAlias: i32* %gep_rec_1, <vscale x 4 x i32>* %p
+; CHECK-DAG: NoAlias: i32* %gep, <vscale x 4 x i32>* %p
+; CHECK-DAG: NoAlias: i32* %gep_rec_1, <vscale x 4 x i32>* %p
; CHECK-DAG: NoAlias: i32* %gep, i32* %gep_rec_1
define void @gep_recursion_level_1(ptr %a, ptr %p) {
%gep = getelementptr <vscale x 4 x i32>, ptr %p, i64 1, i64 2
@@ -174,10 +201,10 @@ define void @gep_recursion_level_1(ptr %a, ptr %p) {
; CHECK-LABEL: gep_recursion_level_1_bitcast
; CHECK-DAG: MustAlias: i32* %a, <vscale x 4 x i32>* %a
-; CHECK-DAG: MayAlias: i32* %a, i32* %gep
-; CHECK-DAG: MayAlias: i32* %a, i32* %gep_rec_1
-; CHECK-DAG: MayAlias: <vscale x 4 x i32>* %a, i32* %gep
-; CHECK-DAG: MayAlias: <vscale x 4 x i32>* %a, i32* %gep_rec_1
+; CHECK-DAG: NoAlias: i32* %a, i32* %gep
+; CHECK-DAG: NoAlias: i32* %a, i32* %gep_rec_1
+; CHECK-DAG: NoAlias: <vscale x 4 x i32>* %a, i32* %gep
+; CHECK-DAG: NoAlias: <vscale x 4 x i32>* %a, i32* %gep_rec_1
; CHECK-DAG: NoAlias: i32* %gep, i32* %gep_rec_1
define void @gep_recursion_level_1_bitcast(ptr %a) {
%gep = getelementptr <vscale x 4 x i32>, ptr %a, i64 1, i64 2
@@ -194,9 +221,9 @@ define void @gep_recursion_level_1_bitcast(ptr %a) {
; CHECK-DAG: MayAlias: i32* %a, i32* %gep
; CHECK-DAG: MayAlias: i32* %a, i32* %gep_rec_1
; CHECK-DAG: MayAlias: i32* %a, i32* %gep_rec_2
-; CHECK-DAG: MayAlias: i32* %gep, <vscale x 4 x i32>* %p
-; CHECK-DAG: MayAlias: i32* %gep_rec_1, <vscale x 4 x i32>* %p
-; CHECK-DAG: MayAlias: i32* %gep_rec_2, <vscale x 4 x i32>* %p
+; CHECK-DAG: NoAlias: i32* %gep, <vscale x 4 x i32>* %p
+; CHECK-DAG: NoAlias: i32* %gep_rec_1, <vscale x 4 x i32>* %p
+; CHECK-DAG: NoAlias: i32* %gep_rec_2, <vscale x 4 x i32>* %p
; CHECK-DAG: NoAlias: i32* %gep, i32* %gep_rec_1
; CHECK-DAG: NoAlias: i32* %gep, i32* %gep_rec_2
; CHECK-DAG: NoAlias: i32* %gep_rec_1, i32* %gep_rec_2
@@ -221,34 +248,34 @@ define void @gep_recursion_level_2(ptr %a, ptr %p) {
; CHECK-DAG: MayAlias: i32* %a, i32* %gep_rec_4
; CHECK-DAG: MayAlias: i32* %a, i32* %gep_rec_5
; CHECK-DAG: MayAlias: i32* %a, i32* %gep_rec_6
-; CHECK-DAG: MayAlias: i32* %gep, <vscale x 4 x i32>* %p
-; CHECK-DAG: MayAlias: i32* %gep_rec_1, <vscale x 4 x i32>* %p
-; CHECK-DAG: MayAlias: i32* %gep_rec_2, <vscale x 4 x i32>* %p
-; CHECK-DAG: MayAlias: i32* %gep_rec_3, <vscale x 4 x i32>* %p
-; CHECK-DAG: MayAlias: i32* %gep_rec_4, <vscale x 4 x i32>* %p
-; CHECK-DAG: MayAlias: i32* %gep_rec_5, <vscale x 4 x i32>* %p
+; CHECK-DAG: NoAlias: i32* %gep, <vscale x 4 x i32>* %p
+; CHECK-DAG: NoAlias: i32* %gep_rec_1, <vscale x 4 x i32>* %p
+; CHECK-DAG: NoAlias: i32* %gep_rec_2, <vscale x 4 x i32>* %p
+; CHECK-DAG: NoAlias: i32* %gep_rec_3, <vscale x 4 x i32>* %p
+; CHECK-DAG: NoAlias: i32* %gep_rec_4, <vscale x 4 x i32>* %p
+; CHECK-DAG: NoAlias: i32* %gep_rec_5, <vscale x 4 x i32>* %p
; CHECK-DAG: MayAlias: i32* %gep_rec_6, <vscale x 4 x i32>* %p
; CHECK-DAG: NoAlias: i32* %gep, i32* %gep_rec_1
; CHECK-DAG: NoAlias: i32* %gep, i32* %gep_rec_2
; CHECK-DAG: NoAlias: i32* %gep, i32* %gep_rec_3
; CHECK-DAG: NoAlias: i32* %gep, i32* %gep_rec_4
; CHECK-DAG: NoAlias: i32* %gep, i32* %gep_rec_5
-; CHECK-DAG: NoAlias: i32* %gep, i32* %gep_rec_6
+; CHECK-DAG: MayAlias: i32* %gep, i32* %gep_rec_6
; CHECK-DAG: NoAlias: i32* %gep_rec_1, i32* %gep_rec_2
; CHECK-DAG: NoAlias: i32* %gep_rec_1, i32* %gep_rec_3
; CHECK-DAG: NoAlias: i32* %gep_rec_1, i32* %gep_rec_4
; CHECK-DAG: NoAlias: i32* %gep_rec_1, i32* %gep_rec_5
-; CHECK-DAG: NoAlias: i32* %gep_rec_1, i32* %gep_rec_6
+; CHECK-DAG: MayAlias: i32* %gep_rec_1, i32* %gep_rec_6
; CHECK-DAG: NoAlias: i32* %gep_rec_2, i32* %gep_rec_3
; CHECK-DAG: NoAlias: i32* %gep_rec_2, i32* %gep_rec_4
; CHECK-DAG: NoAlias: i32* %gep_rec_2, i32* %gep_rec_5
-; CHECK-DAG: NoAlias: i32* %gep_rec_2, i32* %gep_rec_6
+; CHECK-DAG: MayAlias: i32* %gep_rec_2, i32* %gep_rec_6
; CHECK-DAG: NoAlias: i32* %gep_rec_3, i32* %gep_rec_4
; CHECK-DAG: NoAlias: i32* %gep_rec_3, i32* %gep_rec_5
-; CHECK-DAG: NoAlias: i32* %gep_rec_3, i32* %gep_rec_6
+; CHECK-DAG: MayAlias: i32* %gep_rec_3, i32* %gep_rec_6
; CHECK-DAG: NoAlias: i32* %gep_rec_4, i32* %gep_rec_5
-; CHECK-DAG: NoAlias: i32* %gep_rec_4, i32* %gep_rec_6
-; CHECK-DAG: NoAlias: i32* %gep_rec_5, i32* %gep_rec_6
+; CHECK-DAG: MayAlias: i32* %gep_rec_4, i32* %gep_rec_6
+; CHECK-DAG: MayAlias: i32* %gep_rec_5, i32* %gep_rec_6
; GEP max lookup depth was set to 6.
define void @gep_recursion_max_lookup_depth_reached(ptr %a, ptr %p) {
%gep = getelementptr <vscale x 4 x i32>, ptr %p, i64 1, i64 2
diff --git a/llvm/test/Transforms/GVN/vscale.ll b/llvm/test/Transforms/GVN/vscale.ll
index 71adaed8e5722bd..3ecae9f54fddc7e 100644
--- a/llvm/test/Transforms/GVN/vscale.ll
+++ b/llvm/test/Transforms/GVN/vscale.ll
@@ -84,10 +84,7 @@ define i32 @load_clobber_load_gep3(ptr %p) {
; CHECK-LABEL: @load_clobber_load_gep3(
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P:%.*]], i64 1, i64 0
; CHECK-NEXT: [[LOAD1:%.*]] = load i32, ptr [[GEP1]], align 4
-; CHECK-NEXT: [[GEP2:%.*]] = getelementptr <vscale x 4 x float>, ptr [[P]], i64 1, i64 0
-; CHECK-NEXT: [[LOAD2:%.*]] = load float, ptr [[GEP2]], align 4
-; CHECK-NEXT: [[CAST:%.*]] = bitcast float [[LOAD2]] to i32
-; CHECK-NEXT: [[ADD:%.*]] = add i32 [[LOAD1]], [[CAST]]
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[LOAD1]], [[LOAD1]]
; CHECK-NEXT: ret i32 [[ADD]]
;
%gep1 = getelementptr <vscale x 4 x i32>, ptr %p, i64 1, i64 0
@@ -277,8 +274,7 @@ define void @redundant_load_elimination_2(i1 %c, ptr %p, ptr %q) {
; CHECK-NEXT: store i32 1, ptr [[GEP2]], align 4
; CHECK-NEXT: br i1 [[C:%.*]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]]
; CHECK: if.then:
-; CHECK-NEXT: [[T:%.*]] = load i32, ptr [[GEP1]], align 4
-; CHECK-NEXT: store i32 [[T]], ptr [[Q:%.*]], align 4
+; CHECK-NEXT: store i32 0, ptr [[Q:%.*]], align 4
; CHECK-NEXT: ret void
; CHECK: if.else:
; CHECK-NEXT: ret void
@@ -367,8 +363,7 @@ define void @missing_load_elimination(i1 %c, ptr %p, ptr %q, <vscale x 4 x i32>
; CHECK-NEXT: store <vscale x 4 x i32> [[V:%.*]], ptr [[P1]], align 16
; CHECK-NEXT: br i1 [[C:%.*]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]]
; CHECK: if.then:
-; CHECK-NEXT: [[T:%.*]] = load <vscale x 4 x i32>, ptr [[P]], align 16
-; CHECK-NEXT: store <vscale x 4 x i32> [[T]], ptr [[Q:%.*]], align 16
+; CHECK-NEXT: store <vscale x 4 x i32> zeroinitializer, ptr [[Q:%.*]], align 16
; CHECK-NEXT: ret void
; CHECK: if.else:
; CHECK-NEXT: ret void
>From d3fb8a001b8db73d90be9f2ffd732bc4b6c55815 Mon Sep 17 00:00:00 2001
From: Harvin Iriawan <harvin.iriawan at arm.com>
Date: Wed, 25 Oct 2023 09:27:03 +0000
Subject: [PATCH 2/2] [SelectionDAG] Update for scalable MemoryType in MMO
Remove getSizeOrUnknown call when MachineMemOperand is created.
For Scalable TypeSize, the MemoryType created becomes a
scalable_vector.
2 MMOs that have scalable memory access can then use the updated
BasicAA that understands scalable LocationSize
---
llvm/include/llvm/Analysis/MemoryLocation.h | 9 ++-
llvm/include/llvm/CodeGen/MachineFunction.h | 21 ++++++
llvm/include/llvm/CodeGen/MachineMemOperand.h | 16 +++--
llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp | 39 +++++++----
llvm/lib/CodeGen/MachineFunction.cpp | 20 ++++++
llvm/lib/CodeGen/MachineInstr.cpp | 43 ++++++++----
llvm/lib/CodeGen/MachineOperand.cpp | 24 ++++++-
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 70 ++++++++++++-------
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 19 +++--
llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll | 6 +-
.../RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll | 8 +--
11 files changed, 198 insertions(+), 77 deletions(-)
diff --git a/llvm/include/llvm/Analysis/MemoryLocation.h b/llvm/include/llvm/Analysis/MemoryLocation.h
index b72a27cab86b349..bea96827a7c9e39 100644
--- a/llvm/include/llvm/Analysis/MemoryLocation.h
+++ b/llvm/include/llvm/Analysis/MemoryLocation.h
@@ -291,8 +291,9 @@ class MemoryLocation {
return MemoryLocation(Ptr, LocationSize::beforeOrAfterPointer(), AATags);
}
- // Return the exact size if the exact size is known at compiletime,
- // otherwise return MemoryLocation::UnknownSize.
+ // TODO: Remove getSizeOrUnknown
+ // interface once llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest is
+ // updated
static uint64_t getSizeOrUnknown(const TypeSize &T) {
return T.isScalable() ? UnknownSize : T.getFixedValue();
}
@@ -303,6 +304,10 @@ class MemoryLocation {
const AAMDNodes &AATags = AAMDNodes())
: Ptr(Ptr), Size(Size), AATags(AATags) {}
+ explicit MemoryLocation(const Value *Ptr, TypeSize Size,
+ const AAMDNodes &AATags = AAMDNodes())
+ : Ptr(Ptr), Size(LocationSize::precise(Size)), AATags(AATags) {}
+
MemoryLocation getWithNewPtr(const Value *NewPtr) const {
MemoryLocation Copy(*this);
Copy.Ptr = NewPtr;
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index 8f1651c2958e591..53c066bccffa812 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -1021,6 +1021,13 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
+ MachineMemOperand *getMachineMemOperand(
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, TypeSize ts,
+ Align base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
+ const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
+ AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
+ AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
+
MachineMemOperand *getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy,
Align base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
@@ -1040,6 +1047,17 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
MMO, Offset, Size == ~UINT64_C(0) ? LLT() : LLT::scalar(8 * Size));
}
+ MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
+ int64_t Offset, TypeSize ts) {
+ return getMachineMemOperand(
+ MMO, Offset,
+ ts.getKnownMinValue() == ~UINT64_C(0)
+ ? LLT()
+ : ts.isScalable()
+ ? LLT::scalable_vector(1, 8 * ts.getKnownMinValue())
+ : LLT::scalar(8 * ts.getKnownMinValue()));
+ }
+
/// getMachineMemOperand - Allocate a new MachineMemOperand by copying
/// an existing one, replacing only the MachinePointerInfo and size.
/// MachineMemOperands are owned by the MachineFunction and need not be
@@ -1047,6 +1065,9 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
uint64_t Size);
+ MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
+ const MachinePointerInfo &PtrInfo,
+ TypeSize ts);
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
LLT Ty);
diff --git a/llvm/include/llvm/CodeGen/MachineMemOperand.h b/llvm/include/llvm/CodeGen/MachineMemOperand.h
index da7fd7cdf02958d..4a575b2d9b32da9 100644
--- a/llvm/include/llvm/CodeGen/MachineMemOperand.h
+++ b/llvm/include/llvm/CodeGen/MachineMemOperand.h
@@ -190,6 +190,12 @@ class MachineMemOperand {
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
+ MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, TypeSize ts,
+ Align a, const AAMDNodes &AAInfo = AAMDNodes(),
+ const MDNode *Ranges = nullptr,
+ SyncScope::ID SSID = SyncScope::System,
+ AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
+ AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, LLT type, Align a,
const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr,
@@ -233,13 +239,15 @@ class MachineMemOperand {
LLT getMemoryType() const { return MemoryType; }
/// Return the size in bytes of the memory reference.
- uint64_t getSize() const {
- return MemoryType.isValid() ? MemoryType.getSizeInBytes() : ~UINT64_C(0);
+ TypeSize getSize() const {
+ return MemoryType.isValid() ? MemoryType.getSizeInBytes()
+ : TypeSize::Fixed(~UINT64_C(0));
}
/// Return the size in bits of the memory reference.
- uint64_t getSizeInBits() const {
- return MemoryType.isValid() ? MemoryType.getSizeInBits() : ~UINT64_C(0);
+ TypeSize getSizeInBits() const {
+ return MemoryType.isValid() ? MemoryType.getSizeInBits()
+ : TypeSize::Fixed(~UINT64_C(0));
}
LLT getType() const {
diff --git a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
index 246aa88b09acf61..b55904665f4e3f3 100644
--- a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
@@ -196,7 +196,7 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
bool IsAtomic;
Register BasePtr;
int64_t Offset;
- uint64_t NumBytes;
+ TypeSize NumBytes;
MachineMemOperand *MMO;
};
@@ -212,8 +212,7 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
Offset = 0;
}
- uint64_t Size = MemoryLocation::getSizeOrUnknown(
- LS->getMMO().getMemoryType().getSizeInBytes());
+ TypeSize Size = LS->getMMO().getMemoryType().getSizeInBytes();
return {LS->isVolatile(), LS->isAtomic(), BaseReg,
Offset /*base offset*/, Size, &LS->getMMO()};
}
@@ -221,7 +220,7 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
// Default.
return {false /*isvolatile*/,
/*isAtomic*/ false, Register(),
- (int64_t)0 /*offset*/, 0 /*size*/,
+ (int64_t)0 /*offset*/, TypeSize::getFixed(0) /*size*/,
(MachineMemOperand *)nullptr};
};
MemUseCharacteristics MUC0 = getCharacteristics(&MI),
@@ -249,10 +248,20 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
return false;
}
+ // if NumBytes is scalable and offset is not 0, conservatively return may
+ // alias
+ if ((MUC0.NumBytes.isScalable() && (MUC0.Offset != 0)) ||
+ (MUC1.NumBytes.isScalable() && (MUC1.Offset != 0)))
+ return true;
+
+ const bool BothNotScalable =
+ !(MUC0.NumBytes.isScalable() || MUC1.NumBytes.isScalable());
+
// Try to prove that there is aliasing, or that there is no aliasing. Either
// way, we can return now. If nothing can be proved, proceed with more tests.
bool IsAlias;
- if (GISelAddressing::aliasIsKnownForLoadStore(MI, Other, IsAlias, MRI))
+ if (BothNotScalable &&
+ GISelAddressing::aliasIsKnownForLoadStore(MI, Other, IsAlias, MRI))
return IsAlias;
// The following all rely on MMO0 and MMO1 being valid.
@@ -262,19 +271,23 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
// FIXME: port the alignment based alias analysis from SDAG's isAlias().
int64_t SrcValOffset0 = MUC0.MMO->getOffset();
int64_t SrcValOffset1 = MUC1.MMO->getOffset();
- uint64_t Size0 = MUC0.NumBytes;
- uint64_t Size1 = MUC1.NumBytes;
+ TypeSize Size0 = MUC0.NumBytes;
+ TypeSize Size1 = MUC1.NumBytes;
if (AA && MUC0.MMO->getValue() && MUC1.MMO->getValue() &&
Size0 != MemoryLocation::UnknownSize &&
Size1 != MemoryLocation::UnknownSize) {
// Use alias analysis information.
int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1);
- int64_t Overlap0 = Size0 + SrcValOffset0 - MinOffset;
- int64_t Overlap1 = Size1 + SrcValOffset1 - MinOffset;
- if (AA->isNoAlias(MemoryLocation(MUC0.MMO->getValue(), Overlap0,
- MUC0.MMO->getAAInfo()),
- MemoryLocation(MUC1.MMO->getValue(), Overlap1,
- MUC1.MMO->getAAInfo())))
+ int64_t Overlap0 = Size0.getKnownMinValue() + SrcValOffset0 - MinOffset;
+ int64_t Overlap1 = Size1.getKnownMinValue() + SrcValOffset1 - MinOffset;
+ LocationSize Loc0 = Size0.isScalable() ? LocationSize::precise(Size0)
+ : LocationSize::precise(Overlap0);
+ LocationSize Loc1 = Size1.isScalable() ? LocationSize::precise(Size1)
+ : LocationSize::precise(Overlap1);
+
+ if (AA->isNoAlias(
+ MemoryLocation(MUC0.MMO->getValue(), Loc0, MUC0.MMO->getAAInfo()),
+ MemoryLocation(MUC1.MMO->getValue(), Loc1, MUC1.MMO->getAAInfo())))
return false;
}
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index 1f14546a25b1c5d..528c053715a3c2f 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -484,6 +484,16 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
SSID, Ordering, FailureOrdering);
}
+MachineMemOperand *MachineFunction::getMachineMemOperand(
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, TypeSize ts,
+ Align base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
+ SyncScope::ID SSID, AtomicOrdering Ordering,
+ AtomicOrdering FailureOrdering) {
+ return new (Allocator)
+ MachineMemOperand(PtrInfo, f, ts, base_alignment, AAInfo, Ranges, SSID,
+ Ordering, FailureOrdering);
+}
+
MachineMemOperand *MachineFunction::getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy,
Align base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
@@ -502,6 +512,16 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
MMO->getSuccessOrdering(), MMO->getFailureOrdering());
}
+MachineMemOperand *
+MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
+ const MachinePointerInfo &PtrInfo,
+ TypeSize ts) {
+ return new (Allocator)
+ MachineMemOperand(PtrInfo, MMO->getFlags(), ts, MMO->getBaseAlign(),
+ AAMDNodes(), nullptr, MMO->getSyncScopeID(),
+ MMO->getSuccessOrdering(), MMO->getFailureOrdering());
+}
+
MachineMemOperand *MachineFunction::getMachineMemOperand(
const MachineMemOperand *MMO, const MachinePointerInfo &PtrInfo, LLT Ty) {
return new (Allocator)
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index d8467e2af8786ec..146c121501d5512 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -1300,10 +1300,11 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
int64_t OffsetB = MMOb->getOffset();
int64_t MinOffset = std::min(OffsetA, OffsetB);
- uint64_t WidthA = MMOa->getSize();
- uint64_t WidthB = MMOb->getSize();
- bool KnownWidthA = WidthA != MemoryLocation::UnknownSize;
- bool KnownWidthB = WidthB != MemoryLocation::UnknownSize;
+ TypeSize WidthA = MMOa->getSize();
+ TypeSize WidthB = MMOb->getSize();
+ bool KnownWidthA = WidthA.getKnownMinValue() != MemoryLocation::UnknownSize;
+ bool KnownWidthB = WidthB.getKnownMinValue() != MemoryLocation::UnknownSize;
+ bool BothMMONonScalable = !WidthA.isScalable() && !WidthB.isScalable();
const Value *ValA = MMOa->getValue();
const Value *ValB = MMOb->getValue();
@@ -1319,11 +1320,12 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
SameVal = true;
}
- if (SameVal) {
+ if (SameVal && BothMMONonScalable) {
if (!KnownWidthA || !KnownWidthB)
return true;
int64_t MaxOffset = std::max(OffsetA, OffsetB);
- int64_t LowWidth = (MinOffset == OffsetA) ? WidthA : WidthB;
+ int64_t LowWidth = (MinOffset == OffsetA) ? WidthA.getKnownMinValue()
+ : WidthB.getKnownMinValue();
return (MinOffset + LowWidth > MaxOffset);
}
@@ -1336,15 +1338,30 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
assert((OffsetA >= 0) && "Negative MachineMemOperand offset");
assert((OffsetB >= 0) && "Negative MachineMemOperand offset");
- int64_t OverlapA =
- KnownWidthA ? WidthA + OffsetA - MinOffset : MemoryLocation::UnknownSize;
- int64_t OverlapB =
- KnownWidthB ? WidthB + OffsetB - MinOffset : MemoryLocation::UnknownSize;
+ // If Scalable Location Size has non-zero offset,
+ // Width + Offset does not work at the moment
+ if ((WidthA.isScalable() && (OffsetA > 0)) ||
+ (WidthB.isScalable() && (OffsetB > 0))) {
+ return true;
+ }
+
+ int64_t OverlapA = KnownWidthA
+ ? WidthA.getKnownMinValue() + OffsetA - MinOffset
+ : MemoryLocation::UnknownSize;
+ int64_t OverlapB = KnownWidthB
+ ? WidthB.getKnownMinValue() + OffsetB - MinOffset
+ : MemoryLocation::UnknownSize;
+
+ LocationSize LocA = WidthA.isScalable() && KnownWidthA
+ ? LocationSize::precise(WidthA)
+ : LocationSize(OverlapA);
+ LocationSize LocB = WidthB.isScalable() && KnownWidthB
+ ? LocationSize::precise(WidthB)
+ : LocationSize(OverlapB);
return !AA->isNoAlias(
- MemoryLocation(ValA, OverlapA, UseTBAA ? MMOa->getAAInfo() : AAMDNodes()),
- MemoryLocation(ValB, OverlapB,
- UseTBAA ? MMOb->getAAInfo() : AAMDNodes()));
+ MemoryLocation(ValA, LocA, UseTBAA ? MMOa->getAAInfo() : AAMDNodes()),
+ MemoryLocation(ValB, LocB, UseTBAA ? MMOb->getAAInfo() : AAMDNodes()));
}
bool MachineInstr::mayAlias(AAResults *AA, const MachineInstr &Other,
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index b6d6a7532d34074..02a58263ff27d24 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -1097,14 +1097,29 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
s == ~UINT64_C(0) ? LLT() : LLT::scalar(8 * s), a,
AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
+MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
+ TypeSize ts, Align a,
+ const AAMDNodes &AAInfo,
+ const MDNode *Ranges, SyncScope::ID SSID,
+ AtomicOrdering Ordering,
+ AtomicOrdering FailureOrdering)
+ : MachineMemOperand(
+ ptrinfo, f,
+ ts.getKnownMinValue() == ~UINT64_C(0)
+ ? LLT()
+ : ts.isScalable()
+ ? LLT::scalable_vector(1, 8 * ts.getKnownMinValue())
+ : LLT::scalar(8 * ts.getKnownMinValue()),
+ a, AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
+
void MachineMemOperand::refineAlignment(const MachineMemOperand *MMO) {
// The Value and Offset may differ due to CSE. But the flags and size
// should be the same.
assert(MMO->getFlags() == getFlags() && "Flags mismatch!");
- assert((MMO->getSize() == ~UINT64_C(0) || getSize() == ~UINT64_C(0) ||
+ assert((MMO->getSize().getKnownMinValue() == ~UINT64_C(0) ||
+ getSize().getKnownMinValue() == ~UINT64_C(0) ||
MMO->getSize() == getSize()) &&
"Size mismatch!");
-
if (MMO->getBaseAlign() >= getBaseAlign()) {
// Update the alignment value.
BaseAlign = MMO->getBaseAlign();
@@ -1226,7 +1241,10 @@ void MachineMemOperand::print(raw_ostream &OS, ModuleSlotTracker &MST,
<< "unknown-address";
}
MachineOperand::printOperandOffset(OS, getOffset());
- if (getSize() > 0 && getAlign() != getSize())
+ if (getSize().isScalable())
+ OS << ", vscale ";
+ if (getSize().getKnownMinValue() > 0 &&
+ getAlign() != getSize().getKnownMinValue())
OS << ", align " << getAlign().value();
if (getAlign() != getBaseAlign())
OS << ", basealign " << getBaseAlign().value();
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index ca5bd4952866886..96d6778d2e8317c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -24167,7 +24167,7 @@ static SDValue narrowExtractedVectorLoad(SDNode *Extract, SelectionDAG &DAG) {
// TODO: Use "BaseIndexOffset" to make this more effective.
SDValue NewAddr = DAG.getMemBasePlusOffset(Ld->getBasePtr(), Offset, DL);
- uint64_t StoreSize = MemoryLocation::getSizeOrUnknown(VT.getStoreSize());
+ TypeSize StoreSize = VT.getStoreSize();
MachineFunction &MF = DAG.getMachineFunction();
MachineMemOperand *MMO;
if (Offset.isScalable()) {
@@ -27786,7 +27786,7 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
bool IsAtomic;
SDValue BasePtr;
int64_t Offset;
- std::optional<int64_t> NumBytes;
+ std::optional<TypeSize> NumBytes;
MachineMemOperand *MMO;
};
@@ -27799,13 +27799,12 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
: (LSN->getAddressingMode() == ISD::PRE_DEC)
? -1 * C->getSExtValue()
: 0;
- uint64_t Size =
- MemoryLocation::getSizeOrUnknown(LSN->getMemoryVT().getStoreSize());
+ TypeSize Size = LSN->getMemoryVT().getStoreSize();
return {LSN->isVolatile(),
LSN->isAtomic(),
LSN->getBasePtr(),
Offset /*base offset*/,
- std::optional<int64_t>(Size),
+ std::optional<TypeSize>(Size),
LSN->getMemOperand()};
}
if (const auto *LN = cast<LifetimeSDNode>(N))
@@ -27813,13 +27812,14 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
/*isAtomic*/ false,
LN->getOperand(1),
(LN->hasOffset()) ? LN->getOffset() : 0,
- (LN->hasOffset()) ? std::optional<int64_t>(LN->getSize())
- : std::optional<int64_t>(),
+ (LN->hasOffset())
+ ? std::optional<TypeSize>(TypeSize::getFixed(LN->getSize()))
+ : std::optional<TypeSize>(),
(MachineMemOperand *)nullptr};
// Default.
return {false /*isvolatile*/,
/*isAtomic*/ false, SDValue(),
- (int64_t)0 /*offset*/, std::optional<int64_t>() /*size*/,
+ (int64_t)0 /*offset*/, std::optional<TypeSize>() /*size*/,
(MachineMemOperand *)nullptr};
};
@@ -27846,10 +27846,20 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
return false;
}
+ // if NumBytes is scalable and offset is not 0, conservatively return may
+ // alias
+ if ((MUC0.NumBytes && MUC0.NumBytes.value().isScalable() &&
+ (MUC0.Offset != 0)) ||
+ (MUC1.NumBytes && MUC1.NumBytes.value().isScalable() &&
+ (MUC1.Offset != 0)))
+ return true;
// Try to prove that there is aliasing, or that there is no aliasing. Either
// way, we can return now. If nothing can be proved, proceed with more tests.
+ const bool BothNotScalable = !(MUC0.NumBytes.value().isScalable() ||
+ MUC1.NumBytes.value().isScalable());
bool IsAlias;
- if (BaseIndexOffset::computeAliasing(Op0, MUC0.NumBytes, Op1, MUC1.NumBytes,
+ if (BothNotScalable &&
+ BaseIndexOffset::computeAliasing(Op0, MUC0.NumBytes, Op1, MUC1.NumBytes,
DAG, IsAlias))
return IsAlias;
@@ -27876,17 +27886,21 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
Align OrigAlignment1 = MUC1.MMO->getBaseAlign();
auto &Size0 = MUC0.NumBytes;
auto &Size1 = MUC1.NumBytes;
- if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 &&
- Size0.has_value() && Size1.has_value() && *Size0 == *Size1 &&
- OrigAlignment0 > *Size0 && SrcValOffset0 % *Size0 == 0 &&
- SrcValOffset1 % *Size1 == 0) {
- int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.value();
- int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.value();
-
- // There is no overlap between these relatively aligned accesses of
- // similar size. Return no alias.
- if ((OffAlign0 + *Size0) <= OffAlign1 || (OffAlign1 + *Size1) <= OffAlign0)
- return false;
+
+ if (BothNotScalable) {
+ if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 &&
+ Size0.has_value() && Size1.has_value() && *Size0 == *Size1 &&
+ OrigAlignment0 > *Size0 && SrcValOffset0 % *Size0 == 0 &&
+ SrcValOffset1 % *Size1 == 0) {
+ int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.value();
+ int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.value();
+
+ // There is no overlap between these relatively aligned accesses of
+ // similar size. Return no alias.
+ if ((OffAlign0 + *Size0) <= OffAlign1 ||
+ (OffAlign1 + *Size1) <= OffAlign0)
+ return false;
+ }
}
bool UseAA = CombinerGlobalAA.getNumOccurrences() > 0
@@ -27902,12 +27916,20 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
Size1) {
// Use alias analysis information.
int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1);
- int64_t Overlap0 = *Size0 + SrcValOffset0 - MinOffset;
- int64_t Overlap1 = *Size1 + SrcValOffset1 - MinOffset;
+ int64_t Overlap0 =
+ Size0.value().getKnownMinValue() + SrcValOffset0 - MinOffset;
+ int64_t Overlap1 =
+ Size1.value().getKnownMinValue() + SrcValOffset1 - MinOffset;
+ LocationSize Loc0 = Size0.value().isScalable()
+ ? LocationSize::precise(Size0.value())
+ : LocationSize::precise(Overlap0);
+ LocationSize Loc1 = Size1.value().isScalable()
+ ? LocationSize::precise(Size1.value())
+ : LocationSize::precise(Overlap1);
if (AA->isNoAlias(
- MemoryLocation(MUC0.MMO->getValue(), Overlap0,
+ MemoryLocation(MUC0.MMO->getValue(), Loc0,
UseTBAA ? MUC0.MMO->getAAInfo() : AAMDNodes()),
- MemoryLocation(MUC1.MMO->getValue(), Overlap1,
+ MemoryLocation(MUC1.MMO->getValue(), Loc1,
UseTBAA ? MUC1.MMO->getAAInfo() : AAMDNodes())))
return false;
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index fcc485dbb2cc512..854fde286433562 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -8497,7 +8497,7 @@ SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
if (PtrInfo.V.isNull())
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
- uint64_t Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize());
+ TypeSize Size = MemVT.getStoreSize();
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
Alignment, AAInfo, Ranges);
@@ -8618,8 +8618,7 @@ SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
MachineFunction &MF = getMachineFunction();
- uint64_t Size =
- MemoryLocation::getSizeOrUnknown(Val.getValueType().getStoreSize());
+ TypeSize Size = Val.getValueType().getStoreSize();
MachineMemOperand *MMO =
MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo);
return getStore(Chain, dl, Val, Ptr, MMO);
@@ -8672,8 +8671,7 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
- PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()),
- Alignment, AAInfo);
+ PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment, AAInfo);
return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
}
@@ -8767,7 +8765,7 @@ SDValue SelectionDAG::getLoadVP(
if (PtrInfo.V.isNull())
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
- uint64_t Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize());
+ TypeSize Size = MemVT.getStoreSize();
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
Alignment, AAInfo, Ranges);
@@ -8920,8 +8918,7 @@ SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
- PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()),
- Alignment, AAInfo);
+ PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment, AAInfo);
return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO,
IsCompressing);
}
@@ -11746,9 +11743,9 @@ MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
// We check here that the size of the memory operand fits within the size of
// the MMO. This is because the MMO might indicate only a possible address
// range instead of specifying the affected memory addresses precisely.
- // TODO: Make MachineMemOperands aware of scalable vectors.
- assert(memvt.getStoreSize().getKnownMinValue() <= MMO->getSize() &&
- "Size mismatch!");
+ if (MMO->getType().isValid())
+ assert(TypeSize::isKnownLE(memvt.getStoreSize(), MMO->getSize()) &&
+ "Size mismatch!");
}
/// Profile - Gather unique data for the node.
diff --git a/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll b/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
index 8ed7059d2e754c4..431796fac1d29ab 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
@@ -3,7 +3,7 @@
define void @UphPNR(target("aarch64.svcount") %predcnt) {
entry:
; CHECK: %0:ppr = COPY $p0
-; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2)
+; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr, vscale )
; CHECK: %1:pnr_p8to15 = COPY %0
; CHECK: INLINEASM &"ld1w {z0.s,z1.s,z2.s,z3.s}, $0/z, [x10]", 1 /* sideeffect attdialect */, 458761 /* reguse:PNR_p8to15 */, %1
; CHECK: RET_ReallyLR
@@ -17,7 +17,7 @@ entry:
define void @UpaPNR(target("aarch64.svcount") %predcnt) {
entry:
; CHECK: %0:ppr = COPY $p0
-; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2)
+; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr, vscale )
; CHECK: %1:pnr = COPY %0
; CHECK: INLINEASM &"ld1w {z0.s,z1.s,z2.s,z3.s}, $0/z, [x10]", 1 /* sideeffect attdialect */, 262153 /* reguse:PNR */, %1
; CHECK: RET_ReallyLR
@@ -31,7 +31,7 @@ entry:
define void @UplPNR(target("aarch64.svcount") %predcnt) {
entry:
; CHECK: %0:ppr = COPY $p0
-; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2)
+; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr, vscale )
; CHECK: %1:pnr_3b = COPY %0
; CHECK: INLINEASM &"fadd z0.h, $0/m, z0.h, #0.5", 1 /* sideeffect attdialect */, 393225 /* reguse:PNR_3b */, %1
; CHECK: RET_ReallyLR
diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
index d612298bf50e737..2391e9fea6e3fa5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
@@ -16,8 +16,8 @@ define void @vpmerge_vpload_store(<vscale x 2 x i32> %passthru, <vscale x 2 x i3
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr = COPY $x10
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vrnov0 = COPY $v8
; CHECK-NEXT: $v0 = COPY [[COPY1]]
- ; CHECK-NEXT: [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], $v0, [[COPY]], 5 /* e32 */, 0 /* tu, mu */
- ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store unknown-size into %ir.p, align 8)
+ ; CHECK-NEXT: [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], $v0, [[COPY]], 5 /* e32 */, 0 /* tu, mu */ :: (load unknown-size from %ir.p, align 8)
+ ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store (<vscale x 1 x s64>) into %ir.p, vscale )
; CHECK-NEXT: PseudoRET
%splat = insertelement <vscale x 2 x i1> poison, i1 -1, i32 0
%mask = shufflevector <vscale x 2 x i1> %splat, <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
@@ -37,8 +37,8 @@ define void @vpselect_vpload_store(<vscale x 2 x i32> %passthru, <vscale x 2 x i
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr = COPY $x10
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vrnov0 = COPY $v8
; CHECK-NEXT: $v0 = COPY [[COPY1]]
- ; CHECK-NEXT: [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], $v0, [[COPY]], 5 /* e32 */, 1 /* ta, mu */
- ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store unknown-size into %ir.p, align 8)
+ ; CHECK-NEXT: [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], $v0, [[COPY]], 5 /* e32 */, 1 /* ta, mu */ :: (load unknown-size from %ir.p, align 8)
+ ; CHECK-NEXT: VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store (<vscale x 1 x s64>) into %ir.p, vscale )
; CHECK-NEXT: PseudoRET
%splat = insertelement <vscale x 2 x i1> poison, i1 -1, i32 0
%mask = shufflevector <vscale x 2 x i1> %splat, <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
More information about the llvm-commits
mailing list