[llvm-branch-commits] [llvm] [AMDGPU] Price buffer fat pointer memory operations (PR #238842)
Dmitry Sidorov via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Nov 5 00:18:52 PST 2026
https://github.com/MrSidims updated https://github.com/llvm/llvm-project/pull/238842
>From bbbc670670658eb8d31fb190dfa2c258836e82f4 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Wed, 4 Nov 2026 14:55:18 +0100
Subject: [PATCH 1/2] [AMDGPU] Price buffer fat pointer memory operations
Price buffer fat pointer accesses as global accesses, except that an
aligned load is not widened and a vector of odd width elements is not
split element by element. Without relaxed buffer bounds checking, a
vector of elements that are a multiple of 32 bits wide is accessed in
pieces no wider than its alignment and no narrower than 4 bytes, and a
vector of other byte-sized or power of 2 wide elements in pieces of the
largest power of 2 bytes that divides the element size. The width comes
from buffer fat pointer lowering, which now exposes it. When the pointer
is a buffer resource plus a constant offset, the offset and a constant
number of records give the same bounds facts lowering uses to access it
in wider pieces.
---
llvm/lib/Target/AMDGPU/AMDGPU.h | 9 +++
.../AMDGPU/AMDGPULowerBufferFatPointers.cpp | 73 ++++++++++---------
.../AMDGPU/AMDGPUTargetTransformInfo.cpp | 70 ++++++++++++++----
llvm/test/Analysis/CostModel/AMDGPU/load.ll | 71 +++++++++++++-----
llvm/test/Analysis/CostModel/AMDGPU/store.ll | 71 +++++++++++++-----
5 files changed, 205 insertions(+), 89 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 75a83c55db2bc9..3fb25890b903ee 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -21,6 +21,7 @@ namespace llvm {
class AMDGPUTargetMachine;
class LazyCallGraph;
+class GCNSubtarget;
class GCNTargetMachine;
class TargetMachine;
@@ -208,6 +209,14 @@ struct AMDGPULowerBufferFatPointersPass
const TargetMachine &TM;
};
+namespace AMDGPU {
+/// Return the width in bits of the widest buffer intrinsic that buffer fat
+/// pointer lowering uses to access \p Ty with alignment \p A.
+uint64_t getMaxBufferIntrinsicWidth(const GCNSubtarget &ST,
+ const DataLayout &DL, Type *Ty, Align A,
+ bool NoWrapFromMax, bool NoPartialOOB);
+} // namespace AMDGPU
+
void initializeAMDGPULowerIntrinsicsLegacyPass(PassRegistry &);
struct AMDGPULowerIntrinsicsPass
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
index 574d39a2899a51..ca4b987f8a8e1d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
@@ -753,32 +753,6 @@ class LegalizeBufferContentTypesVisitor
/// Break up the loads of a struct into the loads of its components
- /// Return the maximum allowed load/store width for the given type and
- /// alignment combination based on subtarget flags.
- /// 1. If unaligned accesses are not enabled, then any load/store that is less
- /// than word-aligned has to be handled one byte or ushort at a time.
- /// 2. If relaxed OOB mode is not set, we must ensure that the in-bounds
- /// part of a partially out of bounds read/write is performed correctly. This
- /// means that any load that isn't naturally aligned has to be split into
- /// parts that are naturally aligned, so that, after bitcasting, we don't have
- /// unaligned loads that could discard valid data.
- ///
- /// For example, if we're loading a <8 x i8>, that's actually a load of a <2 x
- /// i32>, and if we load from an align(2) address, that address might be 2
- /// bytes from the end of the buffer. The hardware will, when performing the
- /// <2 x i32> load, mask off the entire first word, causing the two in-bounds
- /// bytes to be masked off. However,if we know the offset can't be too close
- /// to the number of records in the buffer (if known), we can skip this
- /// expansion.
- ///
- /// Unlike the complete disablement of unaligned accesses from point 1,
- /// this does not apply to unaligned scalars, but will apply to cases like
- /// `load <2 x i32>, align 4` since the left elemenvt might be out of bounds.
- /// Note that if the we know that the base offset is known to be
- /// less than `uint32_max - byte_size(Ty)`, we can skip these alignment
- /// checks.
- uint64_t maxIntrinsicWidth(Type *Ty, Align A, OobProperties OobProps);
-
/// Convert a vector or scalar type that can't be operated on by buffer
/// intrinsics to one that would be legal through bitcasts and/or truncation.
/// Uses the wider of i32, i16, or i8 where possible, clamping to the maximum
@@ -950,14 +924,39 @@ LegalizeBufferContentTypesVisitor::analyzeOobProperties(Value *Ptr, Type *Ty,
return Result;
}
-uint64_t
-LegalizeBufferContentTypesVisitor::maxIntrinsicWidth(Type *T, Align A,
- OobProperties OobProps) {
+/// Return the maximum allowed load/store width for the given type and
+/// alignment combination based on subtarget flags.
+/// 1. If unaligned accesses are not enabled, then any load/store that is less
+/// than word-aligned has to be handled one byte or ushort at a time.
+/// 2. If relaxed OOB mode is not set, we must ensure that the in-bounds
+/// part of a partially out of bounds read/write is performed correctly. This
+/// means that any load that isn't naturally aligned has to be split into
+/// parts that are naturally aligned, so that, after bitcasting, we don't have
+/// unaligned loads that could discard valid data.
+///
+/// For example, if we're loading a <8 x i8>, that's actually a load of a <2 x
+/// i32>, and if we load from an align(2) address, that address might be 2
+/// bytes from the end of the buffer. The hardware will, when performing the
+/// <2 x i32> load, mask off the entire first word, causing the two in-bounds
+/// bytes to be masked off. However,if we know the offset can't be too close
+/// to the number of records in the buffer (if known), we can skip this
+/// expansion.
+///
+/// Unlike the complete disablement of unaligned accesses from point 1,
+/// this does not apply to unaligned scalars, but will apply to cases like
+/// `load <2 x i32>, align 4` since the left elemenvt might be out of bounds.
+/// Note that if the we know that the base offset is known to be
+/// less than `uint32_max - byte_size(Ty)`, we can skip these alignment
+/// checks.
+uint64_t AMDGPU::getMaxBufferIntrinsicWidth(const GCNSubtarget &ST,
+ const DataLayout &DL, Type *T,
+ Align A, bool NoWrapFromMax,
+ bool NoPartialOOB) {
Align Result(16);
- if (!ST->hasUnalignedBufferAccessEnabled() && A < Align(4))
+ if (!ST.hasUnalignedBufferAccessEnabled() && A < Align(4))
Result = A;
auto *VT = dyn_cast<VectorType>(T);
- if (!ST->hasRelaxedBufferOOBMode() && VT) {
+ if (!ST.hasRelaxedBufferOOBMode() && VT) {
TypeSize ElemBits = DL.getTypeSizeInBits(VT->getElementType());
if (ElemBits.isKnownMultipleOf(32)) {
// Word-sized operations are bounds-checked per word. So, the only case we
@@ -965,7 +964,7 @@ LegalizeBufferContentTypesVisitor::maxIntrinsicWidth(Type *T, Align A,
// and those can only become in-bounds on a multiple of their alignment.
// Therefore, we can use the declared alignment of the operation as the
// maximum width, rounding up to 4.
- if (!OobProps.NoWrapFromMax)
+ if (!NoWrapFromMax)
Result = std::min(Result, std::max(A, Align(4)));
} else if ((ElemBits.isKnownMultipleOf(8) ||
isPowerOf2_64(ElemBits.getKnownMinValue()))) {
@@ -984,7 +983,7 @@ LegalizeBufferContentTypesVisitor::maxIntrinsicWidth(Type *T, Align A,
// Strict OOB checking isn't supported if the size of each element is a
// non-power-of-2 value less than 8, since there's no feasible way to
// apply such a strict bounds check.
- if (!OobProps.NoPartialOOB)
+ if (!NoPartialOOB)
Result =
commonAlignment(Result, divideCeil(ElemBits.getKnownMinValue(), 8));
}
@@ -1211,7 +1210,9 @@ bool LegalizeBufferContentTypesVisitor::visitLoadImpl(
Type *ArrayAsVecType = scalarArrayTypeAsVector(PartType);
OobProperties OobProps =
analyzeOobProperties(OrigLI.getPointerOperand(), PartType, AggByteOff);
- uint64_t MaxWidth = maxIntrinsicWidth(ArrayAsVecType, PartAlign, OobProps);
+ uint64_t MaxWidth = AMDGPU::getMaxBufferIntrinsicWidth(
+ *ST, DL, ArrayAsVecType, PartAlign, OobProps.NoWrapFromMax,
+ OobProps.NoPartialOOB);
Type *LegalType = legalNonAggregateForMemOp(ArrayAsVecType, MaxWidth);
SmallVector<VecSlice> Slices;
@@ -1348,7 +1349,9 @@ std::pair<bool, bool> LegalizeBufferContentTypesVisitor::visitStoreImpl(
Align PartAlign = commonAlignment(OrigSI.getAlign(), AggByteOff);
OobProperties OobProps =
analyzeOobProperties(OrigSI.getPointerOperand(), PartType, AggByteOff);
- uint64_t MaxWidth = maxIntrinsicWidth(ArrayAsVecType, PartAlign, OobProps);
+ uint64_t MaxWidth = AMDGPU::getMaxBufferIntrinsicWidth(
+ *ST, DL, ArrayAsVecType, PartAlign, OobProps.NoWrapFromMax,
+ OobProps.NoPartialOOB);
Type *LegalType = legalNonAggregateForMemOp(ArrayAsVecType, MaxWidth);
if (LegalType != ArrayAsVecType) {
NewData = makeLegalNonAggregate(NewData, LegalType, Name);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index f390226ef7297d..e59f7aff6e27df 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -2213,6 +2213,46 @@ bool GCNTTIImpl::hasUniformPointerOperand(const Instruction *I) const {
return isa<Argument, Constant>(Ptr) && !isSourceOfDivergence(Ptr);
}
+static std::pair<bool, bool> getKnownBufferOobProperties(const GCNSubtarget &ST,
+ const DataLayout &DL,
+ const Instruction *I,
+ unsigned Bytes) {
+ const Value *Ptr = I ? getLoadStorePointerOperand(I) : nullptr;
+ if (!Ptr ||
+ Ptr->getType()->getPointerAddressSpace() != AMDGPUAS::BUFFER_FAT_POINTER)
+ return {false, false};
+ APInt Offset(DL.getIndexTypeSizeInBits(Ptr->getType()), 0);
+ while (const auto *GEP = dyn_cast<GEPOperator>(Ptr)) {
+ if (!GEP->accumulateConstantOffset(DL, Offset))
+ return {false, false};
+ Ptr = GEP->getPointerOperand();
+ }
+ const auto *Cast = dyn_cast<AddrSpaceCastInst>(Ptr);
+ if (Cast) {
+ if (Cast->getSrcAddressSpace() != AMDGPUAS::BUFFER_RESOURCE)
+ return {false, false};
+ Ptr = Cast->getPointerOperand();
+ }
+ const auto *Rsrc = dyn_cast<IntrinsicInst>(Ptr);
+ if (Rsrc && Rsrc->getIntrinsicID() != Intrinsic::amdgcn_make_buffer_rsrc)
+ Rsrc = nullptr;
+ if (!Cast && !Rsrc)
+ return {false, false};
+
+ bool NoWrapFromMax =
+ Offset.ule(APInt::getAllOnes(Offset.getBitWidth()) - Bytes);
+ const auto *NumRecords =
+ Rsrc ? dyn_cast<ConstantInt>(Rsrc->getArgOperand(2)) : nullptr;
+ std::optional<unsigned> NumRecordsWidth =
+ ST.getBufferResourceNumRecordsWidth();
+ if (!NumRecords || !NumRecordsWidth)
+ return {NoWrapFromMax, false};
+ APInt Bound = NumRecords->getValue().zextOrTrunc(*NumRecordsWidth);
+ APInt BoundsDiff = Bound.zext(64) - Offset.zext(64);
+ return {NoWrapFromMax, Bound.isAllOnes() || BoundsDiff.sge(Bytes) ||
+ BoundsDiff.isNonPositive()};
+}
+
InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
Align Alignment,
unsigned AddressSpace,
@@ -2228,16 +2268,9 @@ InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
unsigned EltBits = std::max<unsigned>(8, DL.getTypeSizeInBits(EltTy));
- if (AddressSpace == AMDGPUAS::BUFFER_FAT_POINTER) {
- if (Src->isVectorTy() && EltTy->isIntegerTy(8))
- return divideCeil(DL.getTypeSizeInBits(Src) - 1,
- getLoadStoreVecRegBitWidth(AddressSpace));
- return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
- CostKind, OpInfo, I);
- }
-
auto *VecTy = dyn_cast<FixedVectorType>(Src);
- if (VecTy && EltBits % 8 == 0 && !isPowerOf2_32(EltBits) &&
+ if (VecTy && AddressSpace != AMDGPUAS::BUFFER_FAT_POINTER &&
+ EltBits % 8 == 0 && !isPowerOf2_32(EltBits) &&
!TLI->shouldCombineMemoryType(TLI->getValueType(DL, Src))) {
Align EltAlign = commonAlignment(Alignment, EltBits / 8);
return VecTy->getNumElements() * getMemoryOpCost(Opcode, EltTy, EltAlign,
@@ -2288,14 +2321,23 @@ InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
HasDwordx3 = HasDwordx3 &&
(Alignment >= 16 || (Alignment < 4 && HasUnalignedAccess));
}
+ if (AddressSpace == AMDGPUAS::BUFFER_FAT_POINTER) {
+ auto [NoWrapFromMax, NoPartialOOB] =
+ getKnownBufferOobProperties(*ST, DL, I, Bytes);
+ MaxBytes = AMDGPU::getMaxBufferIntrinsicWidth(*ST, DL, Src, Alignment,
+ NoWrapFromMax, NoPartialOOB) /
+ 8;
+ }
if (Alignment < 4 && !HasUnalignedAccess)
MaxBytes = std::min<unsigned>(MaxBytes, Alignment.value());
- bool MergesDwords = ST->hasFlatInstOffsets() &&
- (AMDGPU::isExtendedGlobalAddrSpace(AddressSpace) ||
- (AddressSpace == AMDGPUAS::FLAT_ADDRESS &&
- !ST->hasFlatSegmentOffsetBug()));
+ bool MergesDwords = AddressSpace == AMDGPUAS::BUFFER_FAT_POINTER ||
+ (ST->hasFlatInstOffsets() &&
+ (AMDGPU::isExtendedGlobalAddrSpace(AddressSpace) ||
+ (AddressSpace == AMDGPUAS::FLAT_ADDRESS &&
+ !ST->hasFlatSegmentOffsetBug())));
- if (Opcode == Instruction::Load) {
+ if (Opcode == Instruction::Load &&
+ AddressSpace != AMDGPUAS::BUFFER_FAT_POINTER) {
if (VecTy && Alignment >= PowerOf2Ceil(Bytes))
return divideCeil(Bytes, MaxBytes);
if (EltBits <= 16 && Alignment >= 4 && Bytes % 4) {
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/load.ll b/llvm/test/Analysis/CostModel/AMDGPU/load.ll
index b36de41ef688ff..b94c98bc461b43 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/load.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/load.ll
@@ -765,21 +765,21 @@ define void @loads_addrspace_7(i32 %arg) {
; CHECK-LABEL: 'loads_addrspace_7'
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i32, ptr addrspace(7) poison, align 4
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load i64, ptr addrspace(7) poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <2 x i32>, ptr addrspace(7) poison, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <2 x i32>, ptr addrspace(7) poison, align 4
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <2 x i32>, ptr addrspace(7) poison, align 8
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load <3 x i32>, ptr addrspace(7) poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <3 x i32>, ptr addrspace(7) poison, align 8
+; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %5 = load <3 x i32>, ptr addrspace(7) poison, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = load <3 x i32>, ptr addrspace(7) poison, align 8
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <3 x i32>, ptr addrspace(7) poison, align 16
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <4 x i32>, ptr addrspace(7) poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <4 x i32>, ptr addrspace(7) poison, align 8
+; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %8 = load <4 x i32>, ptr addrspace(7) poison, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %9 = load <4 x i32>, ptr addrspace(7) poison, align 8
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <4 x i32>, ptr addrspace(7) poison, align 16
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <2 x i64>, ptr addrspace(7) poison, align 8
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = load <8 x i32>, ptr addrspace(7) poison, align 32
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %13 = load <4 x i8>, ptr addrspace(7) poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %14 = load <2 x i16>, ptr addrspace(7) poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %15 = load <8 x half>, ptr addrspace(7) poison, align 16
-; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %16 = load <4 x i24>, ptr addrspace(7) poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %17 = load <3 x i12>, ptr addrspace(7) poison, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %11 = load <2 x i64>, ptr addrspace(7) poison, align 8
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %12 = load <8 x i32>, ptr addrspace(7) poison, align 32
+; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %13 = load <4 x i8>, ptr addrspace(7) poison, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %14 = load <2 x i16>, ptr addrspace(7) poison, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %15 = load <8 x half>, ptr addrspace(7) poison, align 16
+; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %16 = load <4 x i24>, ptr addrspace(7) poison, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %17 = load <3 x i12>, ptr addrspace(7) poison, align 4
; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
load i32, ptr addrspace(7) poison, align 4
@@ -802,16 +802,47 @@ define void @loads_addrspace_7(i32 %arg) {
ret void
}
+define void @loads_addrspace_7_known_bounds(ptr addrspace(1) %base, ptr addrspace(8) %rsrc) {
+; CHECK-LABEL: 'loads_addrspace_7_known_bounds'
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <4 x i32>, ptr addrspace(7) %p, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <16 x i8>, ptr addrspace(7) %p, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <8 x i16>, ptr addrspace(7) %p, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %tail = getelementptr i8, ptr addrspace(7) %p, i32 1012
+; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %4 = load <16 x i8>, ptr addrspace(7) %tail, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %end = getelementptr i8, ptr addrspace(7) %p, i32 1024
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load <16 x i8>, ptr addrspace(7) %end, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <4 x i32>, ptr addrspace(7) %q, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %7 = load <16 x i8>, ptr addrspace(7) %q, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+ %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744)
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ load <4 x i32>, ptr addrspace(7) %p, align 4
+ load <16 x i8>, ptr addrspace(7) %p, align 4
+ load <8 x i16>, ptr addrspace(7) %p, align 4
+ %tail = getelementptr i8, ptr addrspace(7) %p, i32 1012
+ load <16 x i8>, ptr addrspace(7) %tail, align 4
+ %end = getelementptr i8, ptr addrspace(7) %p, i32 1024
+ load <16 x i8>, ptr addrspace(7) %end, align 4
+ %q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7)
+ load <4 x i32>, ptr addrspace(7) %q, align 4
+ load <16 x i8>, ptr addrspace(7) %q, align 4
+ ret void
+}
+
define void @loads_addrspace_7_unknown_offset(ptr addrspace(7) %p) {
; CHECK-LABEL: 'loads_addrspace_7_unknown_offset'
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <4 x i32>, ptr addrspace(7) %p, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i64>, ptr addrspace(7) %p, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <2 x i64>, ptr addrspace(7) %p, align 8
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <3 x i32>, ptr addrspace(7) %p, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load <8 x i32>, ptr addrspace(7) %p, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <3 x double>, ptr addrspace(7) %p, align 8
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <8 x i16>, ptr addrspace(7) %p, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <16 x i8>, ptr addrspace(7) %p, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %1 = load <4 x i32>, ptr addrspace(7) %p, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %2 = load <2 x i64>, ptr addrspace(7) %p, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <2 x i64>, ptr addrspace(7) %p, align 8
+; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %4 = load <3 x i32>, ptr addrspace(7) %p, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %5 = load <8 x i32>, ptr addrspace(7) %p, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %6 = load <3 x double>, ptr addrspace(7) %p, align 8
+; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %7 = load <8 x i16>, ptr addrspace(7) %p, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %8 = load <16 x i8>, ptr addrspace(7) %p, align 4
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <4 x i32>, ptr addrspace(7) %p, align 16
; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/store.ll b/llvm/test/Analysis/CostModel/AMDGPU/store.ll
index 7465f1bee27ced..99b7c717742aaf 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/store.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/store.ll
@@ -765,21 +765,21 @@ define void @stores_addrspace_7(i32 %arg) {
; CHECK-LABEL: 'stores_addrspace_7'
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr addrspace(7) poison, align 4
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i64 poison, ptr addrspace(7) poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr addrspace(7) poison, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i32> poison, ptr addrspace(7) poison, align 4
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr addrspace(7) poison, align 8
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(7) poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(7) poison, align 8
+; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store <3 x i32> poison, ptr addrspace(7) poison, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i32> poison, ptr addrspace(7) poison, align 8
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(7) poison, align 16
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) poison, align 8
+; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr addrspace(7) poison, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i32> poison, ptr addrspace(7) poison, align 8
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) poison, align 16
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i64> poison, ptr addrspace(7) poison, align 8
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <8 x i32> poison, ptr addrspace(7) poison, align 32
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr addrspace(7) poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr addrspace(7) poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <8 x half> poison, ptr addrspace(7) poison, align 16
-; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: store <4 x i24> poison, ptr addrspace(7) poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <3 x i12> poison, ptr addrspace(7) poison, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i64> poison, ptr addrspace(7) poison, align 8
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <8 x i32> poison, ptr addrspace(7) poison, align 32
+; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i8> poison, ptr addrspace(7) poison, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i16> poison, ptr addrspace(7) poison, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: store <8 x half> poison, ptr addrspace(7) poison, align 16
+; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: store <4 x i24> poison, ptr addrspace(7) poison, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i12> poison, ptr addrspace(7) poison, align 4
; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
store i32 poison, ptr addrspace(7) poison, align 4
@@ -802,16 +802,47 @@ define void @stores_addrspace_7(i32 %arg) {
ret void
}
-define void @stores_addrspace_7_unknown_offset(ptr addrspace(7) %p) {
-; CHECK-LABEL: 'stores_addrspace_7_unknown_offset'
+define void @stores_addrspace_7_known_bounds(ptr addrspace(1) %base, ptr addrspace(8) %rsrc) {
+; CHECK-LABEL: 'stores_addrspace_7_known_bounds'
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) %p, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i64> poison, ptr addrspace(7) %p, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i64> poison, ptr addrspace(7) %p, align 8
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(7) %p, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <8 x i32> poison, ptr addrspace(7) %p, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x double> poison, ptr addrspace(7) %p, align 8
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> poison, ptr addrspace(7) %p, align 4
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %p, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> poison, ptr addrspace(7) %p, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %tail = getelementptr i8, ptr addrspace(7) %p, i32 1012
+; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> poison, ptr addrspace(7) %tail, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %end = getelementptr i8, ptr addrspace(7) %p, i32 1024
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %end, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) %q, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> poison, ptr addrspace(7) %q, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+ %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744)
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ store <4 x i32> poison, ptr addrspace(7) %p, align 4
+ store <16 x i8> poison, ptr addrspace(7) %p, align 4
+ store <8 x i16> poison, ptr addrspace(7) %p, align 4
+ %tail = getelementptr i8, ptr addrspace(7) %p, i32 1012
+ store <16 x i8> poison, ptr addrspace(7) %tail, align 4
+ %end = getelementptr i8, ptr addrspace(7) %p, i32 1024
+ store <16 x i8> poison, ptr addrspace(7) %end, align 4
+ %q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7)
+ store <4 x i32> poison, ptr addrspace(7) %q, align 4
+ store <16 x i8> poison, ptr addrspace(7) %q, align 4
+ ret void
+}
+
+define void @stores_addrspace_7_unknown_offset(ptr addrspace(7) %p) {
+; CHECK-LABEL: 'stores_addrspace_7_unknown_offset'
+; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr addrspace(7) %p, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <2 x i64> poison, ptr addrspace(7) %p, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i64> poison, ptr addrspace(7) %p, align 8
+; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store <3 x i32> poison, ptr addrspace(7) %p, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: store <8 x i32> poison, ptr addrspace(7) %p, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store <3 x double> poison, ptr addrspace(7) %p, align 8
+; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: store <8 x i16> poison, ptr addrspace(7) %p, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> poison, ptr addrspace(7) %p, align 4
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) %p, align 16
; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
>From a5dc7d571eff5e56f7c174b3feff5fabfa44a582 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Wed, 4 Nov 2026 23:07:13 +0100
Subject: [PATCH 2/2] apply comments
---
.../AMDGPU/AMDGPULowerBufferFatPointers.cpp | 6 +-
.../AMDGPU/AMDGPUTargetTransformInfo.cpp | 12 +-
.../AMDGPU/buffer-fat-pointer-oob-mode.ll | 354 ++++++++++++++++++
llvm/test/Analysis/CostModel/AMDGPU/load.ll | 22 +-
llvm/test/Analysis/CostModel/AMDGPU/store.ll | 22 +-
5 files changed, 407 insertions(+), 9 deletions(-)
create mode 100644 llvm/test/Analysis/CostModel/AMDGPU/buffer-fat-pointer-oob-mode.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
index ca4b987f8a8e1d..3b9ad2971ae127 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
@@ -938,14 +938,14 @@ LegalizeBufferContentTypesVisitor::analyzeOobProperties(Value *Ptr, Type *Ty,
/// i32>, and if we load from an align(2) address, that address might be 2
/// bytes from the end of the buffer. The hardware will, when performing the
/// <2 x i32> load, mask off the entire first word, causing the two in-bounds
-/// bytes to be masked off. However,if we know the offset can't be too close
+/// bytes to be masked off. However, if we know the offset can't be too close
/// to the number of records in the buffer (if known), we can skip this
/// expansion.
///
/// Unlike the complete disablement of unaligned accesses from point 1,
/// this does not apply to unaligned scalars, but will apply to cases like
-/// `load <2 x i32>, align 4` since the left elemenvt might be out of bounds.
-/// Note that if the we know that the base offset is known to be
+/// `load <2 x i32>, align 4` since the left element might be out of bounds.
+/// Note that if we know that the base offset is known to be
/// less than `uint32_max - byte_size(Ty)`, we can skip these alignment
/// checks.
uint64_t AMDGPU::getMaxBufferIntrinsicWidth(const GCNSubtarget &ST,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index e59f7aff6e27df..b8702707380e9b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -2222,9 +2222,9 @@ static std::pair<bool, bool> getKnownBufferOobProperties(const GCNSubtarget &ST,
Ptr->getType()->getPointerAddressSpace() != AMDGPUAS::BUFFER_FAT_POINTER)
return {false, false};
APInt Offset(DL.getIndexTypeSizeInBits(Ptr->getType()), 0);
+ bool KnownOffset = true;
while (const auto *GEP = dyn_cast<GEPOperator>(Ptr)) {
- if (!GEP->accumulateConstantOffset(DL, Offset))
- return {false, false};
+ KnownOffset = KnownOffset && GEP->accumulateConstantOffset(DL, Offset);
Ptr = GEP->getPointerOperand();
}
const auto *Cast = dyn_cast<AddrSpaceCastInst>(Ptr);
@@ -2240,6 +2240,7 @@ static std::pair<bool, bool> getKnownBufferOobProperties(const GCNSubtarget &ST,
return {false, false};
bool NoWrapFromMax =
+ KnownOffset &&
Offset.ule(APInt::getAllOnes(Offset.getBitWidth()) - Bytes);
const auto *NumRecords =
Rsrc ? dyn_cast<ConstantInt>(Rsrc->getArgOperand(2)) : nullptr;
@@ -2248,9 +2249,12 @@ static std::pair<bool, bool> getKnownBufferOobProperties(const GCNSubtarget &ST,
if (!NumRecords || !NumRecordsWidth)
return {NoWrapFromMax, false};
APInt Bound = NumRecords->getValue().zextOrTrunc(*NumRecordsWidth);
+ if (Bound.isAllOnes())
+ return {NoWrapFromMax, true};
+ if (!KnownOffset)
+ return {false, false};
APInt BoundsDiff = Bound.zext(64) - Offset.zext(64);
- return {NoWrapFromMax, Bound.isAllOnes() || BoundsDiff.sge(Bytes) ||
- BoundsDiff.isNonPositive()};
+ return {NoWrapFromMax, BoundsDiff.sge(Bytes) || BoundsDiff.isNonPositive()};
}
InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/buffer-fat-pointer-oob-mode.ll b/llvm/test/Analysis/CostModel/AMDGPU/buffer-fat-pointer-oob-mode.ll
new file mode 100644
index 00000000000000..80811bfaf3f2ef
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AMDGPU/buffer-fat-pointer-oob-mode.ll
@@ -0,0 +1,354 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: sed 's/BUFFER_OOB_MODE/2/' %s | opt -passes="print<cost-model>" -disable-output -mtriple=amdgpu9.0a-unknown-amdhsa 2>&1 | FileCheck %s --check-prefix=STRICT
+; RUN: sed 's/BUFFER_OOB_MODE/1/' %s | opt -passes="print<cost-model>" -disable-output -mtriple=amdgpu9.0a-unknown-amdhsa 2>&1 | FileCheck %s --check-prefix=RELAXED
+
+define void @loads() {
+; STRICT-LABEL: 'loads'
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i32, ptr addrspace(7) poison, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load i64, ptr addrspace(7) poison, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <2 x i32>, ptr addrspace(7) poison, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <2 x i32>, ptr addrspace(7) poison, align 8
+; STRICT-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %5 = load <3 x i32>, ptr addrspace(7) poison, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = load <3 x i32>, ptr addrspace(7) poison, align 8
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <3 x i32>, ptr addrspace(7) poison, align 16
+; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %8 = load <4 x i32>, ptr addrspace(7) poison, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %9 = load <4 x i32>, ptr addrspace(7) poison, align 8
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <4 x i32>, ptr addrspace(7) poison, align 16
+; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %11 = load <2 x i64>, ptr addrspace(7) poison, align 8
+; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %12 = load <8 x i32>, ptr addrspace(7) poison, align 32
+; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %13 = load <4 x i8>, ptr addrspace(7) poison, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %14 = load <2 x i16>, ptr addrspace(7) poison, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %15 = load <8 x half>, ptr addrspace(7) poison, align 16
+; STRICT-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %16 = load <4 x i24>, ptr addrspace(7) poison, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %17 = load <3 x i12>, ptr addrspace(7) poison, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; RELAXED-LABEL: 'loads'
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i32, ptr addrspace(7) poison, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load i64, ptr addrspace(7) poison, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <2 x i32>, ptr addrspace(7) poison, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <2 x i32>, ptr addrspace(7) poison, align 8
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load <3 x i32>, ptr addrspace(7) poison, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <3 x i32>, ptr addrspace(7) poison, align 8
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <3 x i32>, ptr addrspace(7) poison, align 16
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <4 x i32>, ptr addrspace(7) poison, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <4 x i32>, ptr addrspace(7) poison, align 8
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <4 x i32>, ptr addrspace(7) poison, align 16
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <2 x i64>, ptr addrspace(7) poison, align 8
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %12 = load <8 x i32>, ptr addrspace(7) poison, align 32
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %13 = load <4 x i8>, ptr addrspace(7) poison, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %14 = load <2 x i16>, ptr addrspace(7) poison, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %15 = load <8 x half>, ptr addrspace(7) poison, align 16
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %16 = load <4 x i24>, ptr addrspace(7) poison, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %17 = load <3 x i12>, ptr addrspace(7) poison, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+ load i32, ptr addrspace(7) poison, align 4
+ load i64, ptr addrspace(7) poison, align 4
+ load <2 x i32>, ptr addrspace(7) poison, align 4
+ load <2 x i32>, ptr addrspace(7) poison, align 8
+ load <3 x i32>, ptr addrspace(7) poison, align 4
+ load <3 x i32>, ptr addrspace(7) poison, align 8
+ load <3 x i32>, ptr addrspace(7) poison, align 16
+ load <4 x i32>, ptr addrspace(7) poison, align 4
+ load <4 x i32>, ptr addrspace(7) poison, align 8
+ load <4 x i32>, ptr addrspace(7) poison, align 16
+ load <2 x i64>, ptr addrspace(7) poison, align 8
+ load <8 x i32>, ptr addrspace(7) poison, align 32
+ load <4 x i8>, ptr addrspace(7) poison, align 4
+ load <2 x i16>, ptr addrspace(7) poison, align 4
+ load <8 x half>, ptr addrspace(7) poison, align 16
+ load <4 x i24>, ptr addrspace(7) poison, align 4
+ load <3 x i12>, ptr addrspace(7) poison, align 4
+ ret void
+}
+
+define void @loads_known_bounds(ptr addrspace(1) %base, ptr addrspace(8) %rsrc, i32 %off) {
+; STRICT-LABEL: 'loads_known_bounds'
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744)
+; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <4 x i32>, ptr addrspace(7) %p, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <16 x i8>, ptr addrspace(7) %p, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <8 x i16>, ptr addrspace(7) %p, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %tail = getelementptr i8, ptr addrspace(7) %p, i32 1012
+; STRICT-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %4 = load <16 x i8>, ptr addrspace(7) %tail, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %end = getelementptr i8, ptr addrspace(7) %p, i32 1024
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load <16 x i8>, ptr addrspace(7) %end, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7)
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <4 x i32>, ptr addrspace(7) %q, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %7 = load <16 x i8>, ptr addrspace(7) %q, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %ubuf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 -1, i32 159744)
+; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %u = addrspacecast ptr addrspace(8) %ubuf to ptr addrspace(7)
+; STRICT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %u.var = getelementptr i8, ptr addrspace(7) %u, i32 %off
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <16 x i8>, ptr addrspace(7) %u.var, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %u.var2 = getelementptr i8, ptr addrspace(7) %u, i32 %off
+; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %9 = load <4 x i32>, ptr addrspace(7) %u.var2, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %p.var = getelementptr i8, ptr addrspace(7) %p, i32 %off
+; STRICT-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %10 = load <16 x i8>, ptr addrspace(7) %p.var, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %direct = call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744)
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <16 x i8>, ptr addrspace(7) %direct, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; RELAXED-LABEL: 'loads_known_bounds'
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744)
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <4 x i32>, ptr addrspace(7) %p, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <16 x i8>, ptr addrspace(7) %p, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <8 x i16>, ptr addrspace(7) %p, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %tail = getelementptr i8, ptr addrspace(7) %p, i32 1012
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <16 x i8>, ptr addrspace(7) %tail, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %end = getelementptr i8, ptr addrspace(7) %p, i32 1024
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load <16 x i8>, ptr addrspace(7) %end, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7)
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <4 x i32>, ptr addrspace(7) %q, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <16 x i8>, ptr addrspace(7) %q, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %ubuf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 -1, i32 159744)
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %u = addrspacecast ptr addrspace(8) %ubuf to ptr addrspace(7)
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %u.var = getelementptr i8, ptr addrspace(7) %u, i32 %off
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <16 x i8>, ptr addrspace(7) %u.var, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %u.var2 = getelementptr i8, ptr addrspace(7) %u, i32 %off
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <4 x i32>, ptr addrspace(7) %u.var2, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %p.var = getelementptr i8, ptr addrspace(7) %p, i32 %off
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <16 x i8>, ptr addrspace(7) %p.var, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %direct = call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744)
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <16 x i8>, ptr addrspace(7) %direct, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+ %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744)
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ load <4 x i32>, ptr addrspace(7) %p, align 4
+ load <16 x i8>, ptr addrspace(7) %p, align 4
+ load <8 x i16>, ptr addrspace(7) %p, align 4
+ %tail = getelementptr i8, ptr addrspace(7) %p, i32 1012
+ load <16 x i8>, ptr addrspace(7) %tail, align 4
+ %end = getelementptr i8, ptr addrspace(7) %p, i32 1024
+ load <16 x i8>, ptr addrspace(7) %end, align 4
+ %q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7)
+ load <4 x i32>, ptr addrspace(7) %q, align 4
+ load <16 x i8>, ptr addrspace(7) %q, align 4
+ %ubuf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 -1, i32 159744)
+ %u = addrspacecast ptr addrspace(8) %ubuf to ptr addrspace(7)
+ %u.var = getelementptr i8, ptr addrspace(7) %u, i32 %off
+ load <16 x i8>, ptr addrspace(7) %u.var, align 4
+ %u.var2 = getelementptr i8, ptr addrspace(7) %u, i32 %off
+ load <4 x i32>, ptr addrspace(7) %u.var2, align 4
+ %p.var = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ load <16 x i8>, ptr addrspace(7) %p.var, align 4
+ %direct = call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744)
+ load <16 x i8>, ptr addrspace(7) %direct, align 4
+ ret void
+}
+
+define void @loads_unknown_offset(ptr addrspace(7) %p) {
+; STRICT-LABEL: 'loads_unknown_offset'
+; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %1 = load <4 x i32>, ptr addrspace(7) %p, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %2 = load <2 x i64>, ptr addrspace(7) %p, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <2 x i64>, ptr addrspace(7) %p, align 8
+; STRICT-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %4 = load <3 x i32>, ptr addrspace(7) %p, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %5 = load <8 x i32>, ptr addrspace(7) %p, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %6 = load <3 x double>, ptr addrspace(7) %p, align 8
+; STRICT-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %7 = load <8 x i16>, ptr addrspace(7) %p, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %8 = load <16 x i8>, ptr addrspace(7) %p, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <4 x i32>, ptr addrspace(7) %p, align 16
+; STRICT-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; RELAXED-LABEL: 'loads_unknown_offset'
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <4 x i32>, ptr addrspace(7) %p, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i64>, ptr addrspace(7) %p, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <2 x i64>, ptr addrspace(7) %p, align 8
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <3 x i32>, ptr addrspace(7) %p, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = load <8 x i32>, ptr addrspace(7) %p, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = load <3 x double>, ptr addrspace(7) %p, align 8
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <8 x i16>, ptr addrspace(7) %p, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <16 x i8>, ptr addrspace(7) %p, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <4 x i32>, ptr addrspace(7) %p, align 16
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+ load <4 x i32>, ptr addrspace(7) %p, align 4
+ load <2 x i64>, ptr addrspace(7) %p, align 4
+ load <2 x i64>, ptr addrspace(7) %p, align 8
+ load <3 x i32>, ptr addrspace(7) %p, align 4
+ load <8 x i32>, ptr addrspace(7) %p, align 4
+ load <3 x double>, ptr addrspace(7) %p, align 8
+ load <8 x i16>, ptr addrspace(7) %p, align 4
+ load <16 x i8>, ptr addrspace(7) %p, align 4
+ load <4 x i32>, ptr addrspace(7) %p, align 16
+ ret void
+}
+
+define void @stores() {
+; STRICT-LABEL: 'stores'
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr addrspace(7) poison, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i64 poison, ptr addrspace(7) poison, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i32> poison, ptr addrspace(7) poison, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr addrspace(7) poison, align 8
+; STRICT-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store <3 x i32> poison, ptr addrspace(7) poison, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i32> poison, ptr addrspace(7) poison, align 8
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(7) poison, align 16
+; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr addrspace(7) poison, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i32> poison, ptr addrspace(7) poison, align 8
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) poison, align 16
+; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i64> poison, ptr addrspace(7) poison, align 8
+; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <8 x i32> poison, ptr addrspace(7) poison, align 32
+; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i8> poison, ptr addrspace(7) poison, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i16> poison, ptr addrspace(7) poison, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 8 for instruction: store <8 x half> poison, ptr addrspace(7) poison, align 16
+; STRICT-NEXT: Cost Model: Found an estimated cost of 12 for instruction: store <4 x i24> poison, ptr addrspace(7) poison, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i12> poison, ptr addrspace(7) poison, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; RELAXED-LABEL: 'stores'
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr addrspace(7) poison, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i64 poison, ptr addrspace(7) poison, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr addrspace(7) poison, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr addrspace(7) poison, align 8
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(7) poison, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(7) poison, align 8
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(7) poison, align 16
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) poison, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) poison, align 8
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) poison, align 16
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i64> poison, ptr addrspace(7) poison, align 8
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <8 x i32> poison, ptr addrspace(7) poison, align 32
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr addrspace(7) poison, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr addrspace(7) poison, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <8 x half> poison, ptr addrspace(7) poison, align 16
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i24> poison, ptr addrspace(7) poison, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i12> poison, ptr addrspace(7) poison, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+ store i32 poison, ptr addrspace(7) poison, align 4
+ store i64 poison, ptr addrspace(7) poison, align 4
+ store <2 x i32> poison, ptr addrspace(7) poison, align 4
+ store <2 x i32> poison, ptr addrspace(7) poison, align 8
+ store <3 x i32> poison, ptr addrspace(7) poison, align 4
+ store <3 x i32> poison, ptr addrspace(7) poison, align 8
+ store <3 x i32> poison, ptr addrspace(7) poison, align 16
+ store <4 x i32> poison, ptr addrspace(7) poison, align 4
+ store <4 x i32> poison, ptr addrspace(7) poison, align 8
+ store <4 x i32> poison, ptr addrspace(7) poison, align 16
+ store <2 x i64> poison, ptr addrspace(7) poison, align 8
+ store <8 x i32> poison, ptr addrspace(7) poison, align 32
+ store <4 x i8> poison, ptr addrspace(7) poison, align 4
+ store <2 x i16> poison, ptr addrspace(7) poison, align 4
+ store <8 x half> poison, ptr addrspace(7) poison, align 16
+ store <4 x i24> poison, ptr addrspace(7) poison, align 4
+ store <3 x i12> poison, ptr addrspace(7) poison, align 4
+ ret void
+}
+
+define void @stores_known_bounds(ptr addrspace(1) %base, ptr addrspace(8) %rsrc, i32 %off) {
+; STRICT-LABEL: 'stores_known_bounds'
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744)
+; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) %p, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %p, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> poison, ptr addrspace(7) %p, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %tail = getelementptr i8, ptr addrspace(7) %p, i32 1012
+; STRICT-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> poison, ptr addrspace(7) %tail, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %end = getelementptr i8, ptr addrspace(7) %p, i32 1024
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %end, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7)
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) %q, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> poison, ptr addrspace(7) %q, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %ubuf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 -1, i32 159744)
+; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %u = addrspacecast ptr addrspace(8) %ubuf to ptr addrspace(7)
+; STRICT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %u.var = getelementptr i8, ptr addrspace(7) %u, i32 %off
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %u.var, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %u.var2 = getelementptr i8, ptr addrspace(7) %u, i32 %off
+; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr addrspace(7) %u.var2, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %p.var = getelementptr i8, ptr addrspace(7) %p, i32 %off
+; STRICT-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> poison, ptr addrspace(7) %p.var, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %direct = call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744)
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %direct, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; RELAXED-LABEL: 'stores_known_bounds'
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744)
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) %p, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %p, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> poison, ptr addrspace(7) %p, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %tail = getelementptr i8, ptr addrspace(7) %p, i32 1012
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %tail, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %end = getelementptr i8, ptr addrspace(7) %p, i32 1024
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %end, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7)
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) %q, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %q, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %ubuf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 -1, i32 159744)
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %u = addrspacecast ptr addrspace(8) %ubuf to ptr addrspace(7)
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %u.var = getelementptr i8, ptr addrspace(7) %u, i32 %off
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %u.var, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %u.var2 = getelementptr i8, ptr addrspace(7) %u, i32 %off
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) %u.var2, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %p.var = getelementptr i8, ptr addrspace(7) %p, i32 %off
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %p.var, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %direct = call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744)
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %direct, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+ %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744)
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ store <4 x i32> poison, ptr addrspace(7) %p, align 4
+ store <16 x i8> poison, ptr addrspace(7) %p, align 4
+ store <8 x i16> poison, ptr addrspace(7) %p, align 4
+ %tail = getelementptr i8, ptr addrspace(7) %p, i32 1012
+ store <16 x i8> poison, ptr addrspace(7) %tail, align 4
+ %end = getelementptr i8, ptr addrspace(7) %p, i32 1024
+ store <16 x i8> poison, ptr addrspace(7) %end, align 4
+ %q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7)
+ store <4 x i32> poison, ptr addrspace(7) %q, align 4
+ store <16 x i8> poison, ptr addrspace(7) %q, align 4
+ %ubuf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 -1, i32 159744)
+ %u = addrspacecast ptr addrspace(8) %ubuf to ptr addrspace(7)
+ %u.var = getelementptr i8, ptr addrspace(7) %u, i32 %off
+ store <16 x i8> poison, ptr addrspace(7) %u.var, align 4
+ %u.var2 = getelementptr i8, ptr addrspace(7) %u, i32 %off
+ store <4 x i32> poison, ptr addrspace(7) %u.var2, align 4
+ %p.var = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store <16 x i8> poison, ptr addrspace(7) %p.var, align 4
+ %direct = call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744)
+ store <16 x i8> poison, ptr addrspace(7) %direct, align 4
+ ret void
+}
+
+define void @stores_unknown_offset(ptr addrspace(7) %p) {
+; STRICT-LABEL: 'stores_unknown_offset'
+; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr addrspace(7) %p, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <2 x i64> poison, ptr addrspace(7) %p, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i64> poison, ptr addrspace(7) %p, align 8
+; STRICT-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store <3 x i32> poison, ptr addrspace(7) %p, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 8 for instruction: store <8 x i32> poison, ptr addrspace(7) %p, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store <3 x double> poison, ptr addrspace(7) %p, align 8
+; STRICT-NEXT: Cost Model: Found an estimated cost of 8 for instruction: store <8 x i16> poison, ptr addrspace(7) %p, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> poison, ptr addrspace(7) %p, align 4
+; STRICT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) %p, align 16
+; STRICT-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; RELAXED-LABEL: 'stores_unknown_offset'
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) %p, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i64> poison, ptr addrspace(7) %p, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i64> poison, ptr addrspace(7) %p, align 8
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(7) %p, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <8 x i32> poison, ptr addrspace(7) %p, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x double> poison, ptr addrspace(7) %p, align 8
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> poison, ptr addrspace(7) %p, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %p, align 4
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) %p, align 16
+; RELAXED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+ store <4 x i32> poison, ptr addrspace(7) %p, align 4
+ store <2 x i64> poison, ptr addrspace(7) %p, align 4
+ store <2 x i64> poison, ptr addrspace(7) %p, align 8
+ store <3 x i32> poison, ptr addrspace(7) %p, align 4
+ store <8 x i32> poison, ptr addrspace(7) %p, align 4
+ store <3 x double> poison, ptr addrspace(7) %p, align 8
+ store <8 x i16> poison, ptr addrspace(7) %p, align 4
+ store <16 x i8> poison, ptr addrspace(7) %p, align 4
+ store <4 x i32> poison, ptr addrspace(7) %p, align 16
+ ret void
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 7, !"amdgpu.buffer.oob.mode", i32 BUFFER_OOB_MODE}
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/load.ll b/llvm/test/Analysis/CostModel/AMDGPU/load.ll
index b94c98bc461b43..ed800d3d458400 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/load.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/load.ll
@@ -802,7 +802,7 @@ define void @loads_addrspace_7(i32 %arg) {
ret void
}
-define void @loads_addrspace_7_known_bounds(ptr addrspace(1) %base, ptr addrspace(8) %rsrc) {
+define void @loads_addrspace_7_known_bounds(ptr addrspace(1) %base, ptr addrspace(8) %rsrc, i32 %off) {
; CHECK-LABEL: 'loads_addrspace_7_known_bounds'
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744)
; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -816,6 +816,16 @@ define void @loads_addrspace_7_known_bounds(ptr addrspace(1) %base, ptr addrspac
; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7)
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <4 x i32>, ptr addrspace(7) %q, align 4
; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %7 = load <16 x i8>, ptr addrspace(7) %q, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %ubuf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 -1, i32 159744)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %u = addrspacecast ptr addrspace(8) %ubuf to ptr addrspace(7)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %u.var = getelementptr i8, ptr addrspace(7) %u, i32 %off
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <16 x i8>, ptr addrspace(7) %u.var, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %u.var2 = getelementptr i8, ptr addrspace(7) %u, i32 %off
+; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %9 = load <4 x i32>, ptr addrspace(7) %u.var2, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %p.var = getelementptr i8, ptr addrspace(7) %p, i32 %off
+; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %10 = load <16 x i8>, ptr addrspace(7) %p.var, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %direct = call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <16 x i8>, ptr addrspace(7) %direct, align 4
; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
%buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744)
@@ -830,6 +840,16 @@ define void @loads_addrspace_7_known_bounds(ptr addrspace(1) %base, ptr addrspac
%q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7)
load <4 x i32>, ptr addrspace(7) %q, align 4
load <16 x i8>, ptr addrspace(7) %q, align 4
+ %ubuf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 -1, i32 159744)
+ %u = addrspacecast ptr addrspace(8) %ubuf to ptr addrspace(7)
+ %u.var = getelementptr i8, ptr addrspace(7) %u, i32 %off
+ load <16 x i8>, ptr addrspace(7) %u.var, align 4
+ %u.var2 = getelementptr i8, ptr addrspace(7) %u, i32 %off
+ load <4 x i32>, ptr addrspace(7) %u.var2, align 4
+ %p.var = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ load <16 x i8>, ptr addrspace(7) %p.var, align 4
+ %direct = call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744)
+ load <16 x i8>, ptr addrspace(7) %direct, align 4
ret void
}
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/store.ll b/llvm/test/Analysis/CostModel/AMDGPU/store.ll
index 99b7c717742aaf..eb992b307ee576 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/store.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/store.ll
@@ -802,7 +802,7 @@ define void @stores_addrspace_7(i32 %arg) {
ret void
}
-define void @stores_addrspace_7_known_bounds(ptr addrspace(1) %base, ptr addrspace(8) %rsrc) {
+define void @stores_addrspace_7_known_bounds(ptr addrspace(1) %base, ptr addrspace(8) %rsrc, i32 %off) {
; CHECK-LABEL: 'stores_addrspace_7_known_bounds'
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744)
; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -816,6 +816,16 @@ define void @stores_addrspace_7_known_bounds(ptr addrspace(1) %base, ptr addrspa
; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7)
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(7) %q, align 4
; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> poison, ptr addrspace(7) %q, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %ubuf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 -1, i32 159744)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %u = addrspacecast ptr addrspace(8) %ubuf to ptr addrspace(7)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %u.var = getelementptr i8, ptr addrspace(7) %u, i32 %off
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %u.var, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %u.var2 = getelementptr i8, ptr addrspace(7) %u, i32 %off
+; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr addrspace(7) %u.var2, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %p.var = getelementptr i8, ptr addrspace(7) %p, i32 %off
+; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> poison, ptr addrspace(7) %p.var, align 4
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %direct = call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(7) %direct, align 4
; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
%buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744)
@@ -830,6 +840,16 @@ define void @stores_addrspace_7_known_bounds(ptr addrspace(1) %base, ptr addrspa
%q = addrspacecast ptr addrspace(8) %rsrc to ptr addrspace(7)
store <4 x i32> poison, ptr addrspace(7) %q, align 4
store <16 x i8> poison, ptr addrspace(7) %q, align 4
+ %ubuf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1.i64(ptr addrspace(1) %base, i16 0, i64 -1, i32 159744)
+ %u = addrspacecast ptr addrspace(8) %ubuf to ptr addrspace(7)
+ %u.var = getelementptr i8, ptr addrspace(7) %u, i32 %off
+ store <16 x i8> poison, ptr addrspace(7) %u.var, align 4
+ %u.var2 = getelementptr i8, ptr addrspace(7) %u, i32 %off
+ store <4 x i32> poison, ptr addrspace(7) %u.var2, align 4
+ %p.var = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store <16 x i8> poison, ptr addrspace(7) %p.var, align 4
+ %direct = call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1.i64(ptr addrspace(1) %base, i16 0, i64 1024, i32 159744)
+ store <16 x i8> poison, ptr addrspace(7) %direct, align 4
ret void
}
More information about the llvm-branch-commits
mailing list