[clang] [llvm] [AMDGPU[ Expand `av` built-ins and intrinsics to b8/b16/b32/b64 (PR #229429)
Pierre van Houtryve via cfe-commits
cfe-commits at lists.llvm.org
Wed Oct 7 03:13:08 PDT 2026
https://github.com/Pierre-vh updated https://github.com/llvm/llvm-project/pull/229429
>From 5f88593d4e77b93cbe60236ff331f221cf3055cd Mon Sep 17 00:00:00 2001
From: pvanhout <pierre.vanhoutryve at amd.com>
Date: Tue, 6 Oct 2026 15:51:05 +0200
Subject: [PATCH 1/3] [AMDGPU[ Expand `av` built-ins and intrinsics to
b16/b32/b64
They previously only supported b128, but there is no technical reason why we cannot support all bit widths.
Disclaimer: This is largely Claude assisted. I ran it in short self-contained bursts (to ensure the output of each step was reviewable) to do all the boring steps of plumbing this through the stack. I also manually reverified and cleaned up all the files one by one between each steps.
Assisted-by: Claude Opus 5.5
---
clang/docs/LanguageExtensions.md | 18 +-
clang/include/clang/Basic/BuiltinsAMDGPU.td | 30 +
.../include/clang/Basic/BuiltinsAMDGPUDocs.td | 38 +-
clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp | 40 +-
clang/lib/Sema/SemaAMDGPU.cpp | 6 +
.../builtins-amdgcn-global-load-store.cl | 714 ++++++++++-
clang/test/SemaHIP/amdgpu-av-load-store.hip | 33 +
...builtins-amdgcn-global-load-store-error.cl | 58 +
...s-amdgcn-global-load-store-target-error.cl | 39 +-
llvm/docs/AMDGPUMemoryModel.md | 8 +-
llvm/docs/AMDGPUUsage.rst | 24 +-
llvm/include/llvm/IR/IntrinsicsAMDGPU.td | 21 +-
llvm/lib/IR/VerifierAMDGPU.cpp | 6 +
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 8 +-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 25 +-
llvm/test/CodeGen/AMDGPU/amdgcn-av-scopes.ll | 1062 +++++++++++++++++
.../AMDGPU/llvm.amdgcn.av.load.b128.ll | 828 +++++++++++--
.../AMDGPU/llvm.amdgcn.av.store.b128.ll | 588 +++++++++
.../CodeGen/AMDGPU/unsupported-av-load.ll | 18 +-
.../CodeGen/AMDGPU/unsupported-av-store.ll | 12 +-
llvm/test/Verifier/AMDGPU/intrinsics-av.ll | 42 +
21 files changed, 3425 insertions(+), 193 deletions(-)
diff --git a/clang/docs/LanguageExtensions.md b/clang/docs/LanguageExtensions.md
index 3db8d083c40756..cf88c678de553d 100644
--- a/clang/docs/LanguageExtensions.md
+++ b/clang/docs/LanguageExtensions.md
@@ -5496,28 +5496,34 @@ returns the bit at the position of the current lane. It is almost equivalent to
`(mask & (1 << lane_id)) != 0`, except that its behavior is only defined if
the given mask has the same value for all active lanes of the current wave.
-#### \_\_builtin_amdgcn_av\_{load,store}\_b128
+#### \_\_builtin_amdgcn_av\_{load,store}\_{b16,b32,b64,b128}
Signature:
```c
typedef __attribute__((__vector_size__(4 * sizeof(unsigned int)))) unsigned int v4u;
+unsigned short __builtin_amdgcn_av_load_b16(unsigned short *src, int scope);
+unsigned int __builtin_amdgcn_av_load_b32(unsigned int *src, int scope);
+uint64_t __builtin_amdgcn_av_load_b64(uint64_t *src, int scope);
v4u __builtin_amdgcn_av_load_b128(v4u *src, int scope);
+void __builtin_amdgcn_av_store_b16(unsigned short *dst, unsigned short data, int scope);
+void __builtin_amdgcn_av_store_b32(unsigned int *dst, unsigned int data, int scope);
+void __builtin_amdgcn_av_store_b64(uint64_t *dst, uint64_t data, int scope);
void __builtin_amdgcn_av_store_b128(v4u *dst, v4u data, int scope);
```
-Load or store a vector of 4 unsigned integers from or to memory with cache
-behavior specified by `scope`, which is one of the `__MEMORY_SCOPE_*` macros
-defined for {ref}`scoped atomic builtins <langext-c11-atomic>`.
+Load or store 16, 32, 64 or 128 bits from or to memory with cache behavior
+specified by `scope`, which is one of the `__MEMORY_SCOPE_*` macros defined for
+{ref}`scoped atomic builtins <langext-c11-atomic>`.
The pointer argument must point to the global or generic address space.
These builtins are supported on gfx9, gfx10, gfx11, and gfx12 targets.
-They map to the LLVM intrinsics `llvm.amdgcn.av.load.b128` and
-`llvm.amdgcn.av.store.b128` documented in [User Guide for AMDGPU Backend](https://llvm.org/docs/AMDGPUUsage.html).
+They map to the LLVM intrinsics `llvm.amdgcn.av.load.b{16,32,64,128}` and
+`llvm.amdgcn.av.store.b{16,32,64,128}` documented in [User Guide for AMDGPU Backend](https://llvm.org/docs/AMDGPUUsage.html).
### ARM/AArch64 Language Extensions
diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.td b/clang/include/clang/Basic/BuiltinsAMDGPU.td
index 483df0561fc20e..0d062d7fa2e43d 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.td
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.td
@@ -318,6 +318,36 @@ def __builtin_amdgcn_s_buffer_load_v4f16 : AMDGPUBuiltin<"_Vector<4, _Float16>(_
// Global Available/Visible memory accesses.
//===----------------------------------------------------------------------===//
+def __builtin_amdgcn_av_load_b16
+ : AMDGPUBuiltin<"unsigned short(unsigned short *, int)", [], "flat-global-insts"> {
+ let Documentation = [DocAVLoadB16];
+ let ArgNames = ["ptr", "scope"];
+}
+def __builtin_amdgcn_av_store_b16
+ : AMDGPUBuiltin<"void(unsigned short *, unsigned short, int)", [], "flat-global-insts"> {
+ let Documentation = [DocAVStoreB16];
+ let ArgNames = ["ptr", "data", "scope"];
+}
+def __builtin_amdgcn_av_load_b32
+ : AMDGPUBuiltin<"unsigned int(unsigned int *, int)", [], "flat-global-insts"> {
+ let Documentation = [DocAVLoadB32];
+ let ArgNames = ["ptr", "scope"];
+}
+def __builtin_amdgcn_av_store_b32
+ : AMDGPUBuiltin<"void(unsigned int *, unsigned int, int)", [], "flat-global-insts"> {
+ let Documentation = [DocAVStoreB32];
+ let ArgNames = ["ptr", "data", "scope"];
+}
+def __builtin_amdgcn_av_load_b64
+ : AMDGPUBuiltin<"uint64_t(uint64_t *, int)", [], "flat-global-insts"> {
+ let Documentation = [DocAVLoadB64];
+ let ArgNames = ["ptr", "scope"];
+}
+def __builtin_amdgcn_av_store_b64
+ : AMDGPUBuiltin<"void(uint64_t *, uint64_t, int)", [], "flat-global-insts"> {
+ let Documentation = [DocAVStoreB64];
+ let ArgNames = ["ptr", "data", "scope"];
+}
def __builtin_amdgcn_av_load_b128
: AMDGPUBuiltin<"_ExtVector<4, unsigned int>(_ExtVector<4, unsigned int> *, int)", [], "flat-global-insts"> {
let Documentation = [DocAVLoadB128];
diff --git a/clang/include/clang/Basic/BuiltinsAMDGPUDocs.td b/clang/include/clang/Basic/BuiltinsAMDGPUDocs.td
index fceff497255050..2822d430458e4d 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPUDocs.td
+++ b/clang/include/clang/Basic/BuiltinsAMDGPUDocs.td
@@ -614,29 +614,39 @@ Requires target feature ``wmma-f4-insts`` and wave32.
def DocCatAVLoadStore : DocumentationCategory<"Available/Visible Memory Access Builtins"> {
let Content = [{
-These builtins perform 128-bit global or flat memory loads and stores with
-available/visible (AV) semantics.
+These builtins perform global or flat memory loads and
+stores with available/visible (AV) semantics.
}];
}
-def DocAVLoadB128 : Documentation {
+class DocAVLoadBase<string width> : Documentation {
let Category = DocCatAVLoadStore;
- let Content = [{
-Loads 128 bits (4 x i32) from the pointer ``ptr``. The pointer must be in
-the global or generic address space. The ``scope`` argument specifies the
-synchronization scope using a ``__MEMORY_SCOPE_*`` constant.
-}];
+ let Content = !strconcat("\nLoads ", width, [{ bits from the pointer ``ptr``.
+The pointer must be in the global or generic address space. The ``scope``
+argument specifies the synchronization scope using a ``__MEMORY_SCOPE_*``
+constant.
+}]);
}
-def DocAVStoreB128 : Documentation {
+def DocAVLoadB16 : DocAVLoadBase<"16">;
+def DocAVLoadB32 : DocAVLoadBase<"32">;
+def DocAVLoadB64 : DocAVLoadBase<"64">;
+def DocAVLoadB128 : DocAVLoadBase<"128">;
+
+class DocAVStoreBase<string width> : Documentation {
let Category = DocCatAVLoadStore;
- let Content = [{
-Stores 128 bits (4 x i32) of ``data`` to the pointer ``ptr``. The pointer
-must be in the global or generic address space. The ``scope`` argument
-specifies the synchronization scope using a ``__MEMORY_SCOPE_*`` constant.
-}];
+ let Content = !strconcat("\nStores ", width, [{ bits of ``data`` to the pointer
+``ptr``. The pointer must be in the global or generic address space. The
+``scope`` argument specifies the synchronization scope using a
+``__MEMORY_SCOPE_*`` constant.
+}]);
}
+def DocAVStoreB16 : DocAVStoreBase<"16">;
+def DocAVStoreB32 : DocAVStoreBase<"32">;
+def DocAVStoreB64 : DocAVStoreBase<"64">;
+def DocAVStoreB128 : DocAVStoreBase<"128">;
+
//===----------------------------------------------------------------------===//
// Wave Data Exchange Builtins
//===----------------------------------------------------------------------===//
diff --git a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
index f3bf71dddc341f..bc548a51ab7ecf 100644
--- a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
@@ -1036,9 +1036,42 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
llvm::Function *F = CGM.getIntrinsic(IID, {Args[0]->getType()});
return Builder.CreateCall(F, {Args});
}
+ case AMDGPU::BI__builtin_amdgcn_av_load_b16:
+ case AMDGPU::BI__builtin_amdgcn_av_load_b32:
+ case AMDGPU::BI__builtin_amdgcn_av_load_b64:
case AMDGPU::BI__builtin_amdgcn_av_load_b128:
+ case AMDGPU::BI__builtin_amdgcn_av_store_b16:
+ case AMDGPU::BI__builtin_amdgcn_av_store_b32:
+ case AMDGPU::BI__builtin_amdgcn_av_store_b64:
case AMDGPU::BI__builtin_amdgcn_av_store_b128: {
- const bool IsStore = BuiltinID == AMDGPU::BI__builtin_amdgcn_av_store_b128;
+ Intrinsic::ID IID;
+ switch (BuiltinID) {
+ case AMDGPU::BI__builtin_amdgcn_av_load_b16:
+ IID = Intrinsic::amdgcn_av_load_b16;
+ break;
+ case AMDGPU::BI__builtin_amdgcn_av_load_b32:
+ IID = Intrinsic::amdgcn_av_load_b32;
+ break;
+ case AMDGPU::BI__builtin_amdgcn_av_load_b64:
+ IID = Intrinsic::amdgcn_av_load_b64;
+ break;
+ case AMDGPU::BI__builtin_amdgcn_av_load_b128:
+ IID = Intrinsic::amdgcn_av_load_b128;
+ break;
+ case AMDGPU::BI__builtin_amdgcn_av_store_b16:
+ IID = Intrinsic::amdgcn_av_store_b16;
+ break;
+ case AMDGPU::BI__builtin_amdgcn_av_store_b32:
+ IID = Intrinsic::amdgcn_av_store_b32;
+ break;
+ case AMDGPU::BI__builtin_amdgcn_av_store_b64:
+ IID = Intrinsic::amdgcn_av_store_b64;
+ break;
+ case AMDGPU::BI__builtin_amdgcn_av_store_b128:
+ IID = Intrinsic::amdgcn_av_store_b128;
+ break;
+ }
+ const bool IsStore = E->getType()->isVoidType();
SmallVector<Value *, 5> Args = {EmitScalarExpr(E->getArg(0))}; // addr
if (IsStore)
Args.push_back(EmitScalarExpr(E->getArg(1))); // data
@@ -1046,10 +1079,7 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
auto *ScopeExpr =
cast<llvm::ConstantInt>(EmitScalarExpr(E->getArg(ScopeIdx)));
Args.push_back(emitScopeMD(*this, ScopeExpr->getZExtValue()));
- llvm::Function *F =
- CGM.getIntrinsic(IsStore ? Intrinsic::amdgcn_av_store_b128
- : Intrinsic::amdgcn_av_load_b128,
- {Args[0]->getType()});
+ llvm::Function *F = CGM.getIntrinsic(IID, {Args[0]->getType()});
return Builder.CreateCall(F, Args);
}
case AMDGPU::BI__builtin_amdgcn_get_fpenv: {
diff --git a/clang/lib/Sema/SemaAMDGPU.cpp b/clang/lib/Sema/SemaAMDGPU.cpp
index b1f6fa33649bd1..960bbbf93adad4 100644
--- a/clang/lib/Sema/SemaAMDGPU.cpp
+++ b/clang/lib/Sema/SemaAMDGPU.cpp
@@ -220,8 +220,14 @@ bool SemaAMDGPU::CheckAMDGCNBuiltinFunctionCall(const TargetInfo &TI,
case AMDGPU::BI__builtin_amdgcn_cvt_scale_pk32_f32_bf6:
case AMDGPU::BI__builtin_amdgcn_cvt_scale_pk32_f32_fp6:
return SemaRef.BuiltinConstantArgRange(TheCall, 2, 0, 15);
+ case AMDGPU::BI__builtin_amdgcn_av_load_b16:
+ case AMDGPU::BI__builtin_amdgcn_av_load_b32:
+ case AMDGPU::BI__builtin_amdgcn_av_load_b64:
case AMDGPU::BI__builtin_amdgcn_av_load_b128:
return checkAVLoadStore(TheCall, /*IsStore=*/false);
+ case AMDGPU::BI__builtin_amdgcn_av_store_b16:
+ case AMDGPU::BI__builtin_amdgcn_av_store_b32:
+ case AMDGPU::BI__builtin_amdgcn_av_store_b64:
case AMDGPU::BI__builtin_amdgcn_av_store_b128:
return checkAVLoadStore(TheCall, /*IsStore=*/true);
case AMDGPU::BI__builtin_amdgcn_cooperative_atomic_load_32x4B:
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-store.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-store.cl
index 7f8651c837bfe3..64080920f225fc 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-store.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-store.cl
@@ -9,6 +9,12 @@
typedef __attribute__((__vector_size__(4 * sizeof(unsigned int)))) unsigned int v4u32;
typedef v4u32 __global *global_ptr_to_v4u32;
+typedef unsigned short u16;
+typedef unsigned int u32;
+typedef unsigned long u64;
+typedef u16 __global *global_ptr_to_u16;
+typedef u32 __global *global_ptr_to_u32;
+typedef u64 __global *global_ptr_to_u64;
//------------------------------------------------------------------------------
// Global Load
@@ -16,7 +22,7 @@ typedef v4u32 __global *global_ptr_to_v4u32;
// CHECK-LABEL: @test_amdgcn_global_load_b128_wave(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.av.load.b128.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META7:![0-9]+]])
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.av.load.b128.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META8:![0-9]+]])
// CHECK-NEXT: ret <4 x i32> [[TMP0]]
//
v4u32 test_amdgcn_global_load_b128_wave(global_ptr_to_v4u32 ptr) {
@@ -25,7 +31,7 @@ v4u32 test_amdgcn_global_load_b128_wave(global_ptr_to_v4u32 ptr) {
// CHECK-LABEL: @test_amdgcn_global_load_b128_workgroup(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.av.load.b128.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META8:![0-9]+]])
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.av.load.b128.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META9:![0-9]+]])
// CHECK-NEXT: ret <4 x i32> [[TMP0]]
//
v4u32 test_amdgcn_global_load_b128_workgroup(global_ptr_to_v4u32 ptr) {
@@ -34,7 +40,7 @@ v4u32 test_amdgcn_global_load_b128_workgroup(global_ptr_to_v4u32 ptr) {
// CHECK-LABEL: @test_amdgcn_global_load_b128_device(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.av.load.b128.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META9:![0-9]+]])
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.av.load.b128.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META10:![0-9]+]])
// CHECK-NEXT: ret <4 x i32> [[TMP0]]
//
v4u32 test_amdgcn_global_load_b128_device(global_ptr_to_v4u32 ptr) {
@@ -43,7 +49,7 @@ v4u32 test_amdgcn_global_load_b128_device(global_ptr_to_v4u32 ptr) {
// CHECK-LABEL: @test_amdgcn_global_load_b128_system(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.av.load.b128.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META10:![0-9]+]])
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.av.load.b128.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META11:![0-9]+]])
// CHECK-NEXT: ret <4 x i32> [[TMP0]]
//
v4u32 test_amdgcn_global_load_b128_system(global_ptr_to_v4u32 ptr) {
@@ -52,7 +58,7 @@ v4u32 test_amdgcn_global_load_b128_system(global_ptr_to_v4u32 ptr) {
// CHECK-LABEL: @test_amdgcn_global_load_b128_single(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.av.load.b128.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META11:![0-9]+]])
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.av.load.b128.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META12:![0-9]+]])
// CHECK-NEXT: ret <4 x i32> [[TMP0]]
//
v4u32 test_amdgcn_global_load_b128_single(global_ptr_to_v4u32 ptr) {
@@ -61,20 +67,182 @@ v4u32 test_amdgcn_global_load_b128_single(global_ptr_to_v4u32 ptr) {
// CHECK-LABEL: @test_amdgcn_global_load_b128_cluster(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.av.load.b128.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META12:![0-9]+]])
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.av.load.b128.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META13:![0-9]+]])
// CHECK-NEXT: ret <4 x i32> [[TMP0]]
//
v4u32 test_amdgcn_global_load_b128_cluster(global_ptr_to_v4u32 ptr) {
return __builtin_amdgcn_av_load_b128(ptr, __MEMORY_SCOPE_CLUSTR);
}
+// CHECK-LABEL: @test_amdgcn_global_load_b16_wave(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i16 @llvm.amdgcn.av.load.b16.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META8]])
+// CHECK-NEXT: ret i16 [[TMP0]]
+//
+u16 test_amdgcn_global_load_b16_wave(global_ptr_to_u16 ptr) {
+ return __builtin_amdgcn_av_load_b16(ptr, __MEMORY_SCOPE_WVFRNT);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_load_b16_workgroup(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i16 @llvm.amdgcn.av.load.b16.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META9]])
+// CHECK-NEXT: ret i16 [[TMP0]]
+//
+u16 test_amdgcn_global_load_b16_workgroup(global_ptr_to_u16 ptr) {
+ return __builtin_amdgcn_av_load_b16(ptr, __MEMORY_SCOPE_WRKGRP);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_load_b16_device(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i16 @llvm.amdgcn.av.load.b16.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META10]])
+// CHECK-NEXT: ret i16 [[TMP0]]
+//
+u16 test_amdgcn_global_load_b16_device(global_ptr_to_u16 ptr) {
+ return __builtin_amdgcn_av_load_b16(ptr, __MEMORY_SCOPE_DEVICE);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_load_b16_system(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i16 @llvm.amdgcn.av.load.b16.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META11]])
+// CHECK-NEXT: ret i16 [[TMP0]]
+//
+u16 test_amdgcn_global_load_b16_system(global_ptr_to_u16 ptr) {
+ return __builtin_amdgcn_av_load_b16(ptr, __MEMORY_SCOPE_SYSTEM);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_load_b16_single(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i16 @llvm.amdgcn.av.load.b16.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META12]])
+// CHECK-NEXT: ret i16 [[TMP0]]
+//
+u16 test_amdgcn_global_load_b16_single(global_ptr_to_u16 ptr) {
+ return __builtin_amdgcn_av_load_b16(ptr, __MEMORY_SCOPE_SINGLE);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_load_b16_cluster(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i16 @llvm.amdgcn.av.load.b16.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META13]])
+// CHECK-NEXT: ret i16 [[TMP0]]
+//
+u16 test_amdgcn_global_load_b16_cluster(global_ptr_to_u16 ptr) {
+ return __builtin_amdgcn_av_load_b16(ptr, __MEMORY_SCOPE_CLUSTR);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_load_b32_wave(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.av.load.b32.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META8]])
+// CHECK-NEXT: ret i32 [[TMP0]]
+//
+u32 test_amdgcn_global_load_b32_wave(global_ptr_to_u32 ptr) {
+ return __builtin_amdgcn_av_load_b32(ptr, __MEMORY_SCOPE_WVFRNT);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_load_b32_workgroup(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.av.load.b32.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META9]])
+// CHECK-NEXT: ret i32 [[TMP0]]
+//
+u32 test_amdgcn_global_load_b32_workgroup(global_ptr_to_u32 ptr) {
+ return __builtin_amdgcn_av_load_b32(ptr, __MEMORY_SCOPE_WRKGRP);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_load_b32_device(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.av.load.b32.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META10]])
+// CHECK-NEXT: ret i32 [[TMP0]]
+//
+u32 test_amdgcn_global_load_b32_device(global_ptr_to_u32 ptr) {
+ return __builtin_amdgcn_av_load_b32(ptr, __MEMORY_SCOPE_DEVICE);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_load_b32_system(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.av.load.b32.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META11]])
+// CHECK-NEXT: ret i32 [[TMP0]]
+//
+u32 test_amdgcn_global_load_b32_system(global_ptr_to_u32 ptr) {
+ return __builtin_amdgcn_av_load_b32(ptr, __MEMORY_SCOPE_SYSTEM);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_load_b32_single(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.av.load.b32.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META12]])
+// CHECK-NEXT: ret i32 [[TMP0]]
+//
+u32 test_amdgcn_global_load_b32_single(global_ptr_to_u32 ptr) {
+ return __builtin_amdgcn_av_load_b32(ptr, __MEMORY_SCOPE_SINGLE);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_load_b32_cluster(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.av.load.b32.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META13]])
+// CHECK-NEXT: ret i32 [[TMP0]]
+//
+u32 test_amdgcn_global_load_b32_cluster(global_ptr_to_u32 ptr) {
+ return __builtin_amdgcn_av_load_b32(ptr, __MEMORY_SCOPE_CLUSTR);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_load_b64_wave(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META8]])
+// CHECK-NEXT: ret i64 [[TMP0]]
+//
+u64 test_amdgcn_global_load_b64_wave(global_ptr_to_u64 ptr) {
+ return __builtin_amdgcn_av_load_b64(ptr, __MEMORY_SCOPE_WVFRNT);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_load_b64_workgroup(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META9]])
+// CHECK-NEXT: ret i64 [[TMP0]]
+//
+u64 test_amdgcn_global_load_b64_workgroup(global_ptr_to_u64 ptr) {
+ return __builtin_amdgcn_av_load_b64(ptr, __MEMORY_SCOPE_WRKGRP);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_load_b64_device(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META10]])
+// CHECK-NEXT: ret i64 [[TMP0]]
+//
+u64 test_amdgcn_global_load_b64_device(global_ptr_to_u64 ptr) {
+ return __builtin_amdgcn_av_load_b64(ptr, __MEMORY_SCOPE_DEVICE);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_load_b64_system(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META11]])
+// CHECK-NEXT: ret i64 [[TMP0]]
+//
+u64 test_amdgcn_global_load_b64_system(global_ptr_to_u64 ptr) {
+ return __builtin_amdgcn_av_load_b64(ptr, __MEMORY_SCOPE_SYSTEM);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_load_b64_single(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META12]])
+// CHECK-NEXT: ret i64 [[TMP0]]
+//
+u64 test_amdgcn_global_load_b64_single(global_ptr_to_u64 ptr) {
+ return __builtin_amdgcn_av_load_b64(ptr, __MEMORY_SCOPE_SINGLE);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_load_b64_cluster(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META13]])
+// CHECK-NEXT: ret i64 [[TMP0]]
+//
+u64 test_amdgcn_global_load_b64_cluster(global_ptr_to_u64 ptr) {
+ return __builtin_amdgcn_av_load_b64(ptr, __MEMORY_SCOPE_CLUSTR);
+}
+
//------------------------------------------------------------------------------
// Global Store
//------------------------------------------------------------------------------
// CHECK-LABEL: @test_amdgcn_global_store_b128_wave(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b128.p1(ptr addrspace(1) [[PTR:%.*]], <4 x i32> [[DATA:%.*]], metadata [[META7]])
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b128.p1(ptr addrspace(1) [[PTR:%.*]], <4 x i32> [[DATA:%.*]], metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_global_store_b128_wave(global_ptr_to_v4u32 ptr, v4u32 data) {
@@ -83,7 +251,7 @@ void test_amdgcn_global_store_b128_wave(global_ptr_to_v4u32 ptr, v4u32 data) {
// CHECK-LABEL: @test_amdgcn_global_store_b128_workgroup(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b128.p1(ptr addrspace(1) [[PTR:%.*]], <4 x i32> [[DATA:%.*]], metadata [[META8]])
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b128.p1(ptr addrspace(1) [[PTR:%.*]], <4 x i32> [[DATA:%.*]], metadata [[META9]])
// CHECK-NEXT: ret void
//
void test_amdgcn_global_store_b128_workgroup(global_ptr_to_v4u32 ptr, v4u32 data) {
@@ -92,7 +260,7 @@ void test_amdgcn_global_store_b128_workgroup(global_ptr_to_v4u32 ptr, v4u32 data
// CHECK-LABEL: @test_amdgcn_global_store_b128_device(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b128.p1(ptr addrspace(1) [[PTR:%.*]], <4 x i32> [[DATA:%.*]], metadata [[META9]])
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b128.p1(ptr addrspace(1) [[PTR:%.*]], <4 x i32> [[DATA:%.*]], metadata [[META10]])
// CHECK-NEXT: ret void
//
void test_amdgcn_global_store_b128_device(global_ptr_to_v4u32 ptr, v4u32 data) {
@@ -101,7 +269,7 @@ void test_amdgcn_global_store_b128_device(global_ptr_to_v4u32 ptr, v4u32 data) {
// CHECK-LABEL: @test_amdgcn_global_store_b128_system(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b128.p1(ptr addrspace(1) [[PTR:%.*]], <4 x i32> [[DATA:%.*]], metadata [[META10]])
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b128.p1(ptr addrspace(1) [[PTR:%.*]], <4 x i32> [[DATA:%.*]], metadata [[META11]])
// CHECK-NEXT: ret void
//
void test_amdgcn_global_store_b128_system(global_ptr_to_v4u32 ptr, v4u32 data) {
@@ -110,7 +278,7 @@ void test_amdgcn_global_store_b128_system(global_ptr_to_v4u32 ptr, v4u32 data) {
// CHECK-LABEL: @test_amdgcn_global_store_b128_single(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b128.p1(ptr addrspace(1) [[PTR:%.*]], <4 x i32> [[DATA:%.*]], metadata [[META11]])
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b128.p1(ptr addrspace(1) [[PTR:%.*]], <4 x i32> [[DATA:%.*]], metadata [[META12]])
// CHECK-NEXT: ret void
//
void test_amdgcn_global_store_b128_single(global_ptr_to_v4u32 ptr, v4u32 data) {
@@ -119,20 +287,182 @@ void test_amdgcn_global_store_b128_single(global_ptr_to_v4u32 ptr, v4u32 data) {
// CHECK-LABEL: @test_amdgcn_global_store_b128_cluster(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b128.p1(ptr addrspace(1) [[PTR:%.*]], <4 x i32> [[DATA:%.*]], metadata [[META12]])
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b128.p1(ptr addrspace(1) [[PTR:%.*]], <4 x i32> [[DATA:%.*]], metadata [[META13]])
// CHECK-NEXT: ret void
//
void test_amdgcn_global_store_b128_cluster(global_ptr_to_v4u32 ptr, v4u32 data) {
__builtin_amdgcn_av_store_b128(ptr, data, __MEMORY_SCOPE_CLUSTR);
}
+// CHECK-LABEL: @test_amdgcn_global_store_b16_wave(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b16.p1(ptr addrspace(1) [[PTR:%.*]], i16 [[DATA:%.*]], metadata [[META8]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_global_store_b16_wave(global_ptr_to_u16 ptr, u16 data) {
+ __builtin_amdgcn_av_store_b16(ptr, data, __MEMORY_SCOPE_WVFRNT);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_store_b16_workgroup(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b16.p1(ptr addrspace(1) [[PTR:%.*]], i16 [[DATA:%.*]], metadata [[META9]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_global_store_b16_workgroup(global_ptr_to_u16 ptr, u16 data) {
+ __builtin_amdgcn_av_store_b16(ptr, data, __MEMORY_SCOPE_WRKGRP);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_store_b16_device(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b16.p1(ptr addrspace(1) [[PTR:%.*]], i16 [[DATA:%.*]], metadata [[META10]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_global_store_b16_device(global_ptr_to_u16 ptr, u16 data) {
+ __builtin_amdgcn_av_store_b16(ptr, data, __MEMORY_SCOPE_DEVICE);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_store_b16_system(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b16.p1(ptr addrspace(1) [[PTR:%.*]], i16 [[DATA:%.*]], metadata [[META11]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_global_store_b16_system(global_ptr_to_u16 ptr, u16 data) {
+ __builtin_amdgcn_av_store_b16(ptr, data, __MEMORY_SCOPE_SYSTEM);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_store_b16_single(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b16.p1(ptr addrspace(1) [[PTR:%.*]], i16 [[DATA:%.*]], metadata [[META12]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_global_store_b16_single(global_ptr_to_u16 ptr, u16 data) {
+ __builtin_amdgcn_av_store_b16(ptr, data, __MEMORY_SCOPE_SINGLE);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_store_b16_cluster(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b16.p1(ptr addrspace(1) [[PTR:%.*]], i16 [[DATA:%.*]], metadata [[META13]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_global_store_b16_cluster(global_ptr_to_u16 ptr, u16 data) {
+ __builtin_amdgcn_av_store_b16(ptr, data, __MEMORY_SCOPE_CLUSTR);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_store_b32_wave(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b32.p1(ptr addrspace(1) [[PTR:%.*]], i32 [[DATA:%.*]], metadata [[META8]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_global_store_b32_wave(global_ptr_to_u32 ptr, u32 data) {
+ __builtin_amdgcn_av_store_b32(ptr, data, __MEMORY_SCOPE_WVFRNT);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_store_b32_workgroup(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b32.p1(ptr addrspace(1) [[PTR:%.*]], i32 [[DATA:%.*]], metadata [[META9]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_global_store_b32_workgroup(global_ptr_to_u32 ptr, u32 data) {
+ __builtin_amdgcn_av_store_b32(ptr, data, __MEMORY_SCOPE_WRKGRP);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_store_b32_device(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b32.p1(ptr addrspace(1) [[PTR:%.*]], i32 [[DATA:%.*]], metadata [[META10]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_global_store_b32_device(global_ptr_to_u32 ptr, u32 data) {
+ __builtin_amdgcn_av_store_b32(ptr, data, __MEMORY_SCOPE_DEVICE);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_store_b32_system(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b32.p1(ptr addrspace(1) [[PTR:%.*]], i32 [[DATA:%.*]], metadata [[META11]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_global_store_b32_system(global_ptr_to_u32 ptr, u32 data) {
+ __builtin_amdgcn_av_store_b32(ptr, data, __MEMORY_SCOPE_SYSTEM);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_store_b32_single(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b32.p1(ptr addrspace(1) [[PTR:%.*]], i32 [[DATA:%.*]], metadata [[META12]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_global_store_b32_single(global_ptr_to_u32 ptr, u32 data) {
+ __builtin_amdgcn_av_store_b32(ptr, data, __MEMORY_SCOPE_SINGLE);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_store_b32_cluster(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b32.p1(ptr addrspace(1) [[PTR:%.*]], i32 [[DATA:%.*]], metadata [[META13]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_global_store_b32_cluster(global_ptr_to_u32 ptr, u32 data) {
+ __builtin_amdgcn_av_store_b32(ptr, data, __MEMORY_SCOPE_CLUSTR);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_store_b64_wave(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) [[PTR:%.*]], i64 [[DATA:%.*]], metadata [[META8]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_global_store_b64_wave(global_ptr_to_u64 ptr, u64 data) {
+ __builtin_amdgcn_av_store_b64(ptr, data, __MEMORY_SCOPE_WVFRNT);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_store_b64_workgroup(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) [[PTR:%.*]], i64 [[DATA:%.*]], metadata [[META9]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_global_store_b64_workgroup(global_ptr_to_u64 ptr, u64 data) {
+ __builtin_amdgcn_av_store_b64(ptr, data, __MEMORY_SCOPE_WRKGRP);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_store_b64_device(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) [[PTR:%.*]], i64 [[DATA:%.*]], metadata [[META10]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_global_store_b64_device(global_ptr_to_u64 ptr, u64 data) {
+ __builtin_amdgcn_av_store_b64(ptr, data, __MEMORY_SCOPE_DEVICE);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_store_b64_system(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) [[PTR:%.*]], i64 [[DATA:%.*]], metadata [[META11]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_global_store_b64_system(global_ptr_to_u64 ptr, u64 data) {
+ __builtin_amdgcn_av_store_b64(ptr, data, __MEMORY_SCOPE_SYSTEM);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_store_b64_single(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) [[PTR:%.*]], i64 [[DATA:%.*]], metadata [[META12]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_global_store_b64_single(global_ptr_to_u64 ptr, u64 data) {
+ __builtin_amdgcn_av_store_b64(ptr, data, __MEMORY_SCOPE_SINGLE);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_store_b64_cluster(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) [[PTR:%.*]], i64 [[DATA:%.*]], metadata [[META13]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_global_store_b64_cluster(global_ptr_to_u64 ptr, u64 data) {
+ __builtin_amdgcn_av_store_b64(ptr, data, __MEMORY_SCOPE_CLUSTR);
+}
+
//------------------------------------------------------------------------------
// Flat Load
//------------------------------------------------------------------------------
// CHECK-LABEL: @test_amdgcn_flat_load_b128_wave(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.av.load.b128.p0(ptr [[PTR:%.*]], metadata [[META7]])
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.av.load.b128.p0(ptr [[PTR:%.*]], metadata [[META8]])
// CHECK-NEXT: ret <4 x i32> [[TMP0]]
//
v4u32 test_amdgcn_flat_load_b128_wave(v4u32 * ptr) {
@@ -141,7 +471,7 @@ v4u32 test_amdgcn_flat_load_b128_wave(v4u32 * ptr) {
// CHECK-LABEL: @test_amdgcn_flat_load_b128_workgroup(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.av.load.b128.p0(ptr [[PTR:%.*]], metadata [[META8]])
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.av.load.b128.p0(ptr [[PTR:%.*]], metadata [[META9]])
// CHECK-NEXT: ret <4 x i32> [[TMP0]]
//
v4u32 test_amdgcn_flat_load_b128_workgroup(v4u32 * ptr) {
@@ -150,7 +480,7 @@ v4u32 test_amdgcn_flat_load_b128_workgroup(v4u32 * ptr) {
// CHECK-LABEL: @test_amdgcn_flat_load_b128_device(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.av.load.b128.p0(ptr [[PTR:%.*]], metadata [[META9]])
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.av.load.b128.p0(ptr [[PTR:%.*]], metadata [[META10]])
// CHECK-NEXT: ret <4 x i32> [[TMP0]]
//
v4u32 test_amdgcn_flat_load_b128_device(v4u32 * ptr) {
@@ -159,7 +489,7 @@ v4u32 test_amdgcn_flat_load_b128_device(v4u32 * ptr) {
// CHECK-LABEL: @test_amdgcn_flat_load_b128_system(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.av.load.b128.p0(ptr [[PTR:%.*]], metadata [[META10]])
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.av.load.b128.p0(ptr [[PTR:%.*]], metadata [[META11]])
// CHECK-NEXT: ret <4 x i32> [[TMP0]]
//
v4u32 test_amdgcn_flat_load_b128_system(v4u32 * ptr) {
@@ -168,7 +498,7 @@ v4u32 test_amdgcn_flat_load_b128_system(v4u32 * ptr) {
// CHECK-LABEL: @test_amdgcn_flat_load_b128_single(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.av.load.b128.p0(ptr [[PTR:%.*]], metadata [[META11]])
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.av.load.b128.p0(ptr [[PTR:%.*]], metadata [[META12]])
// CHECK-NEXT: ret <4 x i32> [[TMP0]]
//
v4u32 test_amdgcn_flat_load_b128_single(v4u32 * ptr) {
@@ -177,20 +507,182 @@ v4u32 test_amdgcn_flat_load_b128_single(v4u32 * ptr) {
// CHECK-LABEL: @test_amdgcn_flat_load_b128_cluster(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.av.load.b128.p0(ptr [[PTR:%.*]], metadata [[META12]])
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i32> @llvm.amdgcn.av.load.b128.p0(ptr [[PTR:%.*]], metadata [[META13]])
// CHECK-NEXT: ret <4 x i32> [[TMP0]]
//
v4u32 test_amdgcn_flat_load_b128_cluster(v4u32 * ptr) {
return __builtin_amdgcn_av_load_b128(ptr, __MEMORY_SCOPE_CLUSTR);
}
+// CHECK-LABEL: @test_amdgcn_flat_load_b16_wave(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i16 @llvm.amdgcn.av.load.b16.p0(ptr [[PTR:%.*]], metadata [[META8]])
+// CHECK-NEXT: ret i16 [[TMP0]]
+//
+u16 test_amdgcn_flat_load_b16_wave(u16 * ptr) {
+ return __builtin_amdgcn_av_load_b16(ptr, __MEMORY_SCOPE_WVFRNT);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_load_b16_workgroup(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i16 @llvm.amdgcn.av.load.b16.p0(ptr [[PTR:%.*]], metadata [[META9]])
+// CHECK-NEXT: ret i16 [[TMP0]]
+//
+u16 test_amdgcn_flat_load_b16_workgroup(u16 * ptr) {
+ return __builtin_amdgcn_av_load_b16(ptr, __MEMORY_SCOPE_WRKGRP);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_load_b16_device(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i16 @llvm.amdgcn.av.load.b16.p0(ptr [[PTR:%.*]], metadata [[META10]])
+// CHECK-NEXT: ret i16 [[TMP0]]
+//
+u16 test_amdgcn_flat_load_b16_device(u16 * ptr) {
+ return __builtin_amdgcn_av_load_b16(ptr, __MEMORY_SCOPE_DEVICE);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_load_b16_system(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i16 @llvm.amdgcn.av.load.b16.p0(ptr [[PTR:%.*]], metadata [[META11]])
+// CHECK-NEXT: ret i16 [[TMP0]]
+//
+u16 test_amdgcn_flat_load_b16_system(u16 * ptr) {
+ return __builtin_amdgcn_av_load_b16(ptr, __MEMORY_SCOPE_SYSTEM);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_load_b16_single(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i16 @llvm.amdgcn.av.load.b16.p0(ptr [[PTR:%.*]], metadata [[META12]])
+// CHECK-NEXT: ret i16 [[TMP0]]
+//
+u16 test_amdgcn_flat_load_b16_single(u16 * ptr) {
+ return __builtin_amdgcn_av_load_b16(ptr, __MEMORY_SCOPE_SINGLE);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_load_b16_cluster(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i16 @llvm.amdgcn.av.load.b16.p0(ptr [[PTR:%.*]], metadata [[META13]])
+// CHECK-NEXT: ret i16 [[TMP0]]
+//
+u16 test_amdgcn_flat_load_b16_cluster(u16 * ptr) {
+ return __builtin_amdgcn_av_load_b16(ptr, __MEMORY_SCOPE_CLUSTR);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_load_b32_wave(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.av.load.b32.p0(ptr [[PTR:%.*]], metadata [[META8]])
+// CHECK-NEXT: ret i32 [[TMP0]]
+//
+u32 test_amdgcn_flat_load_b32_wave(u32 * ptr) {
+ return __builtin_amdgcn_av_load_b32(ptr, __MEMORY_SCOPE_WVFRNT);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_load_b32_workgroup(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.av.load.b32.p0(ptr [[PTR:%.*]], metadata [[META9]])
+// CHECK-NEXT: ret i32 [[TMP0]]
+//
+u32 test_amdgcn_flat_load_b32_workgroup(u32 * ptr) {
+ return __builtin_amdgcn_av_load_b32(ptr, __MEMORY_SCOPE_WRKGRP);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_load_b32_device(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.av.load.b32.p0(ptr [[PTR:%.*]], metadata [[META10]])
+// CHECK-NEXT: ret i32 [[TMP0]]
+//
+u32 test_amdgcn_flat_load_b32_device(u32 * ptr) {
+ return __builtin_amdgcn_av_load_b32(ptr, __MEMORY_SCOPE_DEVICE);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_load_b32_system(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.av.load.b32.p0(ptr [[PTR:%.*]], metadata [[META11]])
+// CHECK-NEXT: ret i32 [[TMP0]]
+//
+u32 test_amdgcn_flat_load_b32_system(u32 * ptr) {
+ return __builtin_amdgcn_av_load_b32(ptr, __MEMORY_SCOPE_SYSTEM);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_load_b32_single(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.av.load.b32.p0(ptr [[PTR:%.*]], metadata [[META12]])
+// CHECK-NEXT: ret i32 [[TMP0]]
+//
+u32 test_amdgcn_flat_load_b32_single(u32 * ptr) {
+ return __builtin_amdgcn_av_load_b32(ptr, __MEMORY_SCOPE_SINGLE);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_load_b32_cluster(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.av.load.b32.p0(ptr [[PTR:%.*]], metadata [[META13]])
+// CHECK-NEXT: ret i32 [[TMP0]]
+//
+u32 test_amdgcn_flat_load_b32_cluster(u32 * ptr) {
+ return __builtin_amdgcn_av_load_b32(ptr, __MEMORY_SCOPE_CLUSTR);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_load_b64_wave(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.amdgcn.av.load.b64.p0(ptr [[PTR:%.*]], metadata [[META8]])
+// CHECK-NEXT: ret i64 [[TMP0]]
+//
+u64 test_amdgcn_flat_load_b64_wave(u64 * ptr) {
+ return __builtin_amdgcn_av_load_b64(ptr, __MEMORY_SCOPE_WVFRNT);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_load_b64_workgroup(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.amdgcn.av.load.b64.p0(ptr [[PTR:%.*]], metadata [[META9]])
+// CHECK-NEXT: ret i64 [[TMP0]]
+//
+u64 test_amdgcn_flat_load_b64_workgroup(u64 * ptr) {
+ return __builtin_amdgcn_av_load_b64(ptr, __MEMORY_SCOPE_WRKGRP);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_load_b64_device(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.amdgcn.av.load.b64.p0(ptr [[PTR:%.*]], metadata [[META10]])
+// CHECK-NEXT: ret i64 [[TMP0]]
+//
+u64 test_amdgcn_flat_load_b64_device(u64 * ptr) {
+ return __builtin_amdgcn_av_load_b64(ptr, __MEMORY_SCOPE_DEVICE);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_load_b64_system(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.amdgcn.av.load.b64.p0(ptr [[PTR:%.*]], metadata [[META11]])
+// CHECK-NEXT: ret i64 [[TMP0]]
+//
+u64 test_amdgcn_flat_load_b64_system(u64 * ptr) {
+ return __builtin_amdgcn_av_load_b64(ptr, __MEMORY_SCOPE_SYSTEM);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_load_b64_single(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.amdgcn.av.load.b64.p0(ptr [[PTR:%.*]], metadata [[META12]])
+// CHECK-NEXT: ret i64 [[TMP0]]
+//
+u64 test_amdgcn_flat_load_b64_single(u64 * ptr) {
+ return __builtin_amdgcn_av_load_b64(ptr, __MEMORY_SCOPE_SINGLE);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_load_b64_cluster(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.amdgcn.av.load.b64.p0(ptr [[PTR:%.*]], metadata [[META13]])
+// CHECK-NEXT: ret i64 [[TMP0]]
+//
+u64 test_amdgcn_flat_load_b64_cluster(u64 * ptr) {
+ return __builtin_amdgcn_av_load_b64(ptr, __MEMORY_SCOPE_CLUSTR);
+}
+
//------------------------------------------------------------------------------
// Flat Store
//------------------------------------------------------------------------------
// CHECK-LABEL: @test_amdgcn_flat_store_b128_wave(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b128.p0(ptr [[PTR:%.*]], <4 x i32> [[DATA:%.*]], metadata [[META7]])
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b128.p0(ptr [[PTR:%.*]], <4 x i32> [[DATA:%.*]], metadata [[META8]])
// CHECK-NEXT: ret void
//
void test_amdgcn_flat_store_b128_wave(v4u32 * ptr, v4u32 data) {
@@ -199,7 +691,7 @@ void test_amdgcn_flat_store_b128_wave(v4u32 * ptr, v4u32 data) {
// CHECK-LABEL: @test_amdgcn_flat_store_b128_workgroup(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b128.p0(ptr [[PTR:%.*]], <4 x i32> [[DATA:%.*]], metadata [[META8]])
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b128.p0(ptr [[PTR:%.*]], <4 x i32> [[DATA:%.*]], metadata [[META9]])
// CHECK-NEXT: ret void
//
void test_amdgcn_flat_store_b128_workgroup(v4u32 * ptr, v4u32 data) {
@@ -208,7 +700,7 @@ void test_amdgcn_flat_store_b128_workgroup(v4u32 * ptr, v4u32 data) {
// CHECK-LABEL: @test_amdgcn_flat_store_b128_device(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b128.p0(ptr [[PTR:%.*]], <4 x i32> [[DATA:%.*]], metadata [[META9]])
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b128.p0(ptr [[PTR:%.*]], <4 x i32> [[DATA:%.*]], metadata [[META10]])
// CHECK-NEXT: ret void
//
void test_amdgcn_flat_store_b128_device(v4u32 * ptr, v4u32 data) {
@@ -217,7 +709,7 @@ void test_amdgcn_flat_store_b128_device(v4u32 * ptr, v4u32 data) {
// CHECK-LABEL: @test_amdgcn_flat_store_b128_system(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b128.p0(ptr [[PTR:%.*]], <4 x i32> [[DATA:%.*]], metadata [[META10]])
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b128.p0(ptr [[PTR:%.*]], <4 x i32> [[DATA:%.*]], metadata [[META11]])
// CHECK-NEXT: ret void
//
void test_amdgcn_flat_store_b128_system(v4u32 * ptr, v4u32 data) {
@@ -226,7 +718,7 @@ void test_amdgcn_flat_store_b128_system(v4u32 * ptr, v4u32 data) {
// CHECK-LABEL: @test_amdgcn_flat_store_b128_single(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b128.p0(ptr [[PTR:%.*]], <4 x i32> [[DATA:%.*]], metadata [[META11]])
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b128.p0(ptr [[PTR:%.*]], <4 x i32> [[DATA:%.*]], metadata [[META12]])
// CHECK-NEXT: ret void
//
void test_amdgcn_flat_store_b128_single(v4u32 * ptr, v4u32 data) {
@@ -235,17 +727,179 @@ void test_amdgcn_flat_store_b128_single(v4u32 * ptr, v4u32 data) {
// CHECK-LABEL: @test_amdgcn_flat_store_b128_cluster(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b128.p0(ptr [[PTR:%.*]], <4 x i32> [[DATA:%.*]], metadata [[META12]])
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b128.p0(ptr [[PTR:%.*]], <4 x i32> [[DATA:%.*]], metadata [[META13]])
// CHECK-NEXT: ret void
//
void test_amdgcn_flat_store_b128_cluster(v4u32 * ptr, v4u32 data) {
__builtin_amdgcn_av_store_b128(ptr, data, __MEMORY_SCOPE_CLUSTR);
}
+
+// CHECK-LABEL: @test_amdgcn_flat_store_b16_wave(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b16.p0(ptr [[PTR:%.*]], i16 [[DATA:%.*]], metadata [[META8]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_flat_store_b16_wave(u16 * ptr, u16 data) {
+ __builtin_amdgcn_av_store_b16(ptr, data, __MEMORY_SCOPE_WVFRNT);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_store_b16_workgroup(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b16.p0(ptr [[PTR:%.*]], i16 [[DATA:%.*]], metadata [[META9]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_flat_store_b16_workgroup(u16 * ptr, u16 data) {
+ __builtin_amdgcn_av_store_b16(ptr, data, __MEMORY_SCOPE_WRKGRP);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_store_b16_device(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b16.p0(ptr [[PTR:%.*]], i16 [[DATA:%.*]], metadata [[META10]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_flat_store_b16_device(u16 * ptr, u16 data) {
+ __builtin_amdgcn_av_store_b16(ptr, data, __MEMORY_SCOPE_DEVICE);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_store_b16_system(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b16.p0(ptr [[PTR:%.*]], i16 [[DATA:%.*]], metadata [[META11]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_flat_store_b16_system(u16 * ptr, u16 data) {
+ __builtin_amdgcn_av_store_b16(ptr, data, __MEMORY_SCOPE_SYSTEM);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_store_b16_single(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b16.p0(ptr [[PTR:%.*]], i16 [[DATA:%.*]], metadata [[META12]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_flat_store_b16_single(u16 * ptr, u16 data) {
+ __builtin_amdgcn_av_store_b16(ptr, data, __MEMORY_SCOPE_SINGLE);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_store_b16_cluster(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b16.p0(ptr [[PTR:%.*]], i16 [[DATA:%.*]], metadata [[META13]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_flat_store_b16_cluster(u16 * ptr, u16 data) {
+ __builtin_amdgcn_av_store_b16(ptr, data, __MEMORY_SCOPE_CLUSTR);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_store_b32_wave(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b32.p0(ptr [[PTR:%.*]], i32 [[DATA:%.*]], metadata [[META8]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_flat_store_b32_wave(u32 * ptr, u32 data) {
+ __builtin_amdgcn_av_store_b32(ptr, data, __MEMORY_SCOPE_WVFRNT);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_store_b32_workgroup(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b32.p0(ptr [[PTR:%.*]], i32 [[DATA:%.*]], metadata [[META9]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_flat_store_b32_workgroup(u32 * ptr, u32 data) {
+ __builtin_amdgcn_av_store_b32(ptr, data, __MEMORY_SCOPE_WRKGRP);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_store_b32_device(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b32.p0(ptr [[PTR:%.*]], i32 [[DATA:%.*]], metadata [[META10]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_flat_store_b32_device(u32 * ptr, u32 data) {
+ __builtin_amdgcn_av_store_b32(ptr, data, __MEMORY_SCOPE_DEVICE);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_store_b32_system(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b32.p0(ptr [[PTR:%.*]], i32 [[DATA:%.*]], metadata [[META11]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_flat_store_b32_system(u32 * ptr, u32 data) {
+ __builtin_amdgcn_av_store_b32(ptr, data, __MEMORY_SCOPE_SYSTEM);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_store_b32_single(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b32.p0(ptr [[PTR:%.*]], i32 [[DATA:%.*]], metadata [[META12]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_flat_store_b32_single(u32 * ptr, u32 data) {
+ __builtin_amdgcn_av_store_b32(ptr, data, __MEMORY_SCOPE_SINGLE);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_store_b32_cluster(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b32.p0(ptr [[PTR:%.*]], i32 [[DATA:%.*]], metadata [[META13]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_flat_store_b32_cluster(u32 * ptr, u32 data) {
+ __builtin_amdgcn_av_store_b32(ptr, data, __MEMORY_SCOPE_CLUSTR);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_store_b64_wave(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p0(ptr [[PTR:%.*]], i64 [[DATA:%.*]], metadata [[META8]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_flat_store_b64_wave(u64 * ptr, u64 data) {
+ __builtin_amdgcn_av_store_b64(ptr, data, __MEMORY_SCOPE_WVFRNT);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_store_b64_workgroup(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p0(ptr [[PTR:%.*]], i64 [[DATA:%.*]], metadata [[META9]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_flat_store_b64_workgroup(u64 * ptr, u64 data) {
+ __builtin_amdgcn_av_store_b64(ptr, data, __MEMORY_SCOPE_WRKGRP);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_store_b64_device(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p0(ptr [[PTR:%.*]], i64 [[DATA:%.*]], metadata [[META10]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_flat_store_b64_device(u64 * ptr, u64 data) {
+ __builtin_amdgcn_av_store_b64(ptr, data, __MEMORY_SCOPE_DEVICE);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_store_b64_system(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p0(ptr [[PTR:%.*]], i64 [[DATA:%.*]], metadata [[META11]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_flat_store_b64_system(u64 * ptr, u64 data) {
+ __builtin_amdgcn_av_store_b64(ptr, data, __MEMORY_SCOPE_SYSTEM);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_store_b64_single(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p0(ptr [[PTR:%.*]], i64 [[DATA:%.*]], metadata [[META12]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_flat_store_b64_single(u64 * ptr, u64 data) {
+ __builtin_amdgcn_av_store_b64(ptr, data, __MEMORY_SCOPE_SINGLE);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_store_b64_cluster(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p0(ptr [[PTR:%.*]], i64 [[DATA:%.*]], metadata [[META13]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_flat_store_b64_cluster(u64 * ptr, u64 data) {
+ __builtin_amdgcn_av_store_b64(ptr, data, __MEMORY_SCOPE_CLUSTR);
+}
//.
-// CHECK: [[META7]] = !{!"wavefront"}
-// CHECK: [[META8]] = !{!"workgroup"}
-// CHECK: [[META9]] = !{!"agent"}
-// CHECK: [[META10]] = !{!""}
-// CHECK: [[META11]] = !{!"singlethread"}
-// CHECK: [[META12]] = !{!"cluster"}
+// CHECK: [[META8]] = !{!"wavefront"}
+// CHECK: [[META9]] = !{!"workgroup"}
+// CHECK: [[META10]] = !{!"agent"}
+// CHECK: [[META11]] = !{!""}
+// CHECK: [[META12]] = !{!"singlethread"}
+// CHECK: [[META13]] = !{!"cluster"}
//.
diff --git a/clang/test/SemaHIP/amdgpu-av-load-store.hip b/clang/test/SemaHIP/amdgpu-av-load-store.hip
index 061a6939cd39bb..5d4a4e62e179ad 100644
--- a/clang/test/SemaHIP/amdgpu-av-load-store.hip
+++ b/clang/test/SemaHIP/amdgpu-av-load-store.hip
@@ -8,6 +8,9 @@
#define __global__ __attribute__((global))
typedef __attribute__((__vector_size__(4 * sizeof(unsigned int)))) unsigned int v4u32;
+typedef unsigned short u16;
+typedef unsigned int u32;
+typedef unsigned long u64;
__device__ void test_av_load_store_device(v4u32 *ptr, v4u32 data) {
v4u32 res = __builtin_amdgcn_av_load_b128(ptr, 0);
@@ -18,3 +21,33 @@ __global__ void test_av_load_store_kernel(v4u32 *ptr, v4u32 data) {
v4u32 res = __builtin_amdgcn_av_load_b128(ptr, 0);
__builtin_amdgcn_av_store_b128(ptr, data, 0);
}
+
+__device__ void test_av_load_store_b16_device(u16 *ptr, u16 data) {
+ u16 res = __builtin_amdgcn_av_load_b16(ptr, 0);
+ __builtin_amdgcn_av_store_b16(ptr, data, 0);
+}
+
+__global__ void test_av_load_store_b16_kernel(u16 *ptr, u16 data) {
+ u16 res = __builtin_amdgcn_av_load_b16(ptr, 0);
+ __builtin_amdgcn_av_store_b16(ptr, data, 0);
+}
+
+__device__ void test_av_load_store_b32_device(u32 *ptr, u32 data) {
+ u32 res = __builtin_amdgcn_av_load_b32(ptr, 0);
+ __builtin_amdgcn_av_store_b32(ptr, data, 0);
+}
+
+__global__ void test_av_load_store_b32_kernel(u32 *ptr, u32 data) {
+ u32 res = __builtin_amdgcn_av_load_b32(ptr, 0);
+ __builtin_amdgcn_av_store_b32(ptr, data, 0);
+}
+
+__device__ void test_av_load_store_b64_device(u64 *ptr, u64 data) {
+ u64 res = __builtin_amdgcn_av_load_b64(ptr, 0);
+ __builtin_amdgcn_av_store_b64(ptr, data, 0);
+}
+
+__global__ void test_av_load_store_b64_kernel(u64 *ptr, u64 data) {
+ u64 res = __builtin_amdgcn_av_load_b64(ptr, 0);
+ __builtin_amdgcn_av_store_b64(ptr, data, 0);
+}
diff --git a/clang/test/SemaOpenCL/builtins-amdgcn-global-load-store-error.cl b/clang/test/SemaOpenCL/builtins-amdgcn-global-load-store-error.cl
index fea156cccd89fc..4938491be78ba5 100644
--- a/clang/test/SemaOpenCL/builtins-amdgcn-global-load-store-error.cl
+++ b/clang/test/SemaOpenCL/builtins-amdgcn-global-load-store-error.cl
@@ -1,9 +1,19 @@
// RUN: %clang_cc1 -cl-std=CL2.0 -triple amdgpu9.50-unknown-unknown -S -verify -o - %s
// REQUIRES: amdgpu-registered-target
+typedef unsigned short u16;
+typedef unsigned int u32;
+typedef unsigned long u64;
+
typedef __attribute__((__vector_size__(4 * sizeof(unsigned int)))) unsigned int v4u32;
typedef v4u32 __global *global_ptr_to_v4u32;
typedef v4u32 __private *private_ptr_to_v4u32;
+typedef u16 __global *global_ptr_to_u16;
+typedef u16 __private *private_ptr_to_u16;
+typedef u32 __global *global_ptr_to_u32;
+typedef u32 __private *private_ptr_to_u32;
+typedef u64 __global *global_ptr_to_u64;
+typedef u64 __private *private_ptr_to_u64;
void test_amdgcn_av_store_b128_bad_ptr(private_ptr_to_v4u32 ptr, v4u32 data) {
__builtin_amdgcn_av_store_b128(ptr, data, __MEMORY_SCOPE_SYSTEM); //expected-error{{builtin requires a global or generic pointer}}
@@ -20,3 +30,51 @@ v4u32 test_amdgcn_av_load_b128_bad_ptr(private_ptr_to_v4u32 ptr) {
v4u32 test_amdgcn_av_load_b128_bad_scope(global_ptr_to_v4u32 ptr) {
return __builtin_amdgcn_av_load_b128(ptr, 42); //expected-error{{synchronization scope argument to atomic operation is invalid}}
}
+
+void test_amdgcn_av_store_b16_bad_ptr(private_ptr_to_u16 ptr, u16 data) {
+ __builtin_amdgcn_av_store_b16(ptr, data, __MEMORY_SCOPE_SYSTEM); //expected-error{{builtin requires a global or generic pointer}}
+}
+
+void test_amdgcn_av_store_b16_bad_scope(global_ptr_to_u16 ptr, u16 data) {
+ __builtin_amdgcn_av_store_b16(ptr, data, 42); //expected-error{{synchronization scope argument to atomic operation is invalid}}
+}
+
+u16 test_amdgcn_av_load_b16_bad_ptr(private_ptr_to_u16 ptr) {
+ return __builtin_amdgcn_av_load_b16(ptr, __MEMORY_SCOPE_SYSTEM); //expected-error{{builtin requires a global or generic pointer}}
+}
+
+u16 test_amdgcn_av_load_b16_bad_scope(global_ptr_to_u16 ptr) {
+ return __builtin_amdgcn_av_load_b16(ptr, 42); //expected-error{{synchronization scope argument to atomic operation is invalid}}
+}
+
+void test_amdgcn_av_store_b32_bad_ptr(private_ptr_to_u32 ptr, u32 data) {
+ __builtin_amdgcn_av_store_b32(ptr, data, __MEMORY_SCOPE_SYSTEM); //expected-error{{builtin requires a global or generic pointer}}
+}
+
+void test_amdgcn_av_store_b32_bad_scope(global_ptr_to_u32 ptr, u32 data) {
+ __builtin_amdgcn_av_store_b32(ptr, data, 42); //expected-error{{synchronization scope argument to atomic operation is invalid}}
+}
+
+u32 test_amdgcn_av_load_b32_bad_ptr(private_ptr_to_u32 ptr) {
+ return __builtin_amdgcn_av_load_b32(ptr, __MEMORY_SCOPE_SYSTEM); //expected-error{{builtin requires a global or generic pointer}}
+}
+
+u32 test_amdgcn_av_load_b32_bad_scope(global_ptr_to_u32 ptr) {
+ return __builtin_amdgcn_av_load_b32(ptr, 42); //expected-error{{synchronization scope argument to atomic operation is invalid}}
+}
+
+void test_amdgcn_av_store_b64_bad_ptr(private_ptr_to_u64 ptr, u64 data) {
+ __builtin_amdgcn_av_store_b64(ptr, data, __MEMORY_SCOPE_SYSTEM); //expected-error{{builtin requires a global or generic pointer}}
+}
+
+void test_amdgcn_av_store_b64_bad_scope(global_ptr_to_u64 ptr, u64 data) {
+ __builtin_amdgcn_av_store_b64(ptr, data, 42); //expected-error{{synchronization scope argument to atomic operation is invalid}}
+}
+
+u64 test_amdgcn_av_load_b64_bad_ptr(private_ptr_to_u64 ptr) {
+ return __builtin_amdgcn_av_load_b64(ptr, __MEMORY_SCOPE_SYSTEM); //expected-error{{builtin requires a global or generic pointer}}
+}
+
+u64 test_amdgcn_av_load_b64_bad_scope(global_ptr_to_u64 ptr) {
+ return __builtin_amdgcn_av_load_b64(ptr, 42); //expected-error{{synchronization scope argument to atomic operation is invalid}}
+}
diff --git a/clang/test/SemaOpenCL/builtins-amdgcn-global-load-store-target-error.cl b/clang/test/SemaOpenCL/builtins-amdgcn-global-load-store-target-error.cl
index 35f8e6d41ebb6a..5d6d6d5be688bf 100644
--- a/clang/test/SemaOpenCL/builtins-amdgcn-global-load-store-target-error.cl
+++ b/clang/test/SemaOpenCL/builtins-amdgcn-global-load-store-target-error.cl
@@ -1,24 +1,27 @@
-// RUN: split-file %s %t
-// RUN: %clang_cc1 -cl-std=CL2.0 -triple amdgpu6.02-unknown-unknown -S -verify -o - %t/load.cl
-// RUN: %clang_cc1 -cl-std=CL2.0 -triple amdgpu7.05-unknown-unknown -S -verify -o - %t/load.cl
-// RUN: %clang_cc1 -cl-std=CL2.0 -triple amdgpu8.10-unknown-unknown -S -verify -o - %t/load.cl
-// RUN: %clang_cc1 -cl-std=CL2.0 -triple amdgpu6.02-unknown-unknown -S -verify -o - %t/store.cl
-// RUN: %clang_cc1 -cl-std=CL2.0 -triple amdgpu7.05-unknown-unknown -S -verify -o - %t/store.cl
-// RUN: %clang_cc1 -cl-std=CL2.0 -triple amdgpu8.10-unknown-unknown -S -verify -o - %t/store.cl
+// RUN: %clang_cc1 -cl-std=CL2.0 -triple amdgpu6.02-unknown-unknown -S -verify -o - %s
+// RUN: %clang_cc1 -cl-std=CL2.0 -triple amdgpu7.05-unknown-unknown -S -verify -o - %s
+// RUN: %clang_cc1 -cl-std=CL2.0 -triple amdgpu8.10-unknown-unknown -S -verify -o - %s
// REQUIRES: amdgpu-registered-target
-//--- load.cl
typedef __attribute__((__vector_size__(4 * sizeof(unsigned int)))) unsigned int v4u32;
typedef v4u32 __global *global_ptr_to_v4u32;
+typedef unsigned short u16;
+typedef unsigned int u32;
+typedef unsigned long u64;
+typedef u16 __global *global_ptr_to_u16;
+typedef u32 __global *global_ptr_to_u32;
+typedef u64 __global *global_ptr_to_u64;
-v4u32 test_amdgcn_av_load_b128_target(global_ptr_to_v4u32 ptr) {
- return __builtin_amdgcn_av_load_b128(ptr, 0); // expected-error{{'__builtin_amdgcn_av_load_b128' needs target feature flat-global-insts}}
-}
-
-//--- store.cl
-typedef __attribute__((__vector_size__(4 * sizeof(unsigned int)))) unsigned int v4u32;
-typedef v4u32 __global *global_ptr_to_v4u32;
-
-void test_amdgcn_av_store_b128_target(global_ptr_to_v4u32 ptr, v4u32 data) {
- __builtin_amdgcn_av_store_b128(ptr, data, 0); // expected-error{{'__builtin_amdgcn_av_store_b128' needs target feature flat-global-insts}}
+void test_amdgcn_av_load_store_target(global_ptr_to_u16 ptr16, u16 data16,
+ global_ptr_to_u32 ptr32, u32 data32,
+ global_ptr_to_u64 ptr64, u64 data64,
+ global_ptr_to_v4u32 ptr128, v4u32 data128) {
+ __builtin_amdgcn_av_load_b16(ptr16, 0); // expected-error{{'__builtin_amdgcn_av_load_b16' needs target feature flat-global-insts}}
+ __builtin_amdgcn_av_load_b32(ptr32, 0); // expected-error{{'__builtin_amdgcn_av_load_b32' needs target feature flat-global-insts}}
+ __builtin_amdgcn_av_load_b64(ptr64, 0); // expected-error{{'__builtin_amdgcn_av_load_b64' needs target feature flat-global-insts}}
+ __builtin_amdgcn_av_load_b128(ptr128, 0); // expected-error{{'__builtin_amdgcn_av_load_b128' needs target feature flat-global-insts}}
+ __builtin_amdgcn_av_store_b16(ptr16, data16, 0); // expected-error{{'__builtin_amdgcn_av_store_b16' needs target feature flat-global-insts}}
+ __builtin_amdgcn_av_store_b32(ptr32, data32, 0); // expected-error{{'__builtin_amdgcn_av_store_b32' needs target feature flat-global-insts}}
+ __builtin_amdgcn_av_store_b64(ptr64, data64, 0); // expected-error{{'__builtin_amdgcn_av_store_b64' needs target feature flat-global-insts}}
+ __builtin_amdgcn_av_store_b128(ptr128, data128, 0); // expected-error{{'__builtin_amdgcn_av_store_b128' needs target feature flat-global-insts}}
}
diff --git a/llvm/docs/AMDGPUMemoryModel.md b/llvm/docs/AMDGPUMemoryModel.md
index 0a7f023bf19ac8..9b0e9ee2046d63 100644
--- a/llvm/docs/AMDGPUMemoryModel.md
+++ b/llvm/docs/AMDGPUMemoryModel.md
@@ -152,10 +152,10 @@ then propagates those side-effects up to its scope instance `S` and also all
the subscope instances of `S` that contain `X`.
```llvm
- at llvm.amdgcn.av.store.b128(%addr, %value, metadata !scope)
+ at llvm.amdgcn.av.store.b<N>(%addr, %value, metadata !scope)
```
-The `@llvm.amdgcn.av.store.b128` intrinsic performs a non-atomic
+The `@llvm.amdgcn.av.store.b{16,32,64,128}` intrinsics perform a non-atomic
*store-available* operation on `%addr` with scope `!scope`. See
{ref}`amdgpu-av-load-store` for details.
@@ -178,10 +178,10 @@ addition, it makes the retrieved state visible to all subscope instances of `S`
that contain `Y`.
```llvm
-%value = @llvm.amdgcn.av.load.b128(%addr, metadata !scope)
+%value = @llvm.amdgcn.av.load.b<N>(%addr, metadata !scope)
```
-The `@llvm.amdgcn.av.load.b128` intrinsic performs a non-atomic *load-visible*
+The `@llvm.amdgcn.av.load.b{16,32,64,128}` intrinsics perform a non-atomic *load-visible*
operation on `%addr` with scope `!scope`. See {ref}`amdgpu-av-load-store` for
details.
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 6fa17d9a916e0d..23eb8a2c361fc3 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -2252,6 +2252,22 @@ pointer (``addrspace(0)``). Global pointers select ``global_load``/
``global_store`` instructions; flat pointers select ``flat_load``/
``flat_store`` instructions. The cache policy bits are the same in both cases.
+These intrinsics exist for the following data types:
+
+.. table:: AMDGPU Available/Visible Intrinsics
+ :name: amdgpu-av-intrinsics-table
+
+ ========== ============= ====================================== =======================================
+ Width Data Type Load Store
+ ========== ============= ====================================== =======================================
+ 16 bits ``i16`` ``llvm.amdgcn.av.load.b16`` ``llvm.amdgcn.av.store.b16``
+ 32 bits ``i32`` ``llvm.amdgcn.av.load.b32`` ``llvm.amdgcn.av.store.b32``
+ 64 bits ``i64`` ``llvm.amdgcn.av.load.b64`` ``llvm.amdgcn.av.store.b64``
+ 128 bits ``<4 x i32>`` ``llvm.amdgcn.av.load.b128`` ``llvm.amdgcn.av.store.b128``
+ ========== ============= ====================================== =======================================
+
+Example for 128 bits loads and stores:
+
.. code-block:: llvm
<4 x i32> @llvm.amdgcn.av.load.b128.p1(
@@ -2279,9 +2295,11 @@ This section is informational and for **internal reference only**. Users should
not rely on the expansions described below. The only reliable user-level
guarantees are those provided by the :ref:`AMDGPU memory model<amdgpu-memmodel>`.
-The tables below show the cache policy bits for global pointer variants.
-Flat pointer variants use the corresponding ``flat_load``/``flat_store``
-instructions with the same cache policy bits.
+The tables below show the cache policy bits for the 128 bits global pointer
+variants. Other widths use the ``global_load``/``global_store`` instruction of
+the corresponding width with the same cache policy bits. Flat pointer variants
+use the corresponding ``flat_load``/``flat_store`` instructions with the same
+cache policy bits.
**TODO:** Currently the compiler does not support WGP mode on gfx12+. Hence,
``"workgroup"`` scope currently maps to CU scope (no bits). When WGP mode is
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index fa29d08c91d7bd..87a33170a323d0 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -994,10 +994,10 @@ def int_amdgcn_bitop3 :
[LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty],
[ImmArg<ArgIndex<3>>]>;
-class AMDGPUAVStore : Intrinsic <
+class AMDGPUAVStore<LLVMType ty> : Intrinsic <
[],
[llvm_anyptr_ty, // Pointer to store to (flat or global)
- llvm_v4i32_ty, // Data to store
+ ty, // Data to store
llvm_metadata_ty], // Scope
[ IntrWriteMem, WriteOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>,
NonNull<ArgIndex<0>>, IntrWillReturn, IntrNoCallback, IntrNoFree ],
@@ -1005,8 +1005,8 @@ class AMDGPUAVStore : Intrinsic <
[SDNPMemOperand, SDNPMayStore]
>;
-class AMDGPUAVLoad : Intrinsic <
- [llvm_v4i32_ty],
+class AMDGPUAVLoad<LLVMType ty> : Intrinsic <
+ [ty],
[llvm_anyptr_ty, // Pointer to load from (flat or global)
llvm_metadata_ty], // Scope
[ IntrReadMem, ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>,
@@ -1016,8 +1016,17 @@ class AMDGPUAVLoad : Intrinsic <
>;
let TargetFeatures = "flat-global-insts" in {
-def int_amdgcn_av_store_b128 : AMDGPUAVStore;
-def int_amdgcn_av_load_b128 : AMDGPUAVLoad;
+def int_amdgcn_av_store_b16 : AMDGPUAVStore<llvm_i16_ty>;
+def int_amdgcn_av_load_b16 : AMDGPUAVLoad<llvm_i16_ty>;
+
+def int_amdgcn_av_store_b32 : AMDGPUAVStore<llvm_i32_ty>;
+def int_amdgcn_av_load_b32 : AMDGPUAVLoad<llvm_i32_ty>;
+
+def int_amdgcn_av_store_b64 : AMDGPUAVStore<llvm_i64_ty>;
+def int_amdgcn_av_load_b64 : AMDGPUAVLoad<llvm_i64_ty>;
+
+def int_amdgcn_av_store_b128 : AMDGPUAVStore<llvm_v4i32_ty>;
+def int_amdgcn_av_load_b128 : AMDGPUAVLoad<llvm_v4i32_ty>;
} // End TargetFeatures = "flat-global-insts"
} // TargetPrefix = "amdgcn"
diff --git a/llvm/lib/IR/VerifierAMDGPU.cpp b/llvm/lib/IR/VerifierAMDGPU.cpp
index 5d7b46357d2849..068c95c8a34e41 100644
--- a/llvm/lib/IR/VerifierAMDGPU.cpp
+++ b/llvm/lib/IR/VerifierAMDGPU.cpp
@@ -433,7 +433,13 @@ void llvm::verifyAMDGPUIntrinsicCall(VerifierSupport &VS, Intrinsic::ID ID,
&Call, Op);
break;
}
+ case Intrinsic::amdgcn_av_load_b16:
+ case Intrinsic::amdgcn_av_load_b32:
+ case Intrinsic::amdgcn_av_load_b64:
case Intrinsic::amdgcn_av_load_b128:
+ case Intrinsic::amdgcn_av_store_b16:
+ case Intrinsic::amdgcn_av_store_b32:
+ case Intrinsic::amdgcn_av_store_b64:
case Intrinsic::amdgcn_av_store_b128: {
MetadataAsValue *Op =
cast<MetadataAsValue>(Call.getArgOperand(Call.arg_size() - 1));
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 03987fb3f36b7a..68298fa59bb8b1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -8838,10 +8838,16 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
B.buildStore(MI.getOperand(2), MI.getOperand(1), **MI.memoperands_begin());
MI.eraseFromParent();
return true;
+ case Intrinsic::amdgcn_av_load_b16:
+ case Intrinsic::amdgcn_av_load_b32:
+ case Intrinsic::amdgcn_av_load_b64:
case Intrinsic::amdgcn_av_load_b128:
+ case Intrinsic::amdgcn_av_store_b16:
+ case Intrinsic::amdgcn_av_store_b32:
+ case Intrinsic::amdgcn_av_store_b64:
case Intrinsic::amdgcn_av_store_b128: {
assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!");
- if (IntrID == Intrinsic::amdgcn_av_load_b128)
+ if (MI.getNumExplicitDefs())
B.buildLoad(MI.getOperand(0), MI.getOperand(2), **MI.memoperands_begin());
else
B.buildStore(MI.getOperand(2), MI.getOperand(1),
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 1fc3f8c396e4d8..e12c126e01a17b 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1869,13 +1869,20 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
Infos.push_back(Info);
return;
}
+ case Intrinsic::amdgcn_av_load_b16:
+ case Intrinsic::amdgcn_av_load_b32:
+ case Intrinsic::amdgcn_av_load_b64:
case Intrinsic::amdgcn_av_load_b128:
+ case Intrinsic::amdgcn_av_store_b16:
+ case Intrinsic::amdgcn_av_store_b32:
+ case Intrinsic::amdgcn_av_store_b64:
case Intrinsic::amdgcn_av_store_b128: {
- bool IsStore = IntrID == Intrinsic::amdgcn_av_store_b128;
+ bool IsStore = CI.getType()->isVoidTy();
+ Type *DataTy = IsStore ? CI.getArgOperand(1)->getType() : CI.getType();
Info.opc = IsStore ? ISD::INTRINSIC_VOID : ISD::INTRINSIC_W_CHAIN;
- Info.memVT = MVT::v4i32;
+ Info.memVT = MVT::getVT(DataTy);
Info.ptrVal = CI.getArgOperand(0);
- Info.align = Align(16);
+ Info.align = Align(Info.memVT.getStoreSize());
Info.flags |=
IsStore ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad;
// Pretend to be atomic so that SIMemoryLegalizer::expandStore sets cache
@@ -1991,7 +1998,13 @@ bool SITargetLowering::getAddrModeArguments(const IntrinsicInst *II,
case Intrinsic::amdgcn_global_store_async_from_lds_b32:
case Intrinsic::amdgcn_global_store_async_from_lds_b64:
case Intrinsic::amdgcn_global_store_async_from_lds_b128:
+ case Intrinsic::amdgcn_av_load_b16:
+ case Intrinsic::amdgcn_av_load_b32:
+ case Intrinsic::amdgcn_av_load_b64:
case Intrinsic::amdgcn_av_load_b128:
+ case Intrinsic::amdgcn_av_store_b16:
+ case Intrinsic::amdgcn_av_store_b32:
+ case Intrinsic::amdgcn_av_store_b64:
case Intrinsic::amdgcn_av_store_b128:
Ptr = II->getArgOperand(0);
break;
@@ -12516,6 +12529,9 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
return DAG.getAtomicLoad(ISD::NON_EXTLOAD, DL, MII->getMemoryVT(), VT,
Chain, Ptr, MII->getMemOperand());
}
+ case Intrinsic::amdgcn_av_load_b16:
+ case Intrinsic::amdgcn_av_load_b32:
+ case Intrinsic::amdgcn_av_load_b64:
case Intrinsic::amdgcn_av_load_b128: {
MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Op);
SDValue Chain = Op->getOperand(0);
@@ -13225,6 +13241,9 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
return DAG.getAtomic(ISD::ATOMIC_STORE, DL, MII->getMemoryVT(), Chain, Val,
Ptr, MII->getMemOperand());
}
+ case Intrinsic::amdgcn_av_store_b16:
+ case Intrinsic::amdgcn_av_store_b32:
+ case Intrinsic::amdgcn_av_store_b64:
case Intrinsic::amdgcn_av_store_b128: {
MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Op);
SDValue Chain = Op->getOperand(0);
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn-av-scopes.ll b/llvm/test/CodeGen/AMDGPU/amdgcn-av-scopes.ll
index c81286093c2bdb..e82483652e5a27 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn-av-scopes.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn-av-scopes.ll
@@ -2140,6 +2140,1068 @@ define void @flat_store_cluster(ptr %p, <4 x i32> %v) {
ret void
}
+; scope: system
+define i16 @global_load_b16_system(ptr addrspace(1) %p) {
+; GFX942-SDAG-LABEL: global_load_b16_system:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: global_load_ushort v0, v[0:1], off sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: global_load_b16_system:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: global_load_ushort v0, v[0:1], off glc dlc
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: global_load_b16_system:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: global_load_d16_b16 v0, v[0:1], off glc
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: global_load_b16_system:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_load_u16 v0, v[0:1], off scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: global_load_b16_system:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: global_load_ushort v0, v[0:1], off glc dlc
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: global_load_b16_system:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: global_load_d16_b16 v0, v[0:1], off glc
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: global_load_b16_system:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: global_load_ushort v0, v[0:1], off sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: global_load_b16_system:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: global_load_ushort v0, v[0:1], off glc dlc
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: global_load_b16_system:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: global_load_d16_b16 v0, v[0:1], off glc
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: global_load_b16_system:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: global_load_u16 v0, v[0:1], off scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: global_load_b16_system:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: global_load_ushort v0, v[0:1], off glc dlc
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: global_load_b16_system:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: global_load_d16_b16 v0, v[0:1], off glc
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ %v = call i16 @llvm.amdgcn.av.load.b16.p1(ptr addrspace(1) %p, metadata !3)
+ ret i16 %v
+}
+
+; scope: system
+define void @global_store_b16_system(ptr addrspace(1) %p, i16 %v) {
+; GFX942-SDAG-LABEL: global_store_b16_system:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: global_store_short v[0:1], v2, off sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: global_store_b16_system:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: global_store_short v[0:1], v2, off
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: global_store_b16_system:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: global_store_b16_system:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_store_b16 v[0:1], v2, off scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: global_store_b16_system:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: global_store_short v[0:1], v2, off
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: global_store_b16_system:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: global_store_b16_system:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: global_store_short v[0:1], v2, off sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: global_store_b16_system:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: global_store_short v[0:1], v2, off
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: global_store_b16_system:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: global_store_b16_system:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: global_store_b16 v[0:1], v2, off scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: global_store_b16_system:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: global_store_short v[0:1], v2, off
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: global_store_b16_system:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ call void @llvm.amdgcn.av.store.b16.p1(ptr addrspace(1) %p, i16 %v, metadata !3)
+ ret void
+}
+
+; scope: system
+define i16 @flat_load_b16_system(ptr %p) {
+; GFX942-SDAG-LABEL: flat_load_b16_system:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: flat_load_ushort v0, v[0:1] sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: flat_load_b16_system:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: flat_load_ushort v0, v[0:1] glc dlc
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: flat_load_b16_system:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: flat_load_d16_b16 v0, v[0:1] glc
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: flat_load_b16_system:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: flat_load_u16 v0, v[0:1] scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: flat_load_b16_system:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: flat_load_ushort v0, v[0:1] glc dlc
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: flat_load_b16_system:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: flat_load_d16_b16 v0, v[0:1] glc
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: flat_load_b16_system:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: flat_load_ushort v0, v[0:1] sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: flat_load_b16_system:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: flat_load_ushort v0, v[0:1] glc dlc
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: flat_load_b16_system:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: flat_load_d16_b16 v0, v[0:1] glc
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: flat_load_b16_system:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: flat_load_u16 v0, v[0:1] scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: flat_load_b16_system:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: flat_load_ushort v0, v[0:1] glc dlc
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: flat_load_b16_system:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: flat_load_d16_b16 v0, v[0:1] glc
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ %v = call i16 @llvm.amdgcn.av.load.b16.p0(ptr %p, metadata !3)
+ ret i16 %v
+}
+
+; scope: system
+define void @flat_store_b16_system(ptr %p, i16 %v) {
+; GFX942-SDAG-LABEL: flat_store_b16_system:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: flat_store_short v[0:1], v2 sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: flat_store_b16_system:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: flat_store_short v[0:1], v2
+; GFX1012-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: flat_store_b16_system:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: flat_store_b16 v[0:1], v2
+; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: flat_store_b16_system:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: flat_store_b16 v[0:1], v2 scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: flat_store_b16_system:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: flat_store_short v[0:1], v2
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: flat_store_b16_system:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: flat_store_b16 v[0:1], v2
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: flat_store_b16_system:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: flat_store_short v[0:1], v2 sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: flat_store_b16_system:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: flat_store_short v[0:1], v2
+; GFX1012-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: flat_store_b16_system:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: flat_store_b16 v[0:1], v2
+; GFX1100-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: flat_store_b16_system:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: flat_store_b16 v[0:1], v2 scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_wait_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: flat_store_b16_system:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: flat_store_short v[0:1], v2
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: flat_store_b16_system:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: flat_store_b16 v[0:1], v2
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ call void @llvm.amdgcn.av.store.b16.p0(ptr %p, i16 %v, metadata !3)
+ ret void
+}
+
+; scope: system
+define i32 @global_load_b32_system(ptr addrspace(1) %p) {
+; GFX942-SDAG-LABEL: global_load_b32_system:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: global_load_dword v0, v[0:1], off sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: global_load_b32_system:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: global_load_dword v0, v[0:1], off glc dlc
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: global_load_b32_system:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: global_load_b32_system:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: global_load_b32_system:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: global_load_dword v0, v[0:1], off glc dlc
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: global_load_b32_system:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: global_load_b32_system:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: global_load_dword v0, v[0:1], off sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: global_load_b32_system:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: global_load_dword v0, v[0:1], off glc dlc
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: global_load_b32_system:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: global_load_b32_system:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: global_load_b32_system:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: global_load_dword v0, v[0:1], off glc dlc
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: global_load_b32_system:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ %v = call i32 @llvm.amdgcn.av.load.b32.p1(ptr addrspace(1) %p, metadata !3)
+ ret i32 %v
+}
+
+; scope: system
+define void @global_store_b32_system(ptr addrspace(1) %p, i32 %v) {
+; GFX942-SDAG-LABEL: global_store_b32_system:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: global_store_dword v[0:1], v2, off sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: global_store_b32_system:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: global_store_dword v[0:1], v2, off
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: global_store_b32_system:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: global_store_b32 v[0:1], v2, off
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: global_store_b32_system:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_store_b32 v[0:1], v2, off scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: global_store_b32_system:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: global_store_dword v[0:1], v2, off
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: global_store_b32_system:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: global_store_b32 v[0:1], v2, off
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: global_store_b32_system:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: global_store_dword v[0:1], v2, off sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: global_store_b32_system:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: global_store_dword v[0:1], v2, off
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: global_store_b32_system:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: global_store_b32 v[0:1], v2, off
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: global_store_b32_system:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: global_store_b32 v[0:1], v2, off scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: global_store_b32_system:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: global_store_dword v[0:1], v2, off
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: global_store_b32_system:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: global_store_b32 v[0:1], v2, off
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ call void @llvm.amdgcn.av.store.b32.p1(ptr addrspace(1) %p, i32 %v, metadata !3)
+ ret void
+}
+
+; scope: system
+define i32 @flat_load_b32_system(ptr %p) {
+; GFX942-SDAG-LABEL: flat_load_b32_system:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: flat_load_dword v0, v[0:1] sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: flat_load_b32_system:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: flat_load_dword v0, v[0:1] glc dlc
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: flat_load_b32_system:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: flat_load_b32 v0, v[0:1] glc
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: flat_load_b32_system:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: flat_load_b32_system:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: flat_load_dword v0, v[0:1] glc dlc
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: flat_load_b32_system:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: flat_load_b32 v0, v[0:1] glc
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: flat_load_b32_system:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: flat_load_dword v0, v[0:1] sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: flat_load_b32_system:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: flat_load_dword v0, v[0:1] glc dlc
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: flat_load_b32_system:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: flat_load_b32 v0, v[0:1] glc
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: flat_load_b32_system:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: flat_load_b32_system:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: flat_load_dword v0, v[0:1] glc dlc
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: flat_load_b32_system:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: flat_load_b32 v0, v[0:1] glc
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ %v = call i32 @llvm.amdgcn.av.load.b32.p0(ptr %p, metadata !3)
+ ret i32 %v
+}
+
+; scope: system
+define void @flat_store_b32_system(ptr %p, i32 %v) {
+; GFX942-SDAG-LABEL: flat_store_b32_system:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: flat_store_b32_system:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: flat_store_dword v[0:1], v2
+; GFX1012-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: flat_store_b32_system:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: flat_store_b32 v[0:1], v2
+; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: flat_store_b32_system:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: flat_store_b32_system:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: flat_store_dword v[0:1], v2
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: flat_store_b32_system:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: flat_store_b32 v[0:1], v2
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: flat_store_b32_system:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: flat_store_b32_system:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: flat_store_dword v[0:1], v2
+; GFX1012-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: flat_store_b32_system:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: flat_store_b32 v[0:1], v2
+; GFX1100-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: flat_store_b32_system:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_wait_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: flat_store_b32_system:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: flat_store_dword v[0:1], v2
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: flat_store_b32_system:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: flat_store_b32 v[0:1], v2
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ call void @llvm.amdgcn.av.store.b32.p0(ptr %p, i32 %v, metadata !3)
+ ret void
+}
+
+; scope: system
+define i64 @global_load_b64_system(ptr addrspace(1) %p) {
+; GFX942-SDAG-LABEL: global_load_b64_system:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: global_load_dwordx2 v[0:1], v[0:1], off sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: global_load_b64_system:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc dlc
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: global_load_b64_system:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: global_load_b64_system:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: global_load_b64_system:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc dlc
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: global_load_b64_system:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: global_load_b64_system:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: global_load_dwordx2 v[0:1], v[0:1], off sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: global_load_b64_system:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc dlc
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: global_load_b64_system:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: global_load_b64_system:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: global_load_b64_system:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc dlc
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: global_load_b64_system:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ %v = call i64 @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) %p, metadata !3)
+ ret i64 %v
+}
+
+; scope: system
+define void @global_store_b64_system(ptr addrspace(1) %p, i64 %v) {
+; GFX942-SDAG-LABEL: global_store_b64_system:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: global_store_b64_system:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: global_store_b64_system:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: global_store_b64_system:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: global_store_b64_system:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: global_store_b64_system:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: global_store_b64_system:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: global_store_dwordx2 v[0:1], v[2:3], off sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: global_store_b64_system:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: global_store_b64_system:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: global_store_b64_system:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: global_store_b64 v[0:1], v[2:3], off scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: global_store_b64_system:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: global_store_b64_system:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) %p, i64 %v, metadata !3)
+ ret void
+}
+
+; scope: system
+define i64 @flat_load_b64_system(ptr %p) {
+; GFX942-SDAG-LABEL: flat_load_b64_system:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: flat_load_dwordx2 v[0:1], v[0:1] sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: flat_load_b64_system:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc dlc
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: flat_load_b64_system:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] glc
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: flat_load_b64_system:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: flat_load_b64_system:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc dlc
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: flat_load_b64_system:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] glc
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: flat_load_b64_system:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: flat_load_dwordx2 v[0:1], v[0:1] sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: flat_load_b64_system:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc dlc
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: flat_load_b64_system:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: flat_load_b64 v[0:1], v[0:1] glc
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: flat_load_b64_system:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: flat_load_b64_system:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc dlc
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: flat_load_b64_system:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: flat_load_b64 v[0:1], v[0:1] glc
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ %v = call i64 @llvm.amdgcn.av.load.b64.p0(ptr %p, metadata !3)
+ ret i64 %v
+}
+
+; scope: system
+define void @flat_store_b64_system(ptr %p, i64 %v) {
+; GFX942-SDAG-LABEL: flat_store_b64_system:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: flat_store_b64_system:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; GFX1012-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: flat_store_b64_system:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: flat_store_b64 v[0:1], v[2:3]
+; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: flat_store_b64_system:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: flat_store_b64 v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: flat_store_b64_system:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: flat_store_b64_system:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: flat_store_b64 v[0:1], v[2:3]
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: flat_store_b64_system:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: flat_store_dwordx2 v[0:1], v[2:3] sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: flat_store_b64_system:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; GFX1012-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: flat_store_b64_system:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: flat_store_b64 v[0:1], v[2:3]
+; GFX1100-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: flat_store_b64_system:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: flat_store_b64 v[0:1], v[2:3] scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_wait_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: flat_store_b64_system:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: flat_store_b64_system:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: flat_store_b64 v[0:1], v[2:3]
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ call void @llvm.amdgcn.av.store.b64.p0(ptr %p, i64 %v, metadata !3)
+ ret void
+}
+
!0 = !{!"wavefront"}
!1 = !{!"workgroup"}
!2 = !{!"agent"}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.load.b128.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.load.b128.ll
index 0413dfa292d11d..173ad8ce7463dd 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.load.b128.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.load.b128.ll
@@ -866,6 +866,642 @@ entry:
ret <4 x i32> %data
}
+;; vgpr pointer, system scope
+define i16 @av_global_load_b16_0_11(ptr addrspace(1) %addr) {
+; GFX906-SDAG-LABEL: av_global_load_b16_0_11:
+; GFX906-SDAG: ; %bb.0: ; %entry
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-SDAG-NEXT: global_load_ushort v0, v[0:1], off glc
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX906-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-SDAG-LABEL: av_global_load_b16_0_11:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: global_load_ushort v0, v[0:1], off sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: av_global_load_b16_0_11:
+; GFX1012-SDAG: ; %bb.0: ; %entry
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: global_load_ushort v0, v[0:1], off glc dlc
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: av_global_load_b16_0_11:
+; GFX1100-SDAG: ; %bb.0: ; %entry
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: global_load_d16_b16 v0, v[0:1], off glc
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: av_global_load_b16_0_11:
+; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_load_u16 v0, v[0:1], off scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-SDAG-LABEL: av_global_load_b16_0_11:
+; GFX1310-SDAG: ; %bb.0: ; %entry
+; GFX1310-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1310-SDAG-NEXT: global_load_u16 v0, v[0:1], off scope:SCOPE_SYS
+; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX906-ISEL-LABEL: av_global_load_b16_0_11:
+; GFX906-ISEL: ; %bb.0: ; %entry
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-ISEL-NEXT: global_load_ushort v0, v[0:1], off glc
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX906-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: av_global_load_b16_0_11:
+; GFX942-ISEL: ; %bb.0: ; %entry
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: global_load_ushort v0, v[0:1], off sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: av_global_load_b16_0_11:
+; GFX1012-ISEL: ; %bb.0: ; %entry
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: global_load_ushort v0, v[0:1], off glc dlc
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: av_global_load_b16_0_11:
+; GFX1100-ISEL: ; %bb.0: ; %entry
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: global_load_d16_b16 v0, v[0:1], off glc
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: av_global_load_b16_0_11:
+; GFX1250-ISEL: ; %bb.0: ; %entry
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: global_load_u16 v0, v[0:1], off scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-ISEL-LABEL: av_global_load_b16_0_11:
+; GFX1310-ISEL: ; %bb.0: ; %entry
+; GFX1310-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_expcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1310-ISEL-NEXT: global_load_u16 v0, v[0:1], off scope:SCOPE_SYS
+; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %data = call i16 @llvm.amdgcn.av.load.b16.p1(ptr addrspace(1) %addr, metadata !3)
+ ret i16 %data
+}
+
+;; sgpr pointer, system scope
+define i16 @av_global_load_b16_saddr_0_03(ptr addrspace(1) inreg %addr) {
+; GFX906-SDAG-LABEL: av_global_load_b16_saddr_0_03:
+; GFX906-SDAG: ; %bb.0: ; %entry
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX906-SDAG-NEXT: global_load_ushort v0, v0, s[16:17] glc
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX906-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-SDAG-LABEL: av_global_load_b16_saddr_0_03:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-SDAG-NEXT: global_load_ushort v0, v0, s[0:1] sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: av_global_load_b16_saddr_0_03:
+; GFX1012-SDAG: ; %bb.0: ; %entry
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1012-SDAG-NEXT: global_load_ushort v0, v0, s[16:17] glc dlc
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: av_global_load_b16_saddr_0_03:
+; GFX1100-SDAG: ; %bb.0: ; %entry
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1100-SDAG-NEXT: global_load_d16_b16 v0, v0, s[0:1] glc
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: av_global_load_b16_saddr_0_03:
+; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-SDAG-NEXT: global_load_u16 v0, v0, s[0:1] scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-SDAG-LABEL: av_global_load_b16_saddr_0_03:
+; GFX1310-SDAG: ; %bb.0: ; %entry
+; GFX1310-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1310-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1310-SDAG-NEXT: global_load_u16 v0, v0, s[0:1] scope:SCOPE_SYS
+; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX906-ISEL-LABEL: av_global_load_b16_saddr_0_03:
+; GFX906-ISEL: ; %bb.0: ; %entry
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX906-ISEL-NEXT: global_load_ushort v0, v0, s[16:17] glc
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX906-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: av_global_load_b16_saddr_0_03:
+; GFX942-ISEL: ; %bb.0: ; %entry
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-ISEL-NEXT: global_load_ushort v0, v0, s[0:1] sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: av_global_load_b16_saddr_0_03:
+; GFX1012-ISEL: ; %bb.0: ; %entry
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1012-ISEL-NEXT: global_load_ushort v0, v0, s[16:17] glc dlc
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: av_global_load_b16_saddr_0_03:
+; GFX1100-ISEL: ; %bb.0: ; %entry
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1100-ISEL-NEXT: global_load_d16_b16 v0, v0, s[0:1] glc
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: av_global_load_b16_saddr_0_03:
+; GFX1250-ISEL: ; %bb.0: ; %entry
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-ISEL-NEXT: global_load_u16 v0, v0, s[0:1] scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-ISEL-LABEL: av_global_load_b16_saddr_0_03:
+; GFX1310-ISEL: ; %bb.0: ; %entry
+; GFX1310-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_expcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1310-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1310-ISEL-NEXT: global_load_u16 v0, v0, s[0:1] scope:SCOPE_SYS
+; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %data = call i16 @llvm.amdgcn.av.load.b16.p1(ptr addrspace(1) %addr, metadata !3)
+ ret i16 %data
+}
+
+;; vgpr pointer, system scope
+define i32 @av_global_load_b32_0_11(ptr addrspace(1) %addr) {
+; GFX906-SDAG-LABEL: av_global_load_b32_0_11:
+; GFX906-SDAG: ; %bb.0: ; %entry
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-SDAG-NEXT: global_load_dword v0, v[0:1], off glc
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX906-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-SDAG-LABEL: av_global_load_b32_0_11:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: global_load_dword v0, v[0:1], off sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: av_global_load_b32_0_11:
+; GFX1012-SDAG: ; %bb.0: ; %entry
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: global_load_dword v0, v[0:1], off glc dlc
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: av_global_load_b32_0_11:
+; GFX1100-SDAG: ; %bb.0: ; %entry
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: av_global_load_b32_0_11:
+; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-SDAG-LABEL: av_global_load_b32_0_11:
+; GFX1310-SDAG: ; %bb.0: ; %entry
+; GFX1310-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1310-SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_SYS
+; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX906-ISEL-LABEL: av_global_load_b32_0_11:
+; GFX906-ISEL: ; %bb.0: ; %entry
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-ISEL-NEXT: global_load_dword v0, v[0:1], off glc
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX906-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: av_global_load_b32_0_11:
+; GFX942-ISEL: ; %bb.0: ; %entry
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: global_load_dword v0, v[0:1], off sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: av_global_load_b32_0_11:
+; GFX1012-ISEL: ; %bb.0: ; %entry
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: global_load_dword v0, v[0:1], off glc dlc
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: av_global_load_b32_0_11:
+; GFX1100-ISEL: ; %bb.0: ; %entry
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: av_global_load_b32_0_11:
+; GFX1250-ISEL: ; %bb.0: ; %entry
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-ISEL-LABEL: av_global_load_b32_0_11:
+; GFX1310-ISEL: ; %bb.0: ; %entry
+; GFX1310-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_expcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1310-ISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_SYS
+; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %data = call i32 @llvm.amdgcn.av.load.b32.p1(ptr addrspace(1) %addr, metadata !3)
+ ret i32 %data
+}
+
+;; sgpr pointer, system scope
+define i32 @av_global_load_b32_saddr_0_03(ptr addrspace(1) inreg %addr) {
+; GFX906-SDAG-LABEL: av_global_load_b32_saddr_0_03:
+; GFX906-SDAG: ; %bb.0: ; %entry
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX906-SDAG-NEXT: global_load_dword v0, v0, s[16:17] glc
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX906-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-SDAG-LABEL: av_global_load_b32_saddr_0_03:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-SDAG-NEXT: global_load_dword v0, v0, s[0:1] sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: av_global_load_b32_saddr_0_03:
+; GFX1012-SDAG: ; %bb.0: ; %entry
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1012-SDAG-NEXT: global_load_dword v0, v0, s[16:17] glc dlc
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: av_global_load_b32_saddr_0_03:
+; GFX1100-SDAG: ; %bb.0: ; %entry
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1100-SDAG-NEXT: global_load_b32 v0, v0, s[0:1] glc
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: av_global_load_b32_saddr_0_03:
+; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-SDAG-NEXT: global_load_b32 v0, v0, s[0:1] scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-SDAG-LABEL: av_global_load_b32_saddr_0_03:
+; GFX1310-SDAG: ; %bb.0: ; %entry
+; GFX1310-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1310-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1310-SDAG-NEXT: global_load_b32 v0, v0, s[0:1] scope:SCOPE_SYS
+; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX906-ISEL-LABEL: av_global_load_b32_saddr_0_03:
+; GFX906-ISEL: ; %bb.0: ; %entry
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX906-ISEL-NEXT: global_load_dword v0, v0, s[16:17] glc
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX906-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: av_global_load_b32_saddr_0_03:
+; GFX942-ISEL: ; %bb.0: ; %entry
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-ISEL-NEXT: global_load_dword v0, v0, s[0:1] sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: av_global_load_b32_saddr_0_03:
+; GFX1012-ISEL: ; %bb.0: ; %entry
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1012-ISEL-NEXT: global_load_dword v0, v0, s[16:17] glc dlc
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: av_global_load_b32_saddr_0_03:
+; GFX1100-ISEL: ; %bb.0: ; %entry
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1100-ISEL-NEXT: global_load_b32 v0, v0, s[0:1] glc
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: av_global_load_b32_saddr_0_03:
+; GFX1250-ISEL: ; %bb.0: ; %entry
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-ISEL-NEXT: global_load_b32 v0, v0, s[0:1] scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-ISEL-LABEL: av_global_load_b32_saddr_0_03:
+; GFX1310-ISEL: ; %bb.0: ; %entry
+; GFX1310-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_expcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1310-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1310-ISEL-NEXT: global_load_b32 v0, v0, s[0:1] scope:SCOPE_SYS
+; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %data = call i32 @llvm.amdgcn.av.load.b32.p1(ptr addrspace(1) %addr, metadata !3)
+ ret i32 %data
+}
+
+;; vgpr pointer, system scope
+define i64 @av_global_load_b64_0_11(ptr addrspace(1) %addr) {
+; GFX906-SDAG-LABEL: av_global_load_b64_0_11:
+; GFX906-SDAG: ; %bb.0: ; %entry
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-SDAG-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX906-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-SDAG-LABEL: av_global_load_b64_0_11:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: global_load_dwordx2 v[0:1], v[0:1], off sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: av_global_load_b64_0_11:
+; GFX1012-SDAG: ; %bb.0: ; %entry
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc dlc
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: av_global_load_b64_0_11:
+; GFX1100-SDAG: ; %bb.0: ; %entry
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: av_global_load_b64_0_11:
+; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-SDAG-LABEL: av_global_load_b64_0_11:
+; GFX1310-SDAG: ; %bb.0: ; %entry
+; GFX1310-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1310-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
+; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX906-ISEL-LABEL: av_global_load_b64_0_11:
+; GFX906-ISEL: ; %bb.0: ; %entry
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-ISEL-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX906-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: av_global_load_b64_0_11:
+; GFX942-ISEL: ; %bb.0: ; %entry
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: global_load_dwordx2 v[0:1], v[0:1], off sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: av_global_load_b64_0_11:
+; GFX1012-ISEL: ; %bb.0: ; %entry
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc dlc
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: av_global_load_b64_0_11:
+; GFX1100-ISEL: ; %bb.0: ; %entry
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: av_global_load_b64_0_11:
+; GFX1250-ISEL: ; %bb.0: ; %entry
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-ISEL-LABEL: av_global_load_b64_0_11:
+; GFX1310-ISEL: ; %bb.0: ; %entry
+; GFX1310-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_expcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1310-ISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_SYS
+; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %data = call i64 @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) %addr, metadata !3)
+ ret i64 %data
+}
+
+;; sgpr pointer, system scope
+define i64 @av_global_load_b64_saddr_0_03(ptr addrspace(1) inreg %addr) {
+; GFX906-SDAG-LABEL: av_global_load_b64_saddr_0_03:
+; GFX906-SDAG: ; %bb.0: ; %entry
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX906-SDAG-NEXT: global_load_dwordx2 v[0:1], v0, s[16:17] glc
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX906-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-SDAG-LABEL: av_global_load_b64_saddr_0_03:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-SDAG-NEXT: global_load_dwordx2 v[0:1], v0, s[0:1] sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: av_global_load_b64_saddr_0_03:
+; GFX1012-SDAG: ; %bb.0: ; %entry
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1012-SDAG-NEXT: global_load_dwordx2 v[0:1], v0, s[16:17] glc dlc
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: av_global_load_b64_saddr_0_03:
+; GFX1100-SDAG: ; %bb.0: ; %entry
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1100-SDAG-NEXT: global_load_b64 v[0:1], v0, s[0:1] glc
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: av_global_load_b64_saddr_0_03:
+; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-SDAG-NEXT: global_load_b64 v[0:1], v0, s[0:1] scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-SDAG-LABEL: av_global_load_b64_saddr_0_03:
+; GFX1310-SDAG: ; %bb.0: ; %entry
+; GFX1310-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1310-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1310-SDAG-NEXT: global_load_b64 v[0:1], v0, s[0:1] scope:SCOPE_SYS
+; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX906-ISEL-LABEL: av_global_load_b64_saddr_0_03:
+; GFX906-ISEL: ; %bb.0: ; %entry
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX906-ISEL-NEXT: global_load_dwordx2 v[0:1], v0, s[16:17] glc
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX906-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: av_global_load_b64_saddr_0_03:
+; GFX942-ISEL: ; %bb.0: ; %entry
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-ISEL-NEXT: global_load_dwordx2 v[0:1], v0, s[0:1] sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: av_global_load_b64_saddr_0_03:
+; GFX1012-ISEL: ; %bb.0: ; %entry
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1012-ISEL-NEXT: global_load_dwordx2 v[0:1], v0, s[16:17] glc dlc
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: av_global_load_b64_saddr_0_03:
+; GFX1100-ISEL: ; %bb.0: ; %entry
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1100-ISEL-NEXT: global_load_b64 v[0:1], v0, s[0:1] glc
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: av_global_load_b64_saddr_0_03:
+; GFX1250-ISEL: ; %bb.0: ; %entry
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-ISEL-NEXT: global_load_b64 v[0:1], v0, s[0:1] scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-ISEL-LABEL: av_global_load_b64_saddr_0_03:
+; GFX1310-ISEL: ; %bb.0: ; %entry
+; GFX1310-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_expcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1310-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1310-ISEL-NEXT: global_load_b64 v[0:1], v0, s[0:1] scope:SCOPE_SYS
+; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %data = call i64 @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) %addr, metadata !3)
+ ret i64 %data
+}
+
;;==============================================================================
;; Signed offset addressing modes (derived from global-saddr-load.ll)
;;==============================================================================
@@ -7775,11 +8411,11 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX906-SDAG: ; %bb.0: ; %bb
; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX906-SDAG-NEXT: s_mov_b32 s4, -1
-; GFX906-SDAG-NEXT: .LBB60_1: ; %bb3
+; GFX906-SDAG-NEXT: .LBB66_1: ; %bb3
; GFX906-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX906-SDAG-NEXT: s_add_i32 s4, s4, 1
; GFX906-SDAG-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX906-SDAG-NEXT: s_cbranch_scc0 .LBB60_1
+; GFX906-SDAG-NEXT: s_cbranch_scc0 .LBB66_1
; GFX906-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX906-SDAG-NEXT: s_mov_b32 s5, 0
; GFX906-SDAG-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
@@ -7794,11 +8430,11 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX942-SDAG: ; %bb.0: ; %bb
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_mov_b32 s0, -1
-; GFX942-SDAG-NEXT: .LBB60_1: ; %bb3
+; GFX942-SDAG-NEXT: .LBB66_1: ; %bb3
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_i32 s0, s0, 1
; GFX942-SDAG-NEXT: s_cmpk_eq_i32 s0, 0xff
-; GFX942-SDAG-NEXT: s_cbranch_scc0 .LBB60_1
+; GFX942-SDAG-NEXT: s_cbranch_scc0 .LBB66_1
; GFX942-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX942-SDAG-NEXT: s_mov_b32 s1, 0
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[0:1], 2, v[0:1]
@@ -7810,11 +8446,11 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX1012-SDAG: ; %bb.0: ; %bb
; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1012-SDAG-NEXT: s_mov_b32 s4, -1
-; GFX1012-SDAG-NEXT: .LBB60_1: ; %bb3
+; GFX1012-SDAG-NEXT: .LBB66_1: ; %bb3
; GFX1012-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1012-SDAG-NEXT: s_add_i32 s4, s4, 1
; GFX1012-SDAG-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX1012-SDAG-NEXT: s_cbranch_scc0 .LBB60_1
+; GFX1012-SDAG-NEXT: s_cbranch_scc0 .LBB66_1
; GFX1012-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1012-SDAG-NEXT: s_mov_b32 s5, 0
; GFX1012-SDAG-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
@@ -7828,12 +8464,12 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX1100-SDAG: ; %bb.0: ; %bb
; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-SDAG-NEXT: s_mov_b32 s0, -1
-; GFX1100-SDAG-NEXT: .LBB60_1: ; %bb3
+; GFX1100-SDAG-NEXT: .LBB66_1: ; %bb3
; GFX1100-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1100-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1100-SDAG-NEXT: s_add_i32 s0, s0, 1
; GFX1100-SDAG-NEXT: s_cmpk_eq_i32 s0, 0xff
-; GFX1100-SDAG-NEXT: s_cbranch_scc0 .LBB60_1
+; GFX1100-SDAG-NEXT: s_cbranch_scc0 .LBB66_1
; GFX1100-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1100-SDAG-NEXT: s_mov_b32 s1, 0
; GFX1100-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -7850,12 +8486,12 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: s_mov_b32 s0, -1
-; GFX1250-SDAG-NEXT: .LBB60_1: ; %bb3
+; GFX1250-SDAG-NEXT: .LBB66_1: ; %bb3
; GFX1250-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-NEXT: s_add_co_i32 s0, s0, 1
; GFX1250-SDAG-NEXT: s_cmp_eq_u32 s0, 0xff
-; GFX1250-SDAG-NEXT: s_cbranch_scc0 .LBB60_1
+; GFX1250-SDAG-NEXT: s_cbranch_scc0 .LBB66_1
; GFX1250-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1250-SDAG-NEXT: s_mov_b32 s1, 0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -7872,12 +8508,12 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1310-SDAG-NEXT: s_mov_b32 s0, -1
-; GFX1310-SDAG-NEXT: .LBB60_1: ; %bb3
+; GFX1310-SDAG-NEXT: .LBB66_1: ; %bb3
; GFX1310-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1310-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1310-SDAG-NEXT: s_add_co_i32 s0, s0, 1
; GFX1310-SDAG-NEXT: s_cmp_eq_u32 s0, 0xff
-; GFX1310-SDAG-NEXT: s_cbranch_scc0 .LBB60_1
+; GFX1310-SDAG-NEXT: s_cbranch_scc0 .LBB66_1
; GFX1310-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1310-SDAG-NEXT: s_mov_b32 s1, 0
; GFX1310-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -7893,11 +8529,11 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX906-ISEL: ; %bb.0: ; %bb
; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX906-ISEL-NEXT: s_mov_b32 s4, -1
-; GFX906-ISEL-NEXT: .LBB60_1: ; %bb3
+; GFX906-ISEL-NEXT: .LBB66_1: ; %bb3
; GFX906-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX906-ISEL-NEXT: s_add_i32 s4, s4, 1
; GFX906-ISEL-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX906-ISEL-NEXT: s_cbranch_scc0 .LBB60_1
+; GFX906-ISEL-NEXT: s_cbranch_scc0 .LBB66_1
; GFX906-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX906-ISEL-NEXT: s_mov_b32 s5, 0
; GFX906-ISEL-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
@@ -7913,11 +8549,11 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX942-ISEL: ; %bb.0: ; %bb
; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-ISEL-NEXT: s_mov_b32 s0, -1
-; GFX942-ISEL-NEXT: .LBB60_1: ; %bb3
+; GFX942-ISEL-NEXT: .LBB66_1: ; %bb3
; GFX942-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-ISEL-NEXT: s_add_i32 s0, s0, 1
; GFX942-ISEL-NEXT: s_cmpk_eq_i32 s0, 0xff
-; GFX942-ISEL-NEXT: s_cbranch_scc0 .LBB60_1
+; GFX942-ISEL-NEXT: s_cbranch_scc0 .LBB66_1
; GFX942-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX942-ISEL-NEXT: s_mov_b32 s1, 0
; GFX942-ISEL-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
@@ -7933,11 +8569,11 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX1012-ISEL: ; %bb.0: ; %bb
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1012-ISEL-NEXT: s_mov_b32 s4, -1
-; GFX1012-ISEL-NEXT: .LBB60_1: ; %bb3
+; GFX1012-ISEL-NEXT: .LBB66_1: ; %bb3
; GFX1012-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1012-ISEL-NEXT: s_add_i32 s4, s4, 1
; GFX1012-ISEL-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX1012-ISEL-NEXT: s_cbranch_scc0 .LBB60_1
+; GFX1012-ISEL-NEXT: s_cbranch_scc0 .LBB66_1
; GFX1012-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1012-ISEL-NEXT: s_mov_b32 s5, 0
; GFX1012-ISEL-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
@@ -7953,12 +8589,12 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX1100-ISEL: ; %bb.0: ; %bb
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-ISEL-NEXT: s_mov_b32 s0, -1
-; GFX1100-ISEL-NEXT: .LBB60_1: ; %bb3
+; GFX1100-ISEL-NEXT: .LBB66_1: ; %bb3
; GFX1100-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1100-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1100-ISEL-NEXT: s_add_i32 s0, s0, 1
; GFX1100-ISEL-NEXT: s_cmpk_eq_i32 s0, 0xff
-; GFX1100-ISEL-NEXT: s_cbranch_scc0 .LBB60_1
+; GFX1100-ISEL-NEXT: s_cbranch_scc0 .LBB66_1
; GFX1100-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1100-ISEL-NEXT: s_mov_b32 s1, 0
; GFX1100-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -7976,12 +8612,12 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-ISEL-NEXT: s_mov_b32 s0, -1
-; GFX1250-ISEL-NEXT: .LBB60_1: ; %bb3
+; GFX1250-ISEL-NEXT: .LBB66_1: ; %bb3
; GFX1250-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-ISEL-NEXT: s_add_co_i32 s0, s0, 1
; GFX1250-ISEL-NEXT: s_cmp_eq_u32 s0, 0xff
-; GFX1250-ISEL-NEXT: s_cbranch_scc0 .LBB60_1
+; GFX1250-ISEL-NEXT: s_cbranch_scc0 .LBB66_1
; GFX1250-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1250-ISEL-NEXT: s_mov_b32 s1, 0
; GFX1250-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -8002,12 +8638,12 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
; GFX1310-ISEL-NEXT: s_mov_b32 s0, -1
-; GFX1310-ISEL-NEXT: .LBB60_1: ; %bb3
+; GFX1310-ISEL-NEXT: .LBB66_1: ; %bb3
; GFX1310-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1310-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1310-ISEL-NEXT: s_add_co_i32 s0, s0, 1
; GFX1310-ISEL-NEXT: s_cmp_eq_u32 s0, 0xff
-; GFX1310-ISEL-NEXT: s_cbranch_scc0 .LBB60_1
+; GFX1310-ISEL-NEXT: s_cbranch_scc0 .LBB66_1
; GFX1310-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1310-ISEL-NEXT: s_mov_b32 s1, 0
; GFX1310-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -8044,11 +8680,11 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX906-SDAG: ; %bb.0: ; %bb
; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX906-SDAG-NEXT: s_mov_b32 s4, -1
-; GFX906-SDAG-NEXT: .LBB61_1: ; %bb5
+; GFX906-SDAG-NEXT: .LBB67_1: ; %bb5
; GFX906-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX906-SDAG-NEXT: s_add_i32 s4, s4, 1
; GFX906-SDAG-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX906-SDAG-NEXT: s_cbranch_scc0 .LBB61_1
+; GFX906-SDAG-NEXT: s_cbranch_scc0 .LBB67_1
; GFX906-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX906-SDAG-NEXT: s_mov_b32 s5, 0
; GFX906-SDAG-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
@@ -8063,11 +8699,11 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX942-SDAG: ; %bb.0: ; %bb
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_mov_b32 s0, -1
-; GFX942-SDAG-NEXT: .LBB61_1: ; %bb5
+; GFX942-SDAG-NEXT: .LBB67_1: ; %bb5
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_i32 s0, s0, 1
; GFX942-SDAG-NEXT: s_cmpk_eq_i32 s0, 0xff
-; GFX942-SDAG-NEXT: s_cbranch_scc0 .LBB61_1
+; GFX942-SDAG-NEXT: s_cbranch_scc0 .LBB67_1
; GFX942-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX942-SDAG-NEXT: s_mov_b32 s1, 0
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[0:1], 2, v[0:1]
@@ -8079,11 +8715,11 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX1012-SDAG: ; %bb.0: ; %bb
; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1012-SDAG-NEXT: s_mov_b32 s4, -1
-; GFX1012-SDAG-NEXT: .LBB61_1: ; %bb5
+; GFX1012-SDAG-NEXT: .LBB67_1: ; %bb5
; GFX1012-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1012-SDAG-NEXT: s_add_i32 s4, s4, 1
; GFX1012-SDAG-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX1012-SDAG-NEXT: s_cbranch_scc0 .LBB61_1
+; GFX1012-SDAG-NEXT: s_cbranch_scc0 .LBB67_1
; GFX1012-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1012-SDAG-NEXT: s_mov_b32 s5, 0
; GFX1012-SDAG-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
@@ -8097,12 +8733,12 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX1100-SDAG: ; %bb.0: ; %bb
; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-SDAG-NEXT: s_mov_b32 s0, -1
-; GFX1100-SDAG-NEXT: .LBB61_1: ; %bb5
+; GFX1100-SDAG-NEXT: .LBB67_1: ; %bb5
; GFX1100-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1100-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1100-SDAG-NEXT: s_add_i32 s0, s0, 1
; GFX1100-SDAG-NEXT: s_cmpk_eq_i32 s0, 0xff
-; GFX1100-SDAG-NEXT: s_cbranch_scc0 .LBB61_1
+; GFX1100-SDAG-NEXT: s_cbranch_scc0 .LBB67_1
; GFX1100-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1100-SDAG-NEXT: s_mov_b32 s1, 0
; GFX1100-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -8119,12 +8755,12 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: s_mov_b32 s0, -1
-; GFX1250-SDAG-NEXT: .LBB61_1: ; %bb5
+; GFX1250-SDAG-NEXT: .LBB67_1: ; %bb5
; GFX1250-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-NEXT: s_add_co_i32 s0, s0, 1
; GFX1250-SDAG-NEXT: s_cmp_eq_u32 s0, 0xff
-; GFX1250-SDAG-NEXT: s_cbranch_scc0 .LBB61_1
+; GFX1250-SDAG-NEXT: s_cbranch_scc0 .LBB67_1
; GFX1250-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1250-SDAG-NEXT: s_mov_b32 s1, 0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -8141,12 +8777,12 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1310-SDAG-NEXT: s_mov_b32 s0, -1
-; GFX1310-SDAG-NEXT: .LBB61_1: ; %bb5
+; GFX1310-SDAG-NEXT: .LBB67_1: ; %bb5
; GFX1310-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1310-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1310-SDAG-NEXT: s_add_co_i32 s0, s0, 1
; GFX1310-SDAG-NEXT: s_cmp_eq_u32 s0, 0xff
-; GFX1310-SDAG-NEXT: s_cbranch_scc0 .LBB61_1
+; GFX1310-SDAG-NEXT: s_cbranch_scc0 .LBB67_1
; GFX1310-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1310-SDAG-NEXT: s_mov_b32 s1, 0
; GFX1310-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -8162,11 +8798,11 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX906-ISEL: ; %bb.0: ; %bb
; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX906-ISEL-NEXT: s_mov_b32 s4, -1
-; GFX906-ISEL-NEXT: .LBB61_1: ; %bb5
+; GFX906-ISEL-NEXT: .LBB67_1: ; %bb5
; GFX906-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX906-ISEL-NEXT: s_add_i32 s4, s4, 1
; GFX906-ISEL-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX906-ISEL-NEXT: s_cbranch_scc0 .LBB61_1
+; GFX906-ISEL-NEXT: s_cbranch_scc0 .LBB67_1
; GFX906-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX906-ISEL-NEXT: s_mov_b32 s5, 0
; GFX906-ISEL-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
@@ -8182,11 +8818,11 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX942-ISEL: ; %bb.0: ; %bb
; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-ISEL-NEXT: s_mov_b32 s0, -1
-; GFX942-ISEL-NEXT: .LBB61_1: ; %bb5
+; GFX942-ISEL-NEXT: .LBB67_1: ; %bb5
; GFX942-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-ISEL-NEXT: s_add_i32 s0, s0, 1
; GFX942-ISEL-NEXT: s_cmpk_eq_i32 s0, 0xff
-; GFX942-ISEL-NEXT: s_cbranch_scc0 .LBB61_1
+; GFX942-ISEL-NEXT: s_cbranch_scc0 .LBB67_1
; GFX942-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX942-ISEL-NEXT: s_mov_b32 s1, 0
; GFX942-ISEL-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
@@ -8202,11 +8838,11 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX1012-ISEL: ; %bb.0: ; %bb
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1012-ISEL-NEXT: s_mov_b32 s4, -1
-; GFX1012-ISEL-NEXT: .LBB61_1: ; %bb5
+; GFX1012-ISEL-NEXT: .LBB67_1: ; %bb5
; GFX1012-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1012-ISEL-NEXT: s_add_i32 s4, s4, 1
; GFX1012-ISEL-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX1012-ISEL-NEXT: s_cbranch_scc0 .LBB61_1
+; GFX1012-ISEL-NEXT: s_cbranch_scc0 .LBB67_1
; GFX1012-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1012-ISEL-NEXT: s_mov_b32 s5, 0
; GFX1012-ISEL-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
@@ -8222,12 +8858,12 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX1100-ISEL: ; %bb.0: ; %bb
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-ISEL-NEXT: s_mov_b32 s0, -1
-; GFX1100-ISEL-NEXT: .LBB61_1: ; %bb5
+; GFX1100-ISEL-NEXT: .LBB67_1: ; %bb5
; GFX1100-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1100-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1100-ISEL-NEXT: s_add_i32 s0, s0, 1
; GFX1100-ISEL-NEXT: s_cmpk_eq_i32 s0, 0xff
-; GFX1100-ISEL-NEXT: s_cbranch_scc0 .LBB61_1
+; GFX1100-ISEL-NEXT: s_cbranch_scc0 .LBB67_1
; GFX1100-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1100-ISEL-NEXT: s_mov_b32 s1, 0
; GFX1100-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -8245,12 +8881,12 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-ISEL-NEXT: s_mov_b32 s0, -1
-; GFX1250-ISEL-NEXT: .LBB61_1: ; %bb5
+; GFX1250-ISEL-NEXT: .LBB67_1: ; %bb5
; GFX1250-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-ISEL-NEXT: s_add_co_i32 s0, s0, 1
; GFX1250-ISEL-NEXT: s_cmp_eq_u32 s0, 0xff
-; GFX1250-ISEL-NEXT: s_cbranch_scc0 .LBB61_1
+; GFX1250-ISEL-NEXT: s_cbranch_scc0 .LBB67_1
; GFX1250-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1250-ISEL-NEXT: s_mov_b32 s1, 0
; GFX1250-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -8271,12 +8907,12 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
; GFX1310-ISEL-NEXT: s_mov_b32 s0, -1
-; GFX1310-ISEL-NEXT: .LBB61_1: ; %bb5
+; GFX1310-ISEL-NEXT: .LBB67_1: ; %bb5
; GFX1310-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1310-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1310-ISEL-NEXT: s_add_co_i32 s0, s0, 1
; GFX1310-ISEL-NEXT: s_cmp_eq_u32 s0, 0xff
-; GFX1310-ISEL-NEXT: s_cbranch_scc0 .LBB61_1
+; GFX1310-ISEL-NEXT: s_cbranch_scc0 .LBB67_1
; GFX1310-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1310-ISEL-NEXT: s_mov_b32 s1, 0
; GFX1310-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -16449,11 +17085,11 @@ define <4 x float> @global_saddr_64bit_lsr_iv(ptr addrspace(1) inreg %arg) {
; GFX906-SDAG: ; %bb.0: ; %bb
; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX906-SDAG-NEXT: s_mov_b32 s4, -1
-; GFX906-SDAG-NEXT: .LBB114_1: ; %bb3
+; GFX906-SDAG-NEXT: .LBB120_1: ; %bb3
; GFX906-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX906-SDAG-NEXT: s_add_i32 s4, s4, 1
; GFX906-SDAG-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX906-SDAG-NEXT: s_cbranch_scc0 .LBB114_1
+; GFX906-SDAG-NEXT: s_cbranch_scc0 .LBB120_1
; GFX906-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX906-SDAG-NEXT: s_mov_b32 s5, 0
; GFX906-SDAG-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
@@ -16468,11 +17104,11 @@ define <4 x float> @global_saddr_64bit_lsr_iv(ptr addrspace(1) inreg %arg) {
; GFX942-SDAG: ; %bb.0: ; %bb
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_mov_b32 s2, -1
-; GFX942-SDAG-NEXT: .LBB114_1: ; %bb3
+; GFX942-SDAG-NEXT: .LBB120_1: ; %bb3
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_i32 s2, s2, 1
; GFX942-SDAG-NEXT: s_cmpk_eq_i32 s2, 0xff
-; GFX942-SDAG-NEXT: s_cbranch_scc0 .LBB114_1
+; GFX942-SDAG-NEXT: s_cbranch_scc0 .LBB120_1
; GFX942-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX942-SDAG-NEXT: s_mov_b32 s3, 0
; GFX942-SDAG-NEXT: s_lshl_b64 s[2:3], s[2:3], 2
@@ -16487,11 +17123,11 @@ define <4 x float> @global_saddr_64bit_lsr_iv(ptr addrspace(1) inreg %arg) {
; GFX1012-SDAG: ; %bb.0: ; %bb
; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1012-SDAG-NEXT: s_mov_b32 s4, -1
-; GFX1012-SDAG-NEXT: .LBB114_1: ; %bb3
+; GFX1012-SDAG-NEXT: .LBB120_1: ; %bb3
; GFX1012-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1012-SDAG-NEXT: s_add_i32 s4, s4, 1
; GFX1012-SDAG-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX1012-SDAG-NEXT: s_cbranch_scc0 .LBB114_1
+; GFX1012-SDAG-NEXT: s_cbranch_scc0 .LBB120_1
; GFX1012-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1012-SDAG-NEXT: s_mov_b32 s5, 0
; GFX1012-SDAG-NEXT: v_mov_b32_e32 v0, 0
@@ -16506,12 +17142,12 @@ define <4 x float> @global_saddr_64bit_lsr_iv(ptr addrspace(1) inreg %arg) {
; GFX1100-SDAG: ; %bb.0: ; %bb
; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-SDAG-NEXT: s_mov_b32 s2, -1
-; GFX1100-SDAG-NEXT: .LBB114_1: ; %bb3
+; GFX1100-SDAG-NEXT: .LBB120_1: ; %bb3
; GFX1100-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1100-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1100-SDAG-NEXT: s_add_i32 s2, s2, 1
; GFX1100-SDAG-NEXT: s_cmpk_eq_i32 s2, 0xff
-; GFX1100-SDAG-NEXT: s_cbranch_scc0 .LBB114_1
+; GFX1100-SDAG-NEXT: s_cbranch_scc0 .LBB120_1
; GFX1100-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1100-SDAG-NEXT: s_mov_b32 s3, 0
; GFX1100-SDAG-NEXT: v_mov_b32_e32 v0, 0
@@ -16528,12 +17164,12 @@ define <4 x float> @global_saddr_64bit_lsr_iv(ptr addrspace(1) inreg %arg) {
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: s_mov_b32 s2, -1
-; GFX1250-SDAG-NEXT: .LBB114_1: ; %bb3
+; GFX1250-SDAG-NEXT: .LBB120_1: ; %bb3
; GFX1250-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-NEXT: s_add_co_i32 s2, s2, 1
; GFX1250-SDAG-NEXT: s_cmp_eq_u32 s2, 0xff
-; GFX1250-SDAG-NEXT: s_cbranch_scc0 .LBB114_1
+; GFX1250-SDAG-NEXT: s_cbranch_scc0 .LBB120_1
; GFX1250-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1250-SDAG-NEXT: s_mov_b32 s3, 0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
@@ -16552,12 +17188,12 @@ define <4 x float> @global_saddr_64bit_lsr_iv(ptr addrspace(1) inreg %arg) {
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1310-SDAG-NEXT: s_mov_b32 s2, -1
-; GFX1310-SDAG-NEXT: .LBB114_1: ; %bb3
+; GFX1310-SDAG-NEXT: .LBB120_1: ; %bb3
; GFX1310-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1310-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1310-SDAG-NEXT: s_add_co_i32 s2, s2, 1
; GFX1310-SDAG-NEXT: s_cmp_eq_u32 s2, 0xff
-; GFX1310-SDAG-NEXT: s_cbranch_scc0 .LBB114_1
+; GFX1310-SDAG-NEXT: s_cbranch_scc0 .LBB120_1
; GFX1310-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1310-SDAG-NEXT: s_mov_b32 s3, 0
; GFX1310-SDAG-NEXT: v_mov_b32_e32 v0, 0
@@ -16572,11 +17208,11 @@ define <4 x float> @global_saddr_64bit_lsr_iv(ptr addrspace(1) inreg %arg) {
; GFX906-ISEL: ; %bb.0: ; %bb
; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX906-ISEL-NEXT: s_mov_b32 s4, -1
-; GFX906-ISEL-NEXT: .LBB114_1: ; %bb3
+; GFX906-ISEL-NEXT: .LBB120_1: ; %bb3
; GFX906-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX906-ISEL-NEXT: s_add_i32 s4, s4, 1
; GFX906-ISEL-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX906-ISEL-NEXT: s_cbranch_scc0 .LBB114_1
+; GFX906-ISEL-NEXT: s_cbranch_scc0 .LBB120_1
; GFX906-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX906-ISEL-NEXT: s_mov_b32 s5, 0
; GFX906-ISEL-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
@@ -16599,11 +17235,11 @@ define <4 x float> @global_saddr_64bit_lsr_iv(ptr addrspace(1) inreg %arg) {
; GFX942-ISEL: ; %bb.0: ; %bb
; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-ISEL-NEXT: s_mov_b32 s2, -1
-; GFX942-ISEL-NEXT: .LBB114_1: ; %bb3
+; GFX942-ISEL-NEXT: .LBB120_1: ; %bb3
; GFX942-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-ISEL-NEXT: s_add_i32 s2, s2, 1
; GFX942-ISEL-NEXT: s_cmpk_eq_i32 s2, 0xff
-; GFX942-ISEL-NEXT: s_cbranch_scc0 .LBB114_1
+; GFX942-ISEL-NEXT: s_cbranch_scc0 .LBB120_1
; GFX942-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX942-ISEL-NEXT: s_mov_b32 s3, 0
; GFX942-ISEL-NEXT: s_lshl_b64 s[2:3], s[2:3], 2
@@ -16626,11 +17262,11 @@ define <4 x float> @global_saddr_64bit_lsr_iv(ptr addrspace(1) inreg %arg) {
; GFX1012-ISEL: ; %bb.0: ; %bb
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1012-ISEL-NEXT: s_mov_b32 s4, -1
-; GFX1012-ISEL-NEXT: .LBB114_1: ; %bb3
+; GFX1012-ISEL-NEXT: .LBB120_1: ; %bb3
; GFX1012-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1012-ISEL-NEXT: s_add_i32 s4, s4, 1
; GFX1012-ISEL-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX1012-ISEL-NEXT: s_cbranch_scc0 .LBB114_1
+; GFX1012-ISEL-NEXT: s_cbranch_scc0 .LBB120_1
; GFX1012-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1012-ISEL-NEXT: s_mov_b32 s5, 0
; GFX1012-ISEL-NEXT: v_mov_b32_e32 v0, 0
@@ -16653,12 +17289,12 @@ define <4 x float> @global_saddr_64bit_lsr_iv(ptr addrspace(1) inreg %arg) {
; GFX1100-ISEL: ; %bb.0: ; %bb
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-ISEL-NEXT: s_mov_b32 s2, -1
-; GFX1100-ISEL-NEXT: .LBB114_1: ; %bb3
+; GFX1100-ISEL-NEXT: .LBB120_1: ; %bb3
; GFX1100-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1100-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1100-ISEL-NEXT: s_add_i32 s2, s2, 1
; GFX1100-ISEL-NEXT: s_cmpk_eq_i32 s2, 0xff
-; GFX1100-ISEL-NEXT: s_cbranch_scc0 .LBB114_1
+; GFX1100-ISEL-NEXT: s_cbranch_scc0 .LBB120_1
; GFX1100-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1100-ISEL-NEXT: s_mov_b32 s3, 0
; GFX1100-ISEL-NEXT: v_mov_b32_e32 v0, 0
@@ -16682,12 +17318,12 @@ define <4 x float> @global_saddr_64bit_lsr_iv(ptr addrspace(1) inreg %arg) {
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-ISEL-NEXT: s_mov_b32 s2, -1
-; GFX1250-ISEL-NEXT: .LBB114_1: ; %bb3
+; GFX1250-ISEL-NEXT: .LBB120_1: ; %bb3
; GFX1250-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-ISEL-NEXT: s_add_co_i32 s2, s2, 1
; GFX1250-ISEL-NEXT: s_cmp_eq_u32 s2, 0xff
-; GFX1250-ISEL-NEXT: s_cbranch_scc0 .LBB114_1
+; GFX1250-ISEL-NEXT: s_cbranch_scc0 .LBB120_1
; GFX1250-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1250-ISEL-NEXT: s_mov_b32 s3, 0
; GFX1250-ISEL-NEXT: v_mov_b32_e32 v0, 0
@@ -16715,12 +17351,12 @@ define <4 x float> @global_saddr_64bit_lsr_iv(ptr addrspace(1) inreg %arg) {
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
; GFX1310-ISEL-NEXT: s_mov_b32 s2, -1
-; GFX1310-ISEL-NEXT: .LBB114_1: ; %bb3
+; GFX1310-ISEL-NEXT: .LBB120_1: ; %bb3
; GFX1310-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1310-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1310-ISEL-NEXT: s_add_co_i32 s2, s2, 1
; GFX1310-ISEL-NEXT: s_cmp_eq_u32 s2, 0xff
-; GFX1310-ISEL-NEXT: s_cbranch_scc0 .LBB114_1
+; GFX1310-ISEL-NEXT: s_cbranch_scc0 .LBB120_1
; GFX1310-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1310-ISEL-NEXT: s_mov_b32 s3, 0
; GFX1310-ISEL-NEXT: v_mov_b32_e32 v0, 0
@@ -16763,11 +17399,11 @@ define <4 x float> @global_saddr_64bit_lsr_iv_multiload(ptr addrspace(1) inreg %
; GFX906-SDAG: ; %bb.0: ; %bb
; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX906-SDAG-NEXT: s_mov_b32 s4, -1
-; GFX906-SDAG-NEXT: .LBB115_1: ; %bb5
+; GFX906-SDAG-NEXT: .LBB121_1: ; %bb5
; GFX906-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX906-SDAG-NEXT: s_add_i32 s4, s4, 1
; GFX906-SDAG-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX906-SDAG-NEXT: s_cbranch_scc0 .LBB115_1
+; GFX906-SDAG-NEXT: s_cbranch_scc0 .LBB121_1
; GFX906-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX906-SDAG-NEXT: s_mov_b32 s5, 0
; GFX906-SDAG-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
@@ -16782,11 +17418,11 @@ define <4 x float> @global_saddr_64bit_lsr_iv_multiload(ptr addrspace(1) inreg %
; GFX942-SDAG: ; %bb.0: ; %bb
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_mov_b32 s2, -1
-; GFX942-SDAG-NEXT: .LBB115_1: ; %bb5
+; GFX942-SDAG-NEXT: .LBB121_1: ; %bb5
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_i32 s2, s2, 1
; GFX942-SDAG-NEXT: s_cmpk_eq_i32 s2, 0xff
-; GFX942-SDAG-NEXT: s_cbranch_scc0 .LBB115_1
+; GFX942-SDAG-NEXT: s_cbranch_scc0 .LBB121_1
; GFX942-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX942-SDAG-NEXT: s_mov_b32 s3, 0
; GFX942-SDAG-NEXT: s_lshl_b64 s[2:3], s[2:3], 2
@@ -16801,11 +17437,11 @@ define <4 x float> @global_saddr_64bit_lsr_iv_multiload(ptr addrspace(1) inreg %
; GFX1012-SDAG: ; %bb.0: ; %bb
; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1012-SDAG-NEXT: s_mov_b32 s4, -1
-; GFX1012-SDAG-NEXT: .LBB115_1: ; %bb5
+; GFX1012-SDAG-NEXT: .LBB121_1: ; %bb5
; GFX1012-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1012-SDAG-NEXT: s_add_i32 s4, s4, 1
; GFX1012-SDAG-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX1012-SDAG-NEXT: s_cbranch_scc0 .LBB115_1
+; GFX1012-SDAG-NEXT: s_cbranch_scc0 .LBB121_1
; GFX1012-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1012-SDAG-NEXT: s_mov_b32 s5, 0
; GFX1012-SDAG-NEXT: v_mov_b32_e32 v0, 0
@@ -16820,12 +17456,12 @@ define <4 x float> @global_saddr_64bit_lsr_iv_multiload(ptr addrspace(1) inreg %
; GFX1100-SDAG: ; %bb.0: ; %bb
; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-SDAG-NEXT: s_mov_b32 s2, -1
-; GFX1100-SDAG-NEXT: .LBB115_1: ; %bb5
+; GFX1100-SDAG-NEXT: .LBB121_1: ; %bb5
; GFX1100-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1100-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1100-SDAG-NEXT: s_add_i32 s2, s2, 1
; GFX1100-SDAG-NEXT: s_cmpk_eq_i32 s2, 0xff
-; GFX1100-SDAG-NEXT: s_cbranch_scc0 .LBB115_1
+; GFX1100-SDAG-NEXT: s_cbranch_scc0 .LBB121_1
; GFX1100-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1100-SDAG-NEXT: s_mov_b32 s3, 0
; GFX1100-SDAG-NEXT: v_mov_b32_e32 v0, 0
@@ -16842,12 +17478,12 @@ define <4 x float> @global_saddr_64bit_lsr_iv_multiload(ptr addrspace(1) inreg %
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: s_mov_b32 s2, -1
-; GFX1250-SDAG-NEXT: .LBB115_1: ; %bb5
+; GFX1250-SDAG-NEXT: .LBB121_1: ; %bb5
; GFX1250-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-NEXT: s_add_co_i32 s2, s2, 1
; GFX1250-SDAG-NEXT: s_cmp_eq_u32 s2, 0xff
-; GFX1250-SDAG-NEXT: s_cbranch_scc0 .LBB115_1
+; GFX1250-SDAG-NEXT: s_cbranch_scc0 .LBB121_1
; GFX1250-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1250-SDAG-NEXT: s_mov_b32 s3, 0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
@@ -16866,12 +17502,12 @@ define <4 x float> @global_saddr_64bit_lsr_iv_multiload(ptr addrspace(1) inreg %
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1310-SDAG-NEXT: s_mov_b32 s2, -1
-; GFX1310-SDAG-NEXT: .LBB115_1: ; %bb5
+; GFX1310-SDAG-NEXT: .LBB121_1: ; %bb5
; GFX1310-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1310-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1310-SDAG-NEXT: s_add_co_i32 s2, s2, 1
; GFX1310-SDAG-NEXT: s_cmp_eq_u32 s2, 0xff
-; GFX1310-SDAG-NEXT: s_cbranch_scc0 .LBB115_1
+; GFX1310-SDAG-NEXT: s_cbranch_scc0 .LBB121_1
; GFX1310-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1310-SDAG-NEXT: s_mov_b32 s3, 0
; GFX1310-SDAG-NEXT: v_mov_b32_e32 v0, 0
@@ -16886,11 +17522,11 @@ define <4 x float> @global_saddr_64bit_lsr_iv_multiload(ptr addrspace(1) inreg %
; GFX906-ISEL: ; %bb.0: ; %bb
; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX906-ISEL-NEXT: s_mov_b32 s4, -1
-; GFX906-ISEL-NEXT: .LBB115_1: ; %bb5
+; GFX906-ISEL-NEXT: .LBB121_1: ; %bb5
; GFX906-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX906-ISEL-NEXT: s_add_i32 s4, s4, 1
; GFX906-ISEL-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX906-ISEL-NEXT: s_cbranch_scc0 .LBB115_1
+; GFX906-ISEL-NEXT: s_cbranch_scc0 .LBB121_1
; GFX906-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX906-ISEL-NEXT: s_mov_b32 s5, 0
; GFX906-ISEL-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
@@ -16913,11 +17549,11 @@ define <4 x float> @global_saddr_64bit_lsr_iv_multiload(ptr addrspace(1) inreg %
; GFX942-ISEL: ; %bb.0: ; %bb
; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-ISEL-NEXT: s_mov_b32 s2, -1
-; GFX942-ISEL-NEXT: .LBB115_1: ; %bb5
+; GFX942-ISEL-NEXT: .LBB121_1: ; %bb5
; GFX942-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-ISEL-NEXT: s_add_i32 s2, s2, 1
; GFX942-ISEL-NEXT: s_cmpk_eq_i32 s2, 0xff
-; GFX942-ISEL-NEXT: s_cbranch_scc0 .LBB115_1
+; GFX942-ISEL-NEXT: s_cbranch_scc0 .LBB121_1
; GFX942-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX942-ISEL-NEXT: s_mov_b32 s3, 0
; GFX942-ISEL-NEXT: s_lshl_b64 s[2:3], s[2:3], 2
@@ -16940,11 +17576,11 @@ define <4 x float> @global_saddr_64bit_lsr_iv_multiload(ptr addrspace(1) inreg %
; GFX1012-ISEL: ; %bb.0: ; %bb
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1012-ISEL-NEXT: s_mov_b32 s4, -1
-; GFX1012-ISEL-NEXT: .LBB115_1: ; %bb5
+; GFX1012-ISEL-NEXT: .LBB121_1: ; %bb5
; GFX1012-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1012-ISEL-NEXT: s_add_i32 s4, s4, 1
; GFX1012-ISEL-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX1012-ISEL-NEXT: s_cbranch_scc0 .LBB115_1
+; GFX1012-ISEL-NEXT: s_cbranch_scc0 .LBB121_1
; GFX1012-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1012-ISEL-NEXT: s_mov_b32 s5, 0
; GFX1012-ISEL-NEXT: v_mov_b32_e32 v0, 0
@@ -16967,12 +17603,12 @@ define <4 x float> @global_saddr_64bit_lsr_iv_multiload(ptr addrspace(1) inreg %
; GFX1100-ISEL: ; %bb.0: ; %bb
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-ISEL-NEXT: s_mov_b32 s2, -1
-; GFX1100-ISEL-NEXT: .LBB115_1: ; %bb5
+; GFX1100-ISEL-NEXT: .LBB121_1: ; %bb5
; GFX1100-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1100-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1100-ISEL-NEXT: s_add_i32 s2, s2, 1
; GFX1100-ISEL-NEXT: s_cmpk_eq_i32 s2, 0xff
-; GFX1100-ISEL-NEXT: s_cbranch_scc0 .LBB115_1
+; GFX1100-ISEL-NEXT: s_cbranch_scc0 .LBB121_1
; GFX1100-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1100-ISEL-NEXT: s_mov_b32 s3, 0
; GFX1100-ISEL-NEXT: v_mov_b32_e32 v0, 0
@@ -16996,12 +17632,12 @@ define <4 x float> @global_saddr_64bit_lsr_iv_multiload(ptr addrspace(1) inreg %
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-ISEL-NEXT: s_mov_b32 s2, -1
-; GFX1250-ISEL-NEXT: .LBB115_1: ; %bb5
+; GFX1250-ISEL-NEXT: .LBB121_1: ; %bb5
; GFX1250-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-ISEL-NEXT: s_add_co_i32 s2, s2, 1
; GFX1250-ISEL-NEXT: s_cmp_eq_u32 s2, 0xff
-; GFX1250-ISEL-NEXT: s_cbranch_scc0 .LBB115_1
+; GFX1250-ISEL-NEXT: s_cbranch_scc0 .LBB121_1
; GFX1250-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1250-ISEL-NEXT: s_mov_b32 s3, 0
; GFX1250-ISEL-NEXT: v_mov_b32_e32 v0, 0
@@ -17029,12 +17665,12 @@ define <4 x float> @global_saddr_64bit_lsr_iv_multiload(ptr addrspace(1) inreg %
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
; GFX1310-ISEL-NEXT: s_mov_b32 s2, -1
-; GFX1310-ISEL-NEXT: .LBB115_1: ; %bb5
+; GFX1310-ISEL-NEXT: .LBB121_1: ; %bb5
; GFX1310-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1310-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1310-ISEL-NEXT: s_add_co_i32 s2, s2, 1
; GFX1310-ISEL-NEXT: s_cmp_eq_u32 s2, 0xff
-; GFX1310-ISEL-NEXT: s_cbranch_scc0 .LBB115_1
+; GFX1310-ISEL-NEXT: s_cbranch_scc0 .LBB121_1
; GFX1310-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1310-ISEL-NEXT: s_mov_b32 s3, 0
; GFX1310-ISEL-NEXT: v_mov_b32_e32 v0, 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll
index f0909a0cfb30b3..6afa6b8a5c481a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll
@@ -386,6 +386,594 @@ entry:
ret void
}
+;; vgpr pointer, system scope
+define void @av_global_store_b16_0_11(ptr addrspace(1) %addr, i16 %data) {
+; GFX906-SDAG-LABEL: av_global_store_b16_0_11:
+; GFX906-SDAG: ; %bb.0: ; %entry
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-SDAG-NEXT: global_store_short v[0:1], v2, off
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX906-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-SDAG-LABEL: av_global_store_b16_0_11:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: global_store_short v[0:1], v2, off sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: av_global_store_b16_0_11:
+; GFX1012-SDAG: ; %bb.0: ; %entry
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: global_store_short v[0:1], v2, off
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: av_global_store_b16_0_11:
+; GFX1100-SDAG: ; %bb.0: ; %entry
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: av_global_store_b16_0_11:
+; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_store_b16 v[0:1], v2, off scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-SDAG-LABEL: av_global_store_b16_0_11:
+; GFX1310-SDAG: ; %bb.0: ; %entry
+; GFX1310-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1310-SDAG-NEXT: global_store_b16 v[0:1], v2, off scope:SCOPE_SYS
+; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX906-ISEL-LABEL: av_global_store_b16_0_11:
+; GFX906-ISEL: ; %bb.0: ; %entry
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-ISEL-NEXT: global_store_short v[0:1], v2, off
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX906-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: av_global_store_b16_0_11:
+; GFX942-ISEL: ; %bb.0: ; %entry
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: global_store_short v[0:1], v2, off sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: av_global_store_b16_0_11:
+; GFX1012-ISEL: ; %bb.0: ; %entry
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: global_store_short v[0:1], v2, off
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: av_global_store_b16_0_11:
+; GFX1100-ISEL: ; %bb.0: ; %entry
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: av_global_store_b16_0_11:
+; GFX1250-ISEL: ; %bb.0: ; %entry
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: global_store_b16 v[0:1], v2, off scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-ISEL-LABEL: av_global_store_b16_0_11:
+; GFX1310-ISEL: ; %bb.0: ; %entry
+; GFX1310-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_expcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1310-ISEL-NEXT: global_store_b16 v[0:1], v2, off scope:SCOPE_SYS
+; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ call void @llvm.amdgcn.av.store.b16.p1(ptr addrspace(1) %addr, i16 %data, metadata !3)
+ ret void
+}
+
+;; sgpr pointer, system scope
+define void @av_global_store_b16_saddr_0_11(ptr addrspace(1) inreg %addr, i16 %data) {
+; GFX906-SDAG-LABEL: av_global_store_b16_saddr_0_11:
+; GFX906-SDAG: ; %bb.0: ; %entry
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX906-SDAG-NEXT: global_store_short v1, v0, s[16:17]
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX906-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-SDAG-LABEL: av_global_store_b16_saddr_0_11:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX942-SDAG-NEXT: global_store_short v1, v0, s[0:1] sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: av_global_store_b16_saddr_0_11:
+; GFX1012-SDAG: ; %bb.0: ; %entry
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1012-SDAG-NEXT: global_store_short v1, v0, s[16:17]
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: av_global_store_b16_saddr_0_11:
+; GFX1100-SDAG: ; %bb.0: ; %entry
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1100-SDAG-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: av_global_store_b16_saddr_0_11:
+; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: global_store_b16 v1, v0, s[0:1] scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-SDAG-LABEL: av_global_store_b16_saddr_0_11:
+; GFX1310-SDAG: ; %bb.0: ; %entry
+; GFX1310-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1310-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1310-SDAG-NEXT: global_store_b16 v1, v0, s[0:1] scope:SCOPE_SYS
+; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX906-ISEL-LABEL: av_global_store_b16_saddr_0_11:
+; GFX906-ISEL: ; %bb.0: ; %entry
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-ISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX906-ISEL-NEXT: global_store_short v1, v0, s[16:17]
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX906-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: av_global_store_b16_saddr_0_11:
+; GFX942-ISEL: ; %bb.0: ; %entry
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX942-ISEL-NEXT: global_store_short v1, v0, s[0:1] sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: av_global_store_b16_saddr_0_11:
+; GFX1012-ISEL: ; %bb.0: ; %entry
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1012-ISEL-NEXT: global_store_short v1, v0, s[16:17]
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: av_global_store_b16_saddr_0_11:
+; GFX1100-ISEL: ; %bb.0: ; %entry
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1100-ISEL-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: av_global_store_b16_saddr_0_11:
+; GFX1250-ISEL: ; %bb.0: ; %entry
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-ISEL-NEXT: global_store_b16 v1, v0, s[0:1] scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-ISEL-LABEL: av_global_store_b16_saddr_0_11:
+; GFX1310-ISEL: ; %bb.0: ; %entry
+; GFX1310-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_expcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1310-ISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1310-ISEL-NEXT: global_store_b16 v1, v0, s[0:1] scope:SCOPE_SYS
+; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ call void @llvm.amdgcn.av.store.b16.p1(ptr addrspace(1) %addr, i16 %data, metadata !3)
+ ret void
+}
+
+;; vgpr pointer, system scope
+define void @av_global_store_b32_0_11(ptr addrspace(1) %addr, i32 %data) {
+; GFX906-SDAG-LABEL: av_global_store_b32_0_11:
+; GFX906-SDAG: ; %bb.0: ; %entry
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-SDAG-NEXT: global_store_dword v[0:1], v2, off
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX906-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-SDAG-LABEL: av_global_store_b32_0_11:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: global_store_dword v[0:1], v2, off sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: av_global_store_b32_0_11:
+; GFX1012-SDAG: ; %bb.0: ; %entry
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: global_store_dword v[0:1], v2, off
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: av_global_store_b32_0_11:
+; GFX1100-SDAG: ; %bb.0: ; %entry
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: global_store_b32 v[0:1], v2, off
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: av_global_store_b32_0_11:
+; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_store_b32 v[0:1], v2, off scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-SDAG-LABEL: av_global_store_b32_0_11:
+; GFX1310-SDAG: ; %bb.0: ; %entry
+; GFX1310-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1310-SDAG-NEXT: global_store_b32 v[0:1], v2, off scope:SCOPE_SYS
+; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX906-ISEL-LABEL: av_global_store_b32_0_11:
+; GFX906-ISEL: ; %bb.0: ; %entry
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-ISEL-NEXT: global_store_dword v[0:1], v2, off
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX906-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: av_global_store_b32_0_11:
+; GFX942-ISEL: ; %bb.0: ; %entry
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: global_store_dword v[0:1], v2, off sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: av_global_store_b32_0_11:
+; GFX1012-ISEL: ; %bb.0: ; %entry
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: global_store_dword v[0:1], v2, off
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: av_global_store_b32_0_11:
+; GFX1100-ISEL: ; %bb.0: ; %entry
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: global_store_b32 v[0:1], v2, off
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: av_global_store_b32_0_11:
+; GFX1250-ISEL: ; %bb.0: ; %entry
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: global_store_b32 v[0:1], v2, off scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-ISEL-LABEL: av_global_store_b32_0_11:
+; GFX1310-ISEL: ; %bb.0: ; %entry
+; GFX1310-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_expcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1310-ISEL-NEXT: global_store_b32 v[0:1], v2, off scope:SCOPE_SYS
+; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ call void @llvm.amdgcn.av.store.b32.p1(ptr addrspace(1) %addr, i32 %data, metadata !3)
+ ret void
+}
+
+;; sgpr pointer, system scope
+define void @av_global_store_b32_saddr_0_11(ptr addrspace(1) inreg %addr, i32 %data) {
+; GFX906-SDAG-LABEL: av_global_store_b32_saddr_0_11:
+; GFX906-SDAG: ; %bb.0: ; %entry
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX906-SDAG-NEXT: global_store_dword v1, v0, s[16:17]
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX906-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-SDAG-LABEL: av_global_store_b32_saddr_0_11:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX942-SDAG-NEXT: global_store_dword v1, v0, s[0:1] sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: av_global_store_b32_saddr_0_11:
+; GFX1012-SDAG: ; %bb.0: ; %entry
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1012-SDAG-NEXT: global_store_dword v1, v0, s[16:17]
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: av_global_store_b32_saddr_0_11:
+; GFX1100-SDAG: ; %bb.0: ; %entry
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1100-SDAG-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: av_global_store_b32_saddr_0_11:
+; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: global_store_b32 v1, v0, s[0:1] scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-SDAG-LABEL: av_global_store_b32_saddr_0_11:
+; GFX1310-SDAG: ; %bb.0: ; %entry
+; GFX1310-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1310-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1310-SDAG-NEXT: global_store_b32 v1, v0, s[0:1] scope:SCOPE_SYS
+; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX906-ISEL-LABEL: av_global_store_b32_saddr_0_11:
+; GFX906-ISEL: ; %bb.0: ; %entry
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-ISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX906-ISEL-NEXT: global_store_dword v1, v0, s[16:17]
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX906-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: av_global_store_b32_saddr_0_11:
+; GFX942-ISEL: ; %bb.0: ; %entry
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX942-ISEL-NEXT: global_store_dword v1, v0, s[0:1] sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: av_global_store_b32_saddr_0_11:
+; GFX1012-ISEL: ; %bb.0: ; %entry
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1012-ISEL-NEXT: global_store_dword v1, v0, s[16:17]
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: av_global_store_b32_saddr_0_11:
+; GFX1100-ISEL: ; %bb.0: ; %entry
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1100-ISEL-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: av_global_store_b32_saddr_0_11:
+; GFX1250-ISEL: ; %bb.0: ; %entry
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-ISEL-NEXT: global_store_b32 v1, v0, s[0:1] scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-ISEL-LABEL: av_global_store_b32_saddr_0_11:
+; GFX1310-ISEL: ; %bb.0: ; %entry
+; GFX1310-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_expcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1310-ISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1310-ISEL-NEXT: global_store_b32 v1, v0, s[0:1] scope:SCOPE_SYS
+; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ call void @llvm.amdgcn.av.store.b32.p1(ptr addrspace(1) %addr, i32 %data, metadata !3)
+ ret void
+}
+
+;; vgpr pointer, system scope
+define void @av_global_store_b64_0_11(ptr addrspace(1) %addr, i64 %data) {
+; GFX906-SDAG-LABEL: av_global_store_b64_0_11:
+; GFX906-SDAG: ; %bb.0: ; %entry
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-SDAG-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX906-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-SDAG-LABEL: av_global_store_b64_0_11:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: av_global_store_b64_0_11:
+; GFX1012-SDAG: ; %bb.0: ; %entry
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: av_global_store_b64_0_11:
+; GFX1100-SDAG: ; %bb.0: ; %entry
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: av_global_store_b64_0_11:
+; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-SDAG-LABEL: av_global_store_b64_0_11:
+; GFX1310-SDAG: ; %bb.0: ; %entry
+; GFX1310-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1310-SDAG-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX906-ISEL-LABEL: av_global_store_b64_0_11:
+; GFX906-ISEL: ; %bb.0: ; %entry
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-ISEL-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX906-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: av_global_store_b64_0_11:
+; GFX942-ISEL: ; %bb.0: ; %entry
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: global_store_dwordx2 v[0:1], v[2:3], off sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: av_global_store_b64_0_11:
+; GFX1012-ISEL: ; %bb.0: ; %entry
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: av_global_store_b64_0_11:
+; GFX1100-ISEL: ; %bb.0: ; %entry
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: av_global_store_b64_0_11:
+; GFX1250-ISEL: ; %bb.0: ; %entry
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: global_store_b64 v[0:1], v[2:3], off scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-ISEL-LABEL: av_global_store_b64_0_11:
+; GFX1310-ISEL: ; %bb.0: ; %entry
+; GFX1310-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_expcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1310-ISEL-NEXT: global_store_b64 v[0:1], v[2:3], off scope:SCOPE_SYS
+; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) %addr, i64 %data, metadata !3)
+ ret void
+}
+
+;; sgpr pointer, system scope
+define void @av_global_store_b64_saddr_0_11(ptr addrspace(1) inreg %addr, i64 %data) {
+; GFX906-SDAG-LABEL: av_global_store_b64_saddr_0_11:
+; GFX906-SDAG: ; %bb.0: ; %entry
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX906-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[16:17]
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX906-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-SDAG-LABEL: av_global_store_b64_saddr_0_11:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX942-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: av_global_store_b64_saddr_0_11:
+; GFX1012-SDAG: ; %bb.0: ; %entry
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX1012-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[16:17]
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: av_global_store_b64_saddr_0_11:
+; GFX1100-SDAG: ; %bb.0: ; %entry
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX1100-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: av_global_store_b64_saddr_0_11:
+; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-SDAG-LABEL: av_global_store_b64_saddr_0_11:
+; GFX1310-SDAG: ; %bb.0: ; %entry
+; GFX1310-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1310-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX1310-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX906-ISEL-LABEL: av_global_store_b64_saddr_0_11:
+; GFX906-ISEL: ; %bb.0: ; %entry
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-ISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX906-ISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[16:17]
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX906-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: av_global_store_b64_saddr_0_11:
+; GFX942-ISEL: ; %bb.0: ; %entry
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX942-ISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: av_global_store_b64_saddr_0_11:
+; GFX1012-ISEL: ; %bb.0: ; %entry
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX1012-ISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[16:17]
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: av_global_store_b64_saddr_0_11:
+; GFX1100-ISEL: ; %bb.0: ; %entry
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX1100-ISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: av_global_store_b64_saddr_0_11:
+; GFX1250-ISEL: ; %bb.0: ; %entry
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-ISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1] scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-ISEL-LABEL: av_global_store_b64_saddr_0_11:
+; GFX1310-ISEL: ; %bb.0: ; %entry
+; GFX1310-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_expcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1310-ISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX1310-ISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1] scope:SCOPE_SYS
+; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) %addr, i64 %data, metadata !3)
+ ret void
+}
+
;;==============================================================================
;; Signed offset addressing modes (derived from global-saddr-store.ll)
;;==============================================================================
diff --git a/llvm/test/CodeGen/AMDGPU/unsupported-av-load.ll b/llvm/test/CodeGen/AMDGPU/unsupported-av-load.ll
index 9a7a06853cfc65..5539b8526fece3 100644
--- a/llvm/test/CodeGen/AMDGPU/unsupported-av-load.ll
+++ b/llvm/test/CodeGen/AMDGPU/unsupported-av-load.ll
@@ -6,11 +6,21 @@
; RUN: not llc -global-isel=1 -mtriple=amdgpu7.05 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR %s
; RUN: not llc -global-isel=1 -mtriple=amdgpu8.10 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR %s
-define <4 x i32> @av_load_b128(ptr addrspace(1) %addr) {
-; ERR: error: {{.*}}: in function @av_load_b128 {{.*}}: llvm.amdgcn.av.load.b128 requires target feature 'flat-global-insts'
+define void @av_load(ptr addrspace(1) %addr, ptr addrspace(1) %out) {
+; ERR: error: {{.*}}: in function @av_load {{.*}}: llvm.amdgcn.av.load.b16 requires target feature 'flat-global-insts'
+; ERR: error: {{.*}}: in function @av_load {{.*}}: llvm.amdgcn.av.load.b32 requires target feature 'flat-global-insts'
+; ERR: error: {{.*}}: in function @av_load {{.*}}: llvm.amdgcn.av.load.b64 requires target feature 'flat-global-insts'
+; ERR: error: {{.*}}: in function @av_load {{.*}}: llvm.amdgcn.av.load.b128 requires target feature 'flat-global-insts'
entry:
- %data = call <4 x i32> @llvm.amdgcn.av.load.b128.p1(ptr addrspace(1) %addr, metadata !0)
- ret <4 x i32> %data
+ %data16 = call i16 @llvm.amdgcn.av.load.b16.p1(ptr addrspace(1) %addr, metadata !0)
+ %data32 = call i32 @llvm.amdgcn.av.load.b32.p1(ptr addrspace(1) %addr, metadata !0)
+ %data64 = call i64 @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) %addr, metadata !0)
+ %data128 = call <4 x i32> @llvm.amdgcn.av.load.b128.p1(ptr addrspace(1) %addr, metadata !0)
+ store volatile i16 %data16, ptr addrspace(1) %out
+ store volatile i32 %data32, ptr addrspace(1) %out
+ store volatile i64 %data64, ptr addrspace(1) %out
+ store volatile <4 x i32> %data128, ptr addrspace(1) %out
+ ret void
}
!0 = !{!""}
diff --git a/llvm/test/CodeGen/AMDGPU/unsupported-av-store.ll b/llvm/test/CodeGen/AMDGPU/unsupported-av-store.ll
index c0810d4cc4546b..03543c53daaab9 100644
--- a/llvm/test/CodeGen/AMDGPU/unsupported-av-store.ll
+++ b/llvm/test/CodeGen/AMDGPU/unsupported-av-store.ll
@@ -6,10 +6,16 @@
; RUN: not llc -global-isel=1 -mtriple=amdgpu7.05 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR %s
; RUN: not llc -global-isel=1 -mtriple=amdgpu8.10 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR %s
-define void @av_store_b128(ptr addrspace(1) %addr, <4 x i32> %data) {
-; ERR: error: {{.*}}: in function @av_store_b128 {{.*}}: llvm.amdgcn.av.store.b128 requires target feature 'flat-global-insts'
+define void @av_store(ptr addrspace(1) %addr, i16 %data16, i32 %data32, i64 %data64, <4 x i32> %data128) {
+; ERR: error: {{.*}}: in function @av_store {{.*}}: llvm.amdgcn.av.store.b16 requires target feature 'flat-global-insts'
+; ERR: error: {{.*}}: in function @av_store {{.*}}: llvm.amdgcn.av.store.b32 requires target feature 'flat-global-insts'
+; ERR: error: {{.*}}: in function @av_store {{.*}}: llvm.amdgcn.av.store.b64 requires target feature 'flat-global-insts'
+; ERR: error: {{.*}}: in function @av_store {{.*}}: llvm.amdgcn.av.store.b128 requires target feature 'flat-global-insts'
entry:
- call void @llvm.amdgcn.av.store.b128.p1(ptr addrspace(1) %addr, <4 x i32> %data, metadata !0)
+ call void @llvm.amdgcn.av.store.b16.p1(ptr addrspace(1) %addr, i16 %data16, metadata !0)
+ call void @llvm.amdgcn.av.store.b32.p1(ptr addrspace(1) %addr, i32 %data32, metadata !0)
+ call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) %addr, i64 %data64, metadata !0)
+ call void @llvm.amdgcn.av.store.b128.p1(ptr addrspace(1) %addr, <4 x i32> %data128, metadata !0)
ret void
}
diff --git a/llvm/test/Verifier/AMDGPU/intrinsics-av.ll b/llvm/test/Verifier/AMDGPU/intrinsics-av.ll
index d46a25b06a631f..67626843c6588b 100644
--- a/llvm/test/Verifier/AMDGPU/intrinsics-av.ll
+++ b/llvm/test/Verifier/AMDGPU/intrinsics-av.ll
@@ -27,3 +27,45 @@ define void @av_flat_store_b128_bad_metadata(ptr %addr, <4 x i32> %data) {
call void @llvm.amdgcn.av.store.b128.p0(ptr %addr, <4 x i32> %data, metadata i32 0)
ret void
}
+
+; CHECK: the last argument to av load/store intrinsics must be a metadata string
+; CHECK-NEXT: call i16 @llvm.amdgcn.av.load.b16.p1({{.*}})
+define i16 @av_global_load_b16_bad_metadata(ptr addrspace(1) %addr) {
+ %data = call i16 @llvm.amdgcn.av.load.b16.p1(ptr addrspace(1) %addr, metadata i32 0)
+ ret i16 %data
+}
+
+; CHECK: the last argument to av load/store intrinsics must be a metadata string
+; CHECK-NEXT: call void @llvm.amdgcn.av.store.b16.p1({{.*}})
+define void @av_global_store_b16_bad_metadata(ptr addrspace(1) %addr, i16 %data) {
+ call void @llvm.amdgcn.av.store.b16.p1(ptr addrspace(1) %addr, i16 %data, metadata i32 0)
+ ret void
+}
+
+; CHECK: the last argument to av load/store intrinsics must be a metadata string
+; CHECK-NEXT: call i32 @llvm.amdgcn.av.load.b32.p1({{.*}})
+define i32 @av_global_load_b32_bad_metadata(ptr addrspace(1) %addr) {
+ %data = call i32 @llvm.amdgcn.av.load.b32.p1(ptr addrspace(1) %addr, metadata i32 0)
+ ret i32 %data
+}
+
+; CHECK: the last argument to av load/store intrinsics must be a metadata string
+; CHECK-NEXT: call void @llvm.amdgcn.av.store.b32.p1({{.*}})
+define void @av_global_store_b32_bad_metadata(ptr addrspace(1) %addr, i32 %data) {
+ call void @llvm.amdgcn.av.store.b32.p1(ptr addrspace(1) %addr, i32 %data, metadata i32 0)
+ ret void
+}
+
+; CHECK: the last argument to av load/store intrinsics must be a metadata string
+; CHECK-NEXT: call i64 @llvm.amdgcn.av.load.b64.p1({{.*}})
+define i64 @av_global_load_b64_bad_metadata(ptr addrspace(1) %addr) {
+ %data = call i64 @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) %addr, metadata i32 0)
+ ret i64 %data
+}
+
+; CHECK: the last argument to av load/store intrinsics must be a metadata string
+; CHECK-NEXT: call void @llvm.amdgcn.av.store.b64.p1({{.*}})
+define void @av_global_store_b64_bad_metadata(ptr addrspace(1) %addr, i64 %data) {
+ call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) %addr, i64 %data, metadata i32 0)
+ ret void
+}
>From ba3792c5211e4196984f060131e251436e685049 Mon Sep 17 00:00:00 2001
From: pvanhout <pierre.vanhoutryve at amd.com>
Date: Tue, 6 Oct 2026 16:55:46 +0200
Subject: [PATCH 2/3] Add b8
---
clang/docs/LanguageExtensions.md | 10 +-
clang/include/clang/Basic/BuiltinsAMDGPU.td | 10 +
.../include/clang/Basic/BuiltinsAMDGPUDocs.td | 2 +
clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp | 8 +
clang/lib/Sema/SemaAMDGPU.cpp | 2 +
.../builtins-amdgcn-global-load-store.cl | 218 ++
clang/test/SemaHIP/amdgpu-av-load-store.hip | 11 +
...builtins-amdgcn-global-load-store-error.cl | 19 +
...s-amdgcn-global-load-store-target-error.cl | 7 +-
llvm/docs/AMDGPUMemoryModel.md | 4 +-
llvm/docs/AMDGPUUsage.rst | 1 +
llvm/include/llvm/IR/IntrinsicsAMDGPU.td | 3 +
llvm/lib/IR/VerifierAMDGPU.cpp | 2 +
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 2 +
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 6 +
llvm/test/CodeGen/AMDGPU/amdgcn-av-scopes.ll | 2124 +++++++++++++++++
.../AMDGPU/llvm.amdgcn.av.load.b128.ll | 404 +++-
.../AMDGPU/llvm.amdgcn.av.store.b128.ll | 196 ++
.../CodeGen/AMDGPU/unsupported-av-load.ll | 3 +
.../CodeGen/AMDGPU/unsupported-av-store.ll | 4 +-
llvm/test/Verifier/AMDGPU/intrinsics-av.ll | 14 +
21 files changed, 2946 insertions(+), 104 deletions(-)
diff --git a/clang/docs/LanguageExtensions.md b/clang/docs/LanguageExtensions.md
index cf88c678de553d..88cd29412ea255 100644
--- a/clang/docs/LanguageExtensions.md
+++ b/clang/docs/LanguageExtensions.md
@@ -5496,25 +5496,27 @@ returns the bit at the position of the current lane. It is almost equivalent to
`(mask & (1 << lane_id)) != 0`, except that its behavior is only defined if
the given mask has the same value for all active lanes of the current wave.
-#### \_\_builtin_amdgcn_av\_{load,store}\_{b16,b32,b64,b128}
+#### \_\_builtin_amdgcn_av\_{load,store}\_{b8,b16,b32,b64,b128}
Signature:
```c
typedef __attribute__((__vector_size__(4 * sizeof(unsigned int)))) unsigned int v4u;
+unsigned char __builtin_amdgcn_av_load_b8(unsigned char *src, int scope);
unsigned short __builtin_amdgcn_av_load_b16(unsigned short *src, int scope);
unsigned int __builtin_amdgcn_av_load_b32(unsigned int *src, int scope);
uint64_t __builtin_amdgcn_av_load_b64(uint64_t *src, int scope);
v4u __builtin_amdgcn_av_load_b128(v4u *src, int scope);
+void __builtin_amdgcn_av_store_b8(unsigned char *dst, unsigned char data, int scope);
void __builtin_amdgcn_av_store_b16(unsigned short *dst, unsigned short data, int scope);
void __builtin_amdgcn_av_store_b32(unsigned int *dst, unsigned int data, int scope);
void __builtin_amdgcn_av_store_b64(uint64_t *dst, uint64_t data, int scope);
void __builtin_amdgcn_av_store_b128(v4u *dst, v4u data, int scope);
```
-Load or store 16, 32, 64 or 128 bits from or to memory with cache behavior
+Load or store 8, 16, 32, 64 or 128 bits from or to memory with cache behavior
specified by `scope`, which is one of the `__MEMORY_SCOPE_*` macros defined for
{ref}`scoped atomic builtins <langext-c11-atomic>`.
@@ -5522,8 +5524,8 @@ The pointer argument must point to the global or generic address space.
These builtins are supported on gfx9, gfx10, gfx11, and gfx12 targets.
-They map to the LLVM intrinsics `llvm.amdgcn.av.load.b{16,32,64,128}` and
-`llvm.amdgcn.av.store.b{16,32,64,128}` documented in [User Guide for AMDGPU Backend](https://llvm.org/docs/AMDGPUUsage.html).
+They map to the LLVM intrinsics `llvm.amdgcn.av.load.b{8,16,32,64,128}` and
+`llvm.amdgcn.av.store.b{8,16,32,64,128}` documented in [User Guide for AMDGPU Backend](https://llvm.org/docs/AMDGPUUsage.html).
### ARM/AArch64 Language Extensions
diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.td b/clang/include/clang/Basic/BuiltinsAMDGPU.td
index 0d062d7fa2e43d..4cd8bef934912b 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.td
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.td
@@ -318,6 +318,16 @@ def __builtin_amdgcn_s_buffer_load_v4f16 : AMDGPUBuiltin<"_Vector<4, _Float16>(_
// Global Available/Visible memory accesses.
//===----------------------------------------------------------------------===//
+def __builtin_amdgcn_av_load_b8
+ : AMDGPUBuiltin<"unsigned char(unsigned char *, int)", [], "flat-global-insts"> {
+ let Documentation = [DocAVLoadB8];
+ let ArgNames = ["ptr", "scope"];
+}
+def __builtin_amdgcn_av_store_b8
+ : AMDGPUBuiltin<"void(unsigned char *, unsigned char, int)", [], "flat-global-insts"> {
+ let Documentation = [DocAVStoreB8];
+ let ArgNames = ["ptr", "data", "scope"];
+}
def __builtin_amdgcn_av_load_b16
: AMDGPUBuiltin<"unsigned short(unsigned short *, int)", [], "flat-global-insts"> {
let Documentation = [DocAVLoadB16];
diff --git a/clang/include/clang/Basic/BuiltinsAMDGPUDocs.td b/clang/include/clang/Basic/BuiltinsAMDGPUDocs.td
index 2822d430458e4d..87a7663e75e46c 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPUDocs.td
+++ b/clang/include/clang/Basic/BuiltinsAMDGPUDocs.td
@@ -628,6 +628,7 @@ constant.
}]);
}
+def DocAVLoadB8 : DocAVLoadBase<"8">;
def DocAVLoadB16 : DocAVLoadBase<"16">;
def DocAVLoadB32 : DocAVLoadBase<"32">;
def DocAVLoadB64 : DocAVLoadBase<"64">;
@@ -642,6 +643,7 @@ class DocAVStoreBase<string width> : Documentation {
}]);
}
+def DocAVStoreB8 : DocAVStoreBase<"8">;
def DocAVStoreB16 : DocAVStoreBase<"16">;
def DocAVStoreB32 : DocAVStoreBase<"32">;
def DocAVStoreB64 : DocAVStoreBase<"64">;
diff --git a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
index bc548a51ab7ecf..9521346ed574c3 100644
--- a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
@@ -1036,16 +1036,21 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
llvm::Function *F = CGM.getIntrinsic(IID, {Args[0]->getType()});
return Builder.CreateCall(F, {Args});
}
+ case AMDGPU::BI__builtin_amdgcn_av_load_b8:
case AMDGPU::BI__builtin_amdgcn_av_load_b16:
case AMDGPU::BI__builtin_amdgcn_av_load_b32:
case AMDGPU::BI__builtin_amdgcn_av_load_b64:
case AMDGPU::BI__builtin_amdgcn_av_load_b128:
+ case AMDGPU::BI__builtin_amdgcn_av_store_b8:
case AMDGPU::BI__builtin_amdgcn_av_store_b16:
case AMDGPU::BI__builtin_amdgcn_av_store_b32:
case AMDGPU::BI__builtin_amdgcn_av_store_b64:
case AMDGPU::BI__builtin_amdgcn_av_store_b128: {
Intrinsic::ID IID;
switch (BuiltinID) {
+ case AMDGPU::BI__builtin_amdgcn_av_load_b8:
+ IID = Intrinsic::amdgcn_av_load_b8;
+ break;
case AMDGPU::BI__builtin_amdgcn_av_load_b16:
IID = Intrinsic::amdgcn_av_load_b16;
break;
@@ -1058,6 +1063,9 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
case AMDGPU::BI__builtin_amdgcn_av_load_b128:
IID = Intrinsic::amdgcn_av_load_b128;
break;
+ case AMDGPU::BI__builtin_amdgcn_av_store_b8:
+ IID = Intrinsic::amdgcn_av_store_b8;
+ break;
case AMDGPU::BI__builtin_amdgcn_av_store_b16:
IID = Intrinsic::amdgcn_av_store_b16;
break;
diff --git a/clang/lib/Sema/SemaAMDGPU.cpp b/clang/lib/Sema/SemaAMDGPU.cpp
index 960bbbf93adad4..fa68291d88ebcc 100644
--- a/clang/lib/Sema/SemaAMDGPU.cpp
+++ b/clang/lib/Sema/SemaAMDGPU.cpp
@@ -220,11 +220,13 @@ bool SemaAMDGPU::CheckAMDGCNBuiltinFunctionCall(const TargetInfo &TI,
case AMDGPU::BI__builtin_amdgcn_cvt_scale_pk32_f32_bf6:
case AMDGPU::BI__builtin_amdgcn_cvt_scale_pk32_f32_fp6:
return SemaRef.BuiltinConstantArgRange(TheCall, 2, 0, 15);
+ case AMDGPU::BI__builtin_amdgcn_av_load_b8:
case AMDGPU::BI__builtin_amdgcn_av_load_b16:
case AMDGPU::BI__builtin_amdgcn_av_load_b32:
case AMDGPU::BI__builtin_amdgcn_av_load_b64:
case AMDGPU::BI__builtin_amdgcn_av_load_b128:
return checkAVLoadStore(TheCall, /*IsStore=*/false);
+ case AMDGPU::BI__builtin_amdgcn_av_store_b8:
case AMDGPU::BI__builtin_amdgcn_av_store_b16:
case AMDGPU::BI__builtin_amdgcn_av_store_b32:
case AMDGPU::BI__builtin_amdgcn_av_store_b64:
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-store.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-store.cl
index 64080920f225fc..e5fc5453f02f94 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-store.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-store.cl
@@ -9,9 +9,11 @@
typedef __attribute__((__vector_size__(4 * sizeof(unsigned int)))) unsigned int v4u32;
typedef v4u32 __global *global_ptr_to_v4u32;
+typedef unsigned char u8;
typedef unsigned short u16;
typedef unsigned int u32;
typedef unsigned long u64;
+typedef u8 __global *global_ptr_to_u8;
typedef u16 __global *global_ptr_to_u16;
typedef u32 __global *global_ptr_to_u32;
typedef u64 __global *global_ptr_to_u64;
@@ -74,6 +76,60 @@ v4u32 test_amdgcn_global_load_b128_cluster(global_ptr_to_v4u32 ptr) {
return __builtin_amdgcn_av_load_b128(ptr, __MEMORY_SCOPE_CLUSTR);
}
+// CHECK-LABEL: @test_amdgcn_global_load_b8_wave(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i8 @llvm.amdgcn.av.load.b8.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META8]])
+// CHECK-NEXT: ret i8 [[TMP0]]
+//
+u8 test_amdgcn_global_load_b8_wave(global_ptr_to_u8 ptr) {
+ return __builtin_amdgcn_av_load_b8(ptr, __MEMORY_SCOPE_WVFRNT);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_load_b8_workgroup(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i8 @llvm.amdgcn.av.load.b8.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META9]])
+// CHECK-NEXT: ret i8 [[TMP0]]
+//
+u8 test_amdgcn_global_load_b8_workgroup(global_ptr_to_u8 ptr) {
+ return __builtin_amdgcn_av_load_b8(ptr, __MEMORY_SCOPE_WRKGRP);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_load_b8_device(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i8 @llvm.amdgcn.av.load.b8.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META10]])
+// CHECK-NEXT: ret i8 [[TMP0]]
+//
+u8 test_amdgcn_global_load_b8_device(global_ptr_to_u8 ptr) {
+ return __builtin_amdgcn_av_load_b8(ptr, __MEMORY_SCOPE_DEVICE);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_load_b8_system(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i8 @llvm.amdgcn.av.load.b8.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META11]])
+// CHECK-NEXT: ret i8 [[TMP0]]
+//
+u8 test_amdgcn_global_load_b8_system(global_ptr_to_u8 ptr) {
+ return __builtin_amdgcn_av_load_b8(ptr, __MEMORY_SCOPE_SYSTEM);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_load_b8_single(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i8 @llvm.amdgcn.av.load.b8.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META12]])
+// CHECK-NEXT: ret i8 [[TMP0]]
+//
+u8 test_amdgcn_global_load_b8_single(global_ptr_to_u8 ptr) {
+ return __builtin_amdgcn_av_load_b8(ptr, __MEMORY_SCOPE_SINGLE);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_load_b8_cluster(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i8 @llvm.amdgcn.av.load.b8.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META13]])
+// CHECK-NEXT: ret i8 [[TMP0]]
+//
+u8 test_amdgcn_global_load_b8_cluster(global_ptr_to_u8 ptr) {
+ return __builtin_amdgcn_av_load_b8(ptr, __MEMORY_SCOPE_CLUSTR);
+}
+
// CHECK-LABEL: @test_amdgcn_global_load_b16_wave(
// CHECK-NEXT: entry:
// CHECK-NEXT: [[TMP0:%.*]] = tail call i16 @llvm.amdgcn.av.load.b16.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META8]])
@@ -294,6 +350,60 @@ void test_amdgcn_global_store_b128_cluster(global_ptr_to_v4u32 ptr, v4u32 data)
__builtin_amdgcn_av_store_b128(ptr, data, __MEMORY_SCOPE_CLUSTR);
}
+// CHECK-LABEL: @test_amdgcn_global_store_b8_wave(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b8.p1(ptr addrspace(1) [[PTR:%.*]], i8 [[DATA:%.*]], metadata [[META8]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_global_store_b8_wave(global_ptr_to_u8 ptr, u8 data) {
+ __builtin_amdgcn_av_store_b8(ptr, data, __MEMORY_SCOPE_WVFRNT);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_store_b8_workgroup(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b8.p1(ptr addrspace(1) [[PTR:%.*]], i8 [[DATA:%.*]], metadata [[META9]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_global_store_b8_workgroup(global_ptr_to_u8 ptr, u8 data) {
+ __builtin_amdgcn_av_store_b8(ptr, data, __MEMORY_SCOPE_WRKGRP);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_store_b8_device(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b8.p1(ptr addrspace(1) [[PTR:%.*]], i8 [[DATA:%.*]], metadata [[META10]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_global_store_b8_device(global_ptr_to_u8 ptr, u8 data) {
+ __builtin_amdgcn_av_store_b8(ptr, data, __MEMORY_SCOPE_DEVICE);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_store_b8_system(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b8.p1(ptr addrspace(1) [[PTR:%.*]], i8 [[DATA:%.*]], metadata [[META11]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_global_store_b8_system(global_ptr_to_u8 ptr, u8 data) {
+ __builtin_amdgcn_av_store_b8(ptr, data, __MEMORY_SCOPE_SYSTEM);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_store_b8_single(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b8.p1(ptr addrspace(1) [[PTR:%.*]], i8 [[DATA:%.*]], metadata [[META12]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_global_store_b8_single(global_ptr_to_u8 ptr, u8 data) {
+ __builtin_amdgcn_av_store_b8(ptr, data, __MEMORY_SCOPE_SINGLE);
+}
+
+// CHECK-LABEL: @test_amdgcn_global_store_b8_cluster(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b8.p1(ptr addrspace(1) [[PTR:%.*]], i8 [[DATA:%.*]], metadata [[META13]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_global_store_b8_cluster(global_ptr_to_u8 ptr, u8 data) {
+ __builtin_amdgcn_av_store_b8(ptr, data, __MEMORY_SCOPE_CLUSTR);
+}
+
// CHECK-LABEL: @test_amdgcn_global_store_b16_wave(
// CHECK-NEXT: entry:
// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b16.p1(ptr addrspace(1) [[PTR:%.*]], i16 [[DATA:%.*]], metadata [[META8]])
@@ -514,6 +624,60 @@ v4u32 test_amdgcn_flat_load_b128_cluster(v4u32 * ptr) {
return __builtin_amdgcn_av_load_b128(ptr, __MEMORY_SCOPE_CLUSTR);
}
+// CHECK-LABEL: @test_amdgcn_flat_load_b8_wave(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i8 @llvm.amdgcn.av.load.b8.p0(ptr [[PTR:%.*]], metadata [[META8]])
+// CHECK-NEXT: ret i8 [[TMP0]]
+//
+u8 test_amdgcn_flat_load_b8_wave(u8 * ptr) {
+ return __builtin_amdgcn_av_load_b8(ptr, __MEMORY_SCOPE_WVFRNT);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_load_b8_workgroup(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i8 @llvm.amdgcn.av.load.b8.p0(ptr [[PTR:%.*]], metadata [[META9]])
+// CHECK-NEXT: ret i8 [[TMP0]]
+//
+u8 test_amdgcn_flat_load_b8_workgroup(u8 * ptr) {
+ return __builtin_amdgcn_av_load_b8(ptr, __MEMORY_SCOPE_WRKGRP);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_load_b8_device(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i8 @llvm.amdgcn.av.load.b8.p0(ptr [[PTR:%.*]], metadata [[META10]])
+// CHECK-NEXT: ret i8 [[TMP0]]
+//
+u8 test_amdgcn_flat_load_b8_device(u8 * ptr) {
+ return __builtin_amdgcn_av_load_b8(ptr, __MEMORY_SCOPE_DEVICE);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_load_b8_system(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i8 @llvm.amdgcn.av.load.b8.p0(ptr [[PTR:%.*]], metadata [[META11]])
+// CHECK-NEXT: ret i8 [[TMP0]]
+//
+u8 test_amdgcn_flat_load_b8_system(u8 * ptr) {
+ return __builtin_amdgcn_av_load_b8(ptr, __MEMORY_SCOPE_SYSTEM);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_load_b8_single(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i8 @llvm.amdgcn.av.load.b8.p0(ptr [[PTR:%.*]], metadata [[META12]])
+// CHECK-NEXT: ret i8 [[TMP0]]
+//
+u8 test_amdgcn_flat_load_b8_single(u8 * ptr) {
+ return __builtin_amdgcn_av_load_b8(ptr, __MEMORY_SCOPE_SINGLE);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_load_b8_cluster(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call i8 @llvm.amdgcn.av.load.b8.p0(ptr [[PTR:%.*]], metadata [[META13]])
+// CHECK-NEXT: ret i8 [[TMP0]]
+//
+u8 test_amdgcn_flat_load_b8_cluster(u8 * ptr) {
+ return __builtin_amdgcn_av_load_b8(ptr, __MEMORY_SCOPE_CLUSTR);
+}
+
// CHECK-LABEL: @test_amdgcn_flat_load_b16_wave(
// CHECK-NEXT: entry:
// CHECK-NEXT: [[TMP0:%.*]] = tail call i16 @llvm.amdgcn.av.load.b16.p0(ptr [[PTR:%.*]], metadata [[META8]])
@@ -734,6 +898,60 @@ void test_amdgcn_flat_store_b128_cluster(v4u32 * ptr, v4u32 data) {
__builtin_amdgcn_av_store_b128(ptr, data, __MEMORY_SCOPE_CLUSTR);
}
+// CHECK-LABEL: @test_amdgcn_flat_store_b8_wave(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b8.p0(ptr [[PTR:%.*]], i8 [[DATA:%.*]], metadata [[META8]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_flat_store_b8_wave(u8 * ptr, u8 data) {
+ __builtin_amdgcn_av_store_b8(ptr, data, __MEMORY_SCOPE_WVFRNT);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_store_b8_workgroup(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b8.p0(ptr [[PTR:%.*]], i8 [[DATA:%.*]], metadata [[META9]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_flat_store_b8_workgroup(u8 * ptr, u8 data) {
+ __builtin_amdgcn_av_store_b8(ptr, data, __MEMORY_SCOPE_WRKGRP);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_store_b8_device(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b8.p0(ptr [[PTR:%.*]], i8 [[DATA:%.*]], metadata [[META10]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_flat_store_b8_device(u8 * ptr, u8 data) {
+ __builtin_amdgcn_av_store_b8(ptr, data, __MEMORY_SCOPE_DEVICE);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_store_b8_system(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b8.p0(ptr [[PTR:%.*]], i8 [[DATA:%.*]], metadata [[META11]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_flat_store_b8_system(u8 * ptr, u8 data) {
+ __builtin_amdgcn_av_store_b8(ptr, data, __MEMORY_SCOPE_SYSTEM);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_store_b8_single(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b8.p0(ptr [[PTR:%.*]], i8 [[DATA:%.*]], metadata [[META12]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_flat_store_b8_single(u8 * ptr, u8 data) {
+ __builtin_amdgcn_av_store_b8(ptr, data, __MEMORY_SCOPE_SINGLE);
+}
+
+// CHECK-LABEL: @test_amdgcn_flat_store_b8_cluster(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b8.p0(ptr [[PTR:%.*]], i8 [[DATA:%.*]], metadata [[META13]])
+// CHECK-NEXT: ret void
+//
+void test_amdgcn_flat_store_b8_cluster(u8 * ptr, u8 data) {
+ __builtin_amdgcn_av_store_b8(ptr, data, __MEMORY_SCOPE_CLUSTR);
+}
+
// CHECK-LABEL: @test_amdgcn_flat_store_b16_wave(
// CHECK-NEXT: entry:
// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b16.p0(ptr [[PTR:%.*]], i16 [[DATA:%.*]], metadata [[META8]])
diff --git a/clang/test/SemaHIP/amdgpu-av-load-store.hip b/clang/test/SemaHIP/amdgpu-av-load-store.hip
index 5d4a4e62e179ad..b354dba388aff2 100644
--- a/clang/test/SemaHIP/amdgpu-av-load-store.hip
+++ b/clang/test/SemaHIP/amdgpu-av-load-store.hip
@@ -8,6 +8,7 @@
#define __global__ __attribute__((global))
typedef __attribute__((__vector_size__(4 * sizeof(unsigned int)))) unsigned int v4u32;
+typedef unsigned char u8;
typedef unsigned short u16;
typedef unsigned int u32;
typedef unsigned long u64;
@@ -22,6 +23,16 @@ __global__ void test_av_load_store_kernel(v4u32 *ptr, v4u32 data) {
__builtin_amdgcn_av_store_b128(ptr, data, 0);
}
+__device__ void test_av_load_store_b8_device(u8 *ptr, u8 data) {
+ u8 res = __builtin_amdgcn_av_load_b8(ptr, 0);
+ __builtin_amdgcn_av_store_b8(ptr, data, 0);
+}
+
+__global__ void test_av_load_store_b8_kernel(u8 *ptr, u8 data) {
+ u8 res = __builtin_amdgcn_av_load_b8(ptr, 0);
+ __builtin_amdgcn_av_store_b8(ptr, data, 0);
+}
+
__device__ void test_av_load_store_b16_device(u16 *ptr, u16 data) {
u16 res = __builtin_amdgcn_av_load_b16(ptr, 0);
__builtin_amdgcn_av_store_b16(ptr, data, 0);
diff --git a/clang/test/SemaOpenCL/builtins-amdgcn-global-load-store-error.cl b/clang/test/SemaOpenCL/builtins-amdgcn-global-load-store-error.cl
index 4938491be78ba5..dba8e7fbe2b067 100644
--- a/clang/test/SemaOpenCL/builtins-amdgcn-global-load-store-error.cl
+++ b/clang/test/SemaOpenCL/builtins-amdgcn-global-load-store-error.cl
@@ -1,6 +1,7 @@
// RUN: %clang_cc1 -cl-std=CL2.0 -triple amdgpu9.50-unknown-unknown -S -verify -o - %s
// REQUIRES: amdgpu-registered-target
+typedef unsigned char u8;
typedef unsigned short u16;
typedef unsigned int u32;
typedef unsigned long u64;
@@ -8,6 +9,8 @@ typedef unsigned long u64;
typedef __attribute__((__vector_size__(4 * sizeof(unsigned int)))) unsigned int v4u32;
typedef v4u32 __global *global_ptr_to_v4u32;
typedef v4u32 __private *private_ptr_to_v4u32;
+typedef u8 __global *global_ptr_to_u8;
+typedef u8 __private *private_ptr_to_u8;
typedef u16 __global *global_ptr_to_u16;
typedef u16 __private *private_ptr_to_u16;
typedef u32 __global *global_ptr_to_u32;
@@ -31,6 +34,22 @@ v4u32 test_amdgcn_av_load_b128_bad_scope(global_ptr_to_v4u32 ptr) {
return __builtin_amdgcn_av_load_b128(ptr, 42); //expected-error{{synchronization scope argument to atomic operation is invalid}}
}
+void test_amdgcn_av_store_b8_bad_ptr(private_ptr_to_u8 ptr, u8 data) {
+ __builtin_amdgcn_av_store_b8(ptr, data, __MEMORY_SCOPE_SYSTEM); //expected-error{{builtin requires a global or generic pointer}}
+}
+
+void test_amdgcn_av_store_b8_bad_scope(global_ptr_to_u8 ptr, u8 data) {
+ __builtin_amdgcn_av_store_b8(ptr, data, 42); //expected-error{{synchronization scope argument to atomic operation is invalid}}
+}
+
+u8 test_amdgcn_av_load_b8_bad_ptr(private_ptr_to_u8 ptr) {
+ return __builtin_amdgcn_av_load_b8(ptr, __MEMORY_SCOPE_SYSTEM); //expected-error{{builtin requires a global or generic pointer}}
+}
+
+u8 test_amdgcn_av_load_b8_bad_scope(global_ptr_to_u8 ptr) {
+ return __builtin_amdgcn_av_load_b8(ptr, 42); //expected-error{{synchronization scope argument to atomic operation is invalid}}
+}
+
void test_amdgcn_av_store_b16_bad_ptr(private_ptr_to_u16 ptr, u16 data) {
__builtin_amdgcn_av_store_b16(ptr, data, __MEMORY_SCOPE_SYSTEM); //expected-error{{builtin requires a global or generic pointer}}
}
diff --git a/clang/test/SemaOpenCL/builtins-amdgcn-global-load-store-target-error.cl b/clang/test/SemaOpenCL/builtins-amdgcn-global-load-store-target-error.cl
index 5d6d6d5be688bf..fab983a57de1e2 100644
--- a/clang/test/SemaOpenCL/builtins-amdgcn-global-load-store-target-error.cl
+++ b/clang/test/SemaOpenCL/builtins-amdgcn-global-load-store-target-error.cl
@@ -5,21 +5,26 @@
typedef __attribute__((__vector_size__(4 * sizeof(unsigned int)))) unsigned int v4u32;
typedef v4u32 __global *global_ptr_to_v4u32;
+typedef unsigned char u8;
typedef unsigned short u16;
typedef unsigned int u32;
typedef unsigned long u64;
+typedef u8 __global *global_ptr_to_u8;
typedef u16 __global *global_ptr_to_u16;
typedef u32 __global *global_ptr_to_u32;
typedef u64 __global *global_ptr_to_u64;
-void test_amdgcn_av_load_store_target(global_ptr_to_u16 ptr16, u16 data16,
+void test_amdgcn_av_load_store_target(global_ptr_to_u8 ptr8, u8 data8,
+ global_ptr_to_u16 ptr16, u16 data16,
global_ptr_to_u32 ptr32, u32 data32,
global_ptr_to_u64 ptr64, u64 data64,
global_ptr_to_v4u32 ptr128, v4u32 data128) {
+ __builtin_amdgcn_av_load_b8(ptr8, 0); // expected-error{{'__builtin_amdgcn_av_load_b8' needs target feature flat-global-insts}}
__builtin_amdgcn_av_load_b16(ptr16, 0); // expected-error{{'__builtin_amdgcn_av_load_b16' needs target feature flat-global-insts}}
__builtin_amdgcn_av_load_b32(ptr32, 0); // expected-error{{'__builtin_amdgcn_av_load_b32' needs target feature flat-global-insts}}
__builtin_amdgcn_av_load_b64(ptr64, 0); // expected-error{{'__builtin_amdgcn_av_load_b64' needs target feature flat-global-insts}}
__builtin_amdgcn_av_load_b128(ptr128, 0); // expected-error{{'__builtin_amdgcn_av_load_b128' needs target feature flat-global-insts}}
+ __builtin_amdgcn_av_store_b8(ptr8, data8, 0); // expected-error{{'__builtin_amdgcn_av_store_b8' needs target feature flat-global-insts}}
__builtin_amdgcn_av_store_b16(ptr16, data16, 0); // expected-error{{'__builtin_amdgcn_av_store_b16' needs target feature flat-global-insts}}
__builtin_amdgcn_av_store_b32(ptr32, data32, 0); // expected-error{{'__builtin_amdgcn_av_store_b32' needs target feature flat-global-insts}}
__builtin_amdgcn_av_store_b64(ptr64, data64, 0); // expected-error{{'__builtin_amdgcn_av_store_b64' needs target feature flat-global-insts}}
diff --git a/llvm/docs/AMDGPUMemoryModel.md b/llvm/docs/AMDGPUMemoryModel.md
index 9b0e9ee2046d63..f8c88309f72188 100644
--- a/llvm/docs/AMDGPUMemoryModel.md
+++ b/llvm/docs/AMDGPUMemoryModel.md
@@ -155,7 +155,7 @@ the subscope instances of `S` that contain `X`.
@llvm.amdgcn.av.store.b<N>(%addr, %value, metadata !scope)
```
-The `@llvm.amdgcn.av.store.b{16,32,64,128}` intrinsics perform a non-atomic
+The `@llvm.amdgcn.av.store.b{8,16,32,64,128}` intrinsics perform a non-atomic
*store-available* operation on `%addr` with scope `!scope`. See
{ref}`amdgpu-av-load-store` for details.
@@ -181,7 +181,7 @@ that contain `Y`.
%value = @llvm.amdgcn.av.load.b<N>(%addr, metadata !scope)
```
-The `@llvm.amdgcn.av.load.b{16,32,64,128}` intrinsics perform a non-atomic *load-visible*
+The `@llvm.amdgcn.av.load.b{8,16,32,64,128}` intrinsics perform a non-atomic *load-visible*
operation on `%addr` with scope `!scope`. See {ref}`amdgpu-av-load-store` for
details.
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 23eb8a2c361fc3..a6070cf082ff81 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -2260,6 +2260,7 @@ These intrinsics exist for the following data types:
========== ============= ====================================== =======================================
Width Data Type Load Store
========== ============= ====================================== =======================================
+ 8 bits ``i8`` ``llvm.amdgcn.av.load.b8`` ``llvm.amdgcn.av.store.b8``
16 bits ``i16`` ``llvm.amdgcn.av.load.b16`` ``llvm.amdgcn.av.store.b16``
32 bits ``i32`` ``llvm.amdgcn.av.load.b32`` ``llvm.amdgcn.av.store.b32``
64 bits ``i64`` ``llvm.amdgcn.av.load.b64`` ``llvm.amdgcn.av.store.b64``
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 87a33170a323d0..ca6ff65f9da4ef 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -1016,6 +1016,9 @@ class AMDGPUAVLoad<LLVMType ty> : Intrinsic <
>;
let TargetFeatures = "flat-global-insts" in {
+def int_amdgcn_av_store_b8 : AMDGPUAVStore<llvm_i8_ty>;
+def int_amdgcn_av_load_b8 : AMDGPUAVLoad<llvm_i8_ty>;
+
def int_amdgcn_av_store_b16 : AMDGPUAVStore<llvm_i16_ty>;
def int_amdgcn_av_load_b16 : AMDGPUAVLoad<llvm_i16_ty>;
diff --git a/llvm/lib/IR/VerifierAMDGPU.cpp b/llvm/lib/IR/VerifierAMDGPU.cpp
index 068c95c8a34e41..fc20589915b920 100644
--- a/llvm/lib/IR/VerifierAMDGPU.cpp
+++ b/llvm/lib/IR/VerifierAMDGPU.cpp
@@ -433,10 +433,12 @@ void llvm::verifyAMDGPUIntrinsicCall(VerifierSupport &VS, Intrinsic::ID ID,
&Call, Op);
break;
}
+ case Intrinsic::amdgcn_av_load_b8:
case Intrinsic::amdgcn_av_load_b16:
case Intrinsic::amdgcn_av_load_b32:
case Intrinsic::amdgcn_av_load_b64:
case Intrinsic::amdgcn_av_load_b128:
+ case Intrinsic::amdgcn_av_store_b8:
case Intrinsic::amdgcn_av_store_b16:
case Intrinsic::amdgcn_av_store_b32:
case Intrinsic::amdgcn_av_store_b64:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 68298fa59bb8b1..547cc3ad6c6a5b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -8838,10 +8838,12 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
B.buildStore(MI.getOperand(2), MI.getOperand(1), **MI.memoperands_begin());
MI.eraseFromParent();
return true;
+ case Intrinsic::amdgcn_av_load_b8:
case Intrinsic::amdgcn_av_load_b16:
case Intrinsic::amdgcn_av_load_b32:
case Intrinsic::amdgcn_av_load_b64:
case Intrinsic::amdgcn_av_load_b128:
+ case Intrinsic::amdgcn_av_store_b8:
case Intrinsic::amdgcn_av_store_b16:
case Intrinsic::amdgcn_av_store_b32:
case Intrinsic::amdgcn_av_store_b64:
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index e12c126e01a17b..6c213d2931ff1b 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1869,10 +1869,12 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
Infos.push_back(Info);
return;
}
+ case Intrinsic::amdgcn_av_load_b8:
case Intrinsic::amdgcn_av_load_b16:
case Intrinsic::amdgcn_av_load_b32:
case Intrinsic::amdgcn_av_load_b64:
case Intrinsic::amdgcn_av_load_b128:
+ case Intrinsic::amdgcn_av_store_b8:
case Intrinsic::amdgcn_av_store_b16:
case Intrinsic::amdgcn_av_store_b32:
case Intrinsic::amdgcn_av_store_b64:
@@ -1998,10 +2000,12 @@ bool SITargetLowering::getAddrModeArguments(const IntrinsicInst *II,
case Intrinsic::amdgcn_global_store_async_from_lds_b32:
case Intrinsic::amdgcn_global_store_async_from_lds_b64:
case Intrinsic::amdgcn_global_store_async_from_lds_b128:
+ case Intrinsic::amdgcn_av_load_b8:
case Intrinsic::amdgcn_av_load_b16:
case Intrinsic::amdgcn_av_load_b32:
case Intrinsic::amdgcn_av_load_b64:
case Intrinsic::amdgcn_av_load_b128:
+ case Intrinsic::amdgcn_av_store_b8:
case Intrinsic::amdgcn_av_store_b16:
case Intrinsic::amdgcn_av_store_b32:
case Intrinsic::amdgcn_av_store_b64:
@@ -12529,6 +12533,7 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
return DAG.getAtomicLoad(ISD::NON_EXTLOAD, DL, MII->getMemoryVT(), VT,
Chain, Ptr, MII->getMemOperand());
}
+ case Intrinsic::amdgcn_av_load_b8:
case Intrinsic::amdgcn_av_load_b16:
case Intrinsic::amdgcn_av_load_b32:
case Intrinsic::amdgcn_av_load_b64:
@@ -13241,6 +13246,7 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
return DAG.getAtomic(ISD::ATOMIC_STORE, DL, MII->getMemoryVT(), Chain, Val,
Ptr, MII->getMemOperand());
}
+ case Intrinsic::amdgcn_av_store_b8:
case Intrinsic::amdgcn_av_store_b16:
case Intrinsic::amdgcn_av_store_b32:
case Intrinsic::amdgcn_av_store_b64:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn-av-scopes.ll b/llvm/test/CodeGen/AMDGPU/amdgcn-av-scopes.ll
index e82483652e5a27..d59b5059246803 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn-av-scopes.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn-av-scopes.ll
@@ -2140,6 +2140,2130 @@ define void @flat_store_cluster(ptr %p, <4 x i32> %v) {
ret void
}
+; scope: wavefront
+define i8 @global_load_b8_wavefront(ptr addrspace(1) %p) {
+; GFX942-SDAG-LABEL: global_load_b8_wavefront:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: global_load_b8_wavefront:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: global_load_b8_wavefront:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: global_load_d16_u8 v0, v[0:1], off
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: global_load_b8_wavefront:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_load_u8 v0, v[0:1], off
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: global_load_b8_wavefront:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: global_load_b8_wavefront:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: global_load_d16_u8 v0, v[0:1], off
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: global_load_b8_wavefront:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: global_load_b8_wavefront:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: global_load_b8_wavefront:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: global_load_u8 v0, v[0:1], off
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: global_load_b8_wavefront:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: global_load_u8 v0, v[0:1], off
+; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: global_load_b8_wavefront:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: global_load_b8_wavefront:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: global_load_u8 v0, v[0:1], off
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ %v = call i8 @llvm.amdgcn.av.load.b8.p1(ptr addrspace(1) %p, metadata !0)
+ ret i8 %v
+}
+
+; scope: workgroup
+define i8 @global_load_b8_workgroup(ptr addrspace(1) %p) {
+; GFX942-SDAG-LABEL: global_load_b8_workgroup:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: global_load_ubyte v0, v[0:1], off sc0
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: global_load_b8_workgroup:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: global_load_ubyte v0, v[0:1], off glc
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: global_load_b8_workgroup:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: global_load_d16_u8 v0, v[0:1], off glc
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: global_load_b8_workgroup:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_load_u8 v0, v[0:1], off
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: global_load_b8_workgroup:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: global_load_ubyte v0, v[0:1], off glc
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: global_load_b8_workgroup:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: global_load_d16_u8 v0, v[0:1], off glc
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: global_load_b8_workgroup:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: global_load_ubyte v0, v[0:1], off sc0
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: global_load_b8_workgroup:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: global_load_ubyte v0, v[0:1], off glc
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: global_load_b8_workgroup:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: global_load_u8 v0, v[0:1], off glc
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: global_load_b8_workgroup:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: global_load_u8 v0, v[0:1], off
+; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: global_load_b8_workgroup:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: global_load_ubyte v0, v[0:1], off glc
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: global_load_b8_workgroup:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: global_load_u8 v0, v[0:1], off glc
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ %v = call i8 @llvm.amdgcn.av.load.b8.p1(ptr addrspace(1) %p, metadata !1)
+ ret i8 %v
+}
+
+; scope: agent
+define i8 @global_load_b8_agent(ptr addrspace(1) %p) {
+; GFX942-SDAG-LABEL: global_load_b8_agent:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: global_load_ubyte v0, v[0:1], off sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: global_load_b8_agent:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: global_load_ubyte v0, v[0:1], off glc dlc
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: global_load_b8_agent:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: global_load_d16_u8 v0, v[0:1], off glc
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: global_load_b8_agent:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_load_u8 v0, v[0:1], off scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: global_load_b8_agent:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: global_load_ubyte v0, v[0:1], off glc dlc
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: global_load_b8_agent:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: global_load_d16_u8 v0, v[0:1], off glc
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: global_load_b8_agent:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: global_load_ubyte v0, v[0:1], off sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: global_load_b8_agent:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: global_load_ubyte v0, v[0:1], off glc dlc
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: global_load_b8_agent:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: global_load_u8 v0, v[0:1], off glc
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: global_load_b8_agent:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: global_load_u8 v0, v[0:1], off scope:SCOPE_DEV
+; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: global_load_b8_agent:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: global_load_ubyte v0, v[0:1], off glc dlc
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: global_load_b8_agent:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: global_load_u8 v0, v[0:1], off glc
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ %v = call i8 @llvm.amdgcn.av.load.b8.p1(ptr addrspace(1) %p, metadata !2)
+ ret i8 %v
+}
+
+; scope: system
+define i8 @global_load_b8_system(ptr addrspace(1) %p) {
+; GFX942-SDAG-LABEL: global_load_b8_system:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: global_load_ubyte v0, v[0:1], off sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: global_load_b8_system:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: global_load_ubyte v0, v[0:1], off glc dlc
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: global_load_b8_system:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: global_load_d16_u8 v0, v[0:1], off glc
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: global_load_b8_system:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_load_u8 v0, v[0:1], off scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: global_load_b8_system:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: global_load_ubyte v0, v[0:1], off glc dlc
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: global_load_b8_system:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: global_load_d16_u8 v0, v[0:1], off glc
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: global_load_b8_system:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: global_load_ubyte v0, v[0:1], off sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: global_load_b8_system:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: global_load_ubyte v0, v[0:1], off glc dlc
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: global_load_b8_system:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: global_load_u8 v0, v[0:1], off glc
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: global_load_b8_system:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: global_load_u8 v0, v[0:1], off scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: global_load_b8_system:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: global_load_ubyte v0, v[0:1], off glc dlc
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: global_load_b8_system:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: global_load_u8 v0, v[0:1], off glc
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ %v = call i8 @llvm.amdgcn.av.load.b8.p1(ptr addrspace(1) %p, metadata !3)
+ ret i8 %v
+}
+
+; scope: singlethread
+define i8 @global_load_b8_singlethread(ptr addrspace(1) %p) {
+; GFX942-SDAG-LABEL: global_load_b8_singlethread:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: global_load_b8_singlethread:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: global_load_b8_singlethread:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: global_load_d16_u8 v0, v[0:1], off
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: global_load_b8_singlethread:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_load_u8 v0, v[0:1], off
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: global_load_b8_singlethread:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: global_load_b8_singlethread:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: global_load_d16_u8 v0, v[0:1], off
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: global_load_b8_singlethread:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: global_load_b8_singlethread:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: global_load_b8_singlethread:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: global_load_u8 v0, v[0:1], off
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: global_load_b8_singlethread:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: global_load_u8 v0, v[0:1], off
+; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: global_load_b8_singlethread:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: global_load_b8_singlethread:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: global_load_u8 v0, v[0:1], off
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ %v = call i8 @llvm.amdgcn.av.load.b8.p1(ptr addrspace(1) %p, metadata !4)
+ ret i8 %v
+}
+
+; scope: cluster
+define i8 @global_load_b8_cluster(ptr addrspace(1) %p) {
+; GFX942-SDAG-LABEL: global_load_b8_cluster:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: global_load_ubyte v0, v[0:1], off sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: global_load_b8_cluster:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: global_load_ubyte v0, v[0:1], off glc dlc
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: global_load_b8_cluster:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: global_load_d16_u8 v0, v[0:1], off glc
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: global_load_b8_cluster:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_load_u8 v0, v[0:1], off scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: global_load_b8_cluster:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: global_load_ubyte v0, v[0:1], off glc dlc
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: global_load_b8_cluster:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: global_load_d16_u8 v0, v[0:1], off glc
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: global_load_b8_cluster:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: global_load_ubyte v0, v[0:1], off sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: global_load_b8_cluster:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: global_load_ubyte v0, v[0:1], off glc dlc
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: global_load_b8_cluster:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: global_load_u8 v0, v[0:1], off glc
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: global_load_b8_cluster:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: global_load_u8 v0, v[0:1], off scope:SCOPE_SE
+; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: global_load_b8_cluster:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: global_load_ubyte v0, v[0:1], off glc dlc
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: global_load_b8_cluster:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: global_load_u8 v0, v[0:1], off glc
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ %v = call i8 @llvm.amdgcn.av.load.b8.p1(ptr addrspace(1) %p, metadata !5)
+ ret i8 %v
+}
+
+; scope: wavefront
+define void @global_store_b8_wavefront(ptr addrspace(1) %p, i8 %v) {
+; GFX942-SDAG-LABEL: global_store_b8_wavefront:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: global_store_byte v[0:1], v2, off
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: global_store_b8_wavefront:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: global_store_byte v[0:1], v2, off
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: global_store_b8_wavefront:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: global_store_b8_wavefront:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: global_store_b8_wavefront:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: global_store_byte v[0:1], v2, off
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: global_store_b8_wavefront:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: global_store_b8_wavefront:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: global_store_byte v[0:1], v2, off
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: global_store_b8_wavefront:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: global_store_byte v[0:1], v2, off
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: global_store_b8_wavefront:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: global_store_b8_wavefront:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: global_store_b8_wavefront:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: global_store_byte v[0:1], v2, off
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: global_store_b8_wavefront:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ call void @llvm.amdgcn.av.store.b8.p1(ptr addrspace(1) %p, i8 %v, metadata !0)
+ ret void
+}
+
+; scope: workgroup
+define void @global_store_b8_workgroup(ptr addrspace(1) %p, i8 %v) {
+; GFX942-SDAG-LABEL: global_store_b8_workgroup:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: global_store_byte v[0:1], v2, off sc0
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: global_store_b8_workgroup:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: global_store_byte v[0:1], v2, off
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: global_store_b8_workgroup:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: global_store_b8_workgroup:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: global_store_b8_workgroup:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: global_store_byte v[0:1], v2, off
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: global_store_b8_workgroup:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: global_store_b8_workgroup:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: global_store_byte v[0:1], v2, off sc0
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: global_store_b8_workgroup:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: global_store_byte v[0:1], v2, off
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: global_store_b8_workgroup:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: global_store_b8_workgroup:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: global_store_b8_workgroup:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: global_store_byte v[0:1], v2, off
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: global_store_b8_workgroup:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ call void @llvm.amdgcn.av.store.b8.p1(ptr addrspace(1) %p, i8 %v, metadata !1)
+ ret void
+}
+
+; scope: agent
+define void @global_store_b8_agent(ptr addrspace(1) %p, i8 %v) {
+; GFX942-SDAG-LABEL: global_store_b8_agent:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: global_store_byte v[0:1], v2, off sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: global_store_b8_agent:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: global_store_byte v[0:1], v2, off
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: global_store_b8_agent:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: global_store_b8_agent:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_store_b8 v[0:1], v2, off scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: global_store_b8_agent:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: global_store_byte v[0:1], v2, off
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: global_store_b8_agent:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: global_store_b8_agent:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: global_store_byte v[0:1], v2, off sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: global_store_b8_agent:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: global_store_byte v[0:1], v2, off
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: global_store_b8_agent:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: global_store_b8_agent:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: global_store_b8 v[0:1], v2, off scope:SCOPE_DEV
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: global_store_b8_agent:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: global_store_byte v[0:1], v2, off
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: global_store_b8_agent:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ call void @llvm.amdgcn.av.store.b8.p1(ptr addrspace(1) %p, i8 %v, metadata !2)
+ ret void
+}
+
+; scope: system
+define void @global_store_b8_system(ptr addrspace(1) %p, i8 %v) {
+; GFX942-SDAG-LABEL: global_store_b8_system:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: global_store_byte v[0:1], v2, off sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: global_store_b8_system:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: global_store_byte v[0:1], v2, off
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: global_store_b8_system:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: global_store_b8_system:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_store_b8 v[0:1], v2, off scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: global_store_b8_system:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: global_store_byte v[0:1], v2, off
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: global_store_b8_system:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: global_store_b8_system:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: global_store_byte v[0:1], v2, off sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: global_store_b8_system:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: global_store_byte v[0:1], v2, off
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: global_store_b8_system:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: global_store_b8_system:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: global_store_b8 v[0:1], v2, off scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: global_store_b8_system:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: global_store_byte v[0:1], v2, off
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: global_store_b8_system:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ call void @llvm.amdgcn.av.store.b8.p1(ptr addrspace(1) %p, i8 %v, metadata !3)
+ ret void
+}
+
+; scope: singlethread
+define void @global_store_b8_singlethread(ptr addrspace(1) %p, i8 %v) {
+; GFX942-SDAG-LABEL: global_store_b8_singlethread:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: global_store_byte v[0:1], v2, off
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: global_store_b8_singlethread:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: global_store_byte v[0:1], v2, off
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: global_store_b8_singlethread:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: global_store_b8_singlethread:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: global_store_b8_singlethread:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: global_store_byte v[0:1], v2, off
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: global_store_b8_singlethread:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: global_store_b8_singlethread:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: global_store_byte v[0:1], v2, off
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: global_store_b8_singlethread:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: global_store_byte v[0:1], v2, off
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: global_store_b8_singlethread:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: global_store_b8_singlethread:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: global_store_b8_singlethread:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: global_store_byte v[0:1], v2, off
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: global_store_b8_singlethread:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ call void @llvm.amdgcn.av.store.b8.p1(ptr addrspace(1) %p, i8 %v, metadata !4)
+ ret void
+}
+
+; scope: cluster
+define void @global_store_b8_cluster(ptr addrspace(1) %p, i8 %v) {
+; GFX942-SDAG-LABEL: global_store_b8_cluster:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: global_store_byte v[0:1], v2, off sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: global_store_b8_cluster:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: global_store_byte v[0:1], v2, off
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: global_store_b8_cluster:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: global_store_b8_cluster:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_store_b8 v[0:1], v2, off scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: global_store_b8_cluster:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: global_store_byte v[0:1], v2, off
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: global_store_b8_cluster:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: global_store_b8_cluster:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: global_store_byte v[0:1], v2, off sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: global_store_b8_cluster:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: global_store_byte v[0:1], v2, off
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: global_store_b8_cluster:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: global_store_b8_cluster:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: global_store_b8 v[0:1], v2, off scope:SCOPE_SE
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: global_store_b8_cluster:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: global_store_byte v[0:1], v2, off
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: global_store_b8_cluster:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ call void @llvm.amdgcn.av.store.b8.p1(ptr addrspace(1) %p, i8 %v, metadata !5)
+ ret void
+}
+
+; scope: wavefront
+define i8 @flat_load_b8_wavefront(ptr %p) {
+; GFX942-SDAG-LABEL: flat_load_b8_wavefront:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: flat_load_ubyte v0, v[0:1]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: flat_load_b8_wavefront:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: flat_load_ubyte v0, v[0:1]
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: flat_load_b8_wavefront:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: flat_load_d16_u8 v0, v[0:1]
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: flat_load_b8_wavefront:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: flat_load_b8_wavefront:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: flat_load_ubyte v0, v[0:1]
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: flat_load_b8_wavefront:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: flat_load_d16_u8 v0, v[0:1]
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: flat_load_b8_wavefront:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: flat_load_ubyte v0, v[0:1]
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: flat_load_b8_wavefront:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: flat_load_ubyte v0, v[0:1]
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: flat_load_b8_wavefront:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: flat_load_b8_wavefront:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: flat_load_b8_wavefront:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: flat_load_ubyte v0, v[0:1]
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: flat_load_b8_wavefront:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ %v = call i8 @llvm.amdgcn.av.load.b8.p0(ptr %p, metadata !0)
+ ret i8 %v
+}
+
+; scope: workgroup
+define i8 @flat_load_b8_workgroup(ptr %p) {
+; GFX942-SDAG-LABEL: flat_load_b8_workgroup:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: flat_load_ubyte v0, v[0:1] sc0
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: flat_load_b8_workgroup:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: flat_load_ubyte v0, v[0:1] glc
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: flat_load_b8_workgroup:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: flat_load_d16_u8 v0, v[0:1] glc
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: flat_load_b8_workgroup:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: flat_load_b8_workgroup:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: flat_load_ubyte v0, v[0:1] glc
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: flat_load_b8_workgroup:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: flat_load_d16_u8 v0, v[0:1] glc
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: flat_load_b8_workgroup:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: flat_load_ubyte v0, v[0:1] sc0
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: flat_load_b8_workgroup:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: flat_load_ubyte v0, v[0:1] glc
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: flat_load_b8_workgroup:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: flat_load_u8 v0, v[0:1] glc
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: flat_load_b8_workgroup:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: flat_load_b8_workgroup:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: flat_load_ubyte v0, v[0:1] glc
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: flat_load_b8_workgroup:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: flat_load_u8 v0, v[0:1] glc
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ %v = call i8 @llvm.amdgcn.av.load.b8.p0(ptr %p, metadata !1)
+ ret i8 %v
+}
+
+; scope: agent
+define i8 @flat_load_b8_agent(ptr %p) {
+; GFX942-SDAG-LABEL: flat_load_b8_agent:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: flat_load_ubyte v0, v[0:1] sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: flat_load_b8_agent:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: flat_load_ubyte v0, v[0:1] glc dlc
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: flat_load_b8_agent:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: flat_load_d16_u8 v0, v[0:1] glc
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: flat_load_b8_agent:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1] scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: flat_load_b8_agent:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: flat_load_ubyte v0, v[0:1] glc dlc
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: flat_load_b8_agent:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: flat_load_d16_u8 v0, v[0:1] glc
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: flat_load_b8_agent:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: flat_load_ubyte v0, v[0:1] sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: flat_load_b8_agent:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: flat_load_ubyte v0, v[0:1] glc dlc
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: flat_load_b8_agent:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: flat_load_u8 v0, v[0:1] glc
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: flat_load_b8_agent:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: flat_load_u8 v0, v[0:1] scope:SCOPE_DEV
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: flat_load_b8_agent:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: flat_load_ubyte v0, v[0:1] glc dlc
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: flat_load_b8_agent:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: flat_load_u8 v0, v[0:1] glc
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ %v = call i8 @llvm.amdgcn.av.load.b8.p0(ptr %p, metadata !2)
+ ret i8 %v
+}
+
+; scope: system
+define i8 @flat_load_b8_system(ptr %p) {
+; GFX942-SDAG-LABEL: flat_load_b8_system:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: flat_load_ubyte v0, v[0:1] sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: flat_load_b8_system:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: flat_load_ubyte v0, v[0:1] glc dlc
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: flat_load_b8_system:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: flat_load_d16_u8 v0, v[0:1] glc
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: flat_load_b8_system:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1] scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: flat_load_b8_system:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: flat_load_ubyte v0, v[0:1] glc dlc
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: flat_load_b8_system:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: flat_load_d16_u8 v0, v[0:1] glc
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: flat_load_b8_system:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: flat_load_ubyte v0, v[0:1] sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: flat_load_b8_system:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: flat_load_ubyte v0, v[0:1] glc dlc
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: flat_load_b8_system:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: flat_load_u8 v0, v[0:1] glc
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: flat_load_b8_system:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: flat_load_u8 v0, v[0:1] scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: flat_load_b8_system:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: flat_load_ubyte v0, v[0:1] glc dlc
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: flat_load_b8_system:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: flat_load_u8 v0, v[0:1] glc
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ %v = call i8 @llvm.amdgcn.av.load.b8.p0(ptr %p, metadata !3)
+ ret i8 %v
+}
+
+; scope: singlethread
+define i8 @flat_load_b8_singlethread(ptr %p) {
+; GFX942-SDAG-LABEL: flat_load_b8_singlethread:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: flat_load_ubyte v0, v[0:1]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: flat_load_b8_singlethread:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: flat_load_ubyte v0, v[0:1]
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: flat_load_b8_singlethread:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: flat_load_d16_u8 v0, v[0:1]
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: flat_load_b8_singlethread:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: flat_load_b8_singlethread:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: flat_load_ubyte v0, v[0:1]
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: flat_load_b8_singlethread:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: flat_load_d16_u8 v0, v[0:1]
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: flat_load_b8_singlethread:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: flat_load_ubyte v0, v[0:1]
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: flat_load_b8_singlethread:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: flat_load_ubyte v0, v[0:1]
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: flat_load_b8_singlethread:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: flat_load_b8_singlethread:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: flat_load_b8_singlethread:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: flat_load_ubyte v0, v[0:1]
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: flat_load_b8_singlethread:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ %v = call i8 @llvm.amdgcn.av.load.b8.p0(ptr %p, metadata !4)
+ ret i8 %v
+}
+
+; scope: cluster
+define i8 @flat_load_b8_cluster(ptr %p) {
+; GFX942-SDAG-LABEL: flat_load_b8_cluster:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: flat_load_ubyte v0, v[0:1] sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: flat_load_b8_cluster:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: flat_load_ubyte v0, v[0:1] glc dlc
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: flat_load_b8_cluster:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: flat_load_d16_u8 v0, v[0:1] glc
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: flat_load_b8_cluster:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: flat_load_b8_cluster:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: flat_load_ubyte v0, v[0:1] glc dlc
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: flat_load_b8_cluster:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: flat_load_d16_u8 v0, v[0:1] glc
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: flat_load_b8_cluster:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: flat_load_ubyte v0, v[0:1] sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: flat_load_b8_cluster:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: flat_load_ubyte v0, v[0:1] glc dlc
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: flat_load_b8_cluster:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: flat_load_u8 v0, v[0:1] glc
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: flat_load_b8_cluster:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: flat_load_u8 v0, v[0:1] scope:SCOPE_SE
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: flat_load_b8_cluster:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: flat_load_ubyte v0, v[0:1] glc dlc
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: flat_load_b8_cluster:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: flat_load_u8 v0, v[0:1] glc
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ %v = call i8 @llvm.amdgcn.av.load.b8.p0(ptr %p, metadata !5)
+ ret i8 %v
+}
+
+; scope: wavefront
+define void @flat_store_b8_wavefront(ptr %p, i8 %v) {
+; GFX942-SDAG-LABEL: flat_store_b8_wavefront:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: flat_store_byte v[0:1], v2
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: flat_store_b8_wavefront:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: flat_store_byte v[0:1], v2
+; GFX1012-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: flat_store_b8_wavefront:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: flat_store_b8 v[0:1], v2
+; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: flat_store_b8_wavefront:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: flat_store_b8 v[0:1], v2
+; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: flat_store_b8_wavefront:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: flat_store_byte v[0:1], v2
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: flat_store_b8_wavefront:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: flat_store_b8 v[0:1], v2
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: flat_store_b8_wavefront:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: flat_store_byte v[0:1], v2
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: flat_store_b8_wavefront:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: flat_store_byte v[0:1], v2
+; GFX1012-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: flat_store_b8_wavefront:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: flat_store_b8 v[0:1], v2
+; GFX1100-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: flat_store_b8_wavefront:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: flat_store_b8 v[0:1], v2
+; GFX1250-ISEL-NEXT: s_wait_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: flat_store_b8_wavefront:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: flat_store_byte v[0:1], v2
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: flat_store_b8_wavefront:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: flat_store_b8 v[0:1], v2
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ call void @llvm.amdgcn.av.store.b8.p0(ptr %p, i8 %v, metadata !0)
+ ret void
+}
+
+; scope: workgroup
+define void @flat_store_b8_workgroup(ptr %p, i8 %v) {
+; GFX942-SDAG-LABEL: flat_store_b8_workgroup:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: flat_store_byte v[0:1], v2 sc0
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: flat_store_b8_workgroup:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: flat_store_byte v[0:1], v2
+; GFX1012-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: flat_store_b8_workgroup:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: flat_store_b8 v[0:1], v2
+; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: flat_store_b8_workgroup:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: flat_store_b8 v[0:1], v2
+; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: flat_store_b8_workgroup:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: flat_store_byte v[0:1], v2
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: flat_store_b8_workgroup:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: flat_store_b8 v[0:1], v2
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: flat_store_b8_workgroup:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: flat_store_byte v[0:1], v2 sc0
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: flat_store_b8_workgroup:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: flat_store_byte v[0:1], v2
+; GFX1012-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: flat_store_b8_workgroup:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: flat_store_b8 v[0:1], v2
+; GFX1100-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: flat_store_b8_workgroup:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: flat_store_b8 v[0:1], v2
+; GFX1250-ISEL-NEXT: s_wait_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: flat_store_b8_workgroup:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: flat_store_byte v[0:1], v2
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: flat_store_b8_workgroup:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: flat_store_b8 v[0:1], v2
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ call void @llvm.amdgcn.av.store.b8.p0(ptr %p, i8 %v, metadata !1)
+ ret void
+}
+
+; scope: agent
+define void @flat_store_b8_agent(ptr %p, i8 %v) {
+; GFX942-SDAG-LABEL: flat_store_b8_agent:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: flat_store_byte v[0:1], v2 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: flat_store_b8_agent:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: flat_store_byte v[0:1], v2
+; GFX1012-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: flat_store_b8_agent:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: flat_store_b8 v[0:1], v2
+; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: flat_store_b8_agent:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: flat_store_b8 v[0:1], v2 scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: flat_store_b8_agent:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: flat_store_byte v[0:1], v2
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: flat_store_b8_agent:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: flat_store_b8 v[0:1], v2
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: flat_store_b8_agent:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: flat_store_byte v[0:1], v2 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: flat_store_b8_agent:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: flat_store_byte v[0:1], v2
+; GFX1012-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: flat_store_b8_agent:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: flat_store_b8 v[0:1], v2
+; GFX1100-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: flat_store_b8_agent:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: flat_store_b8 v[0:1], v2 scope:SCOPE_DEV
+; GFX1250-ISEL-NEXT: s_wait_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: flat_store_b8_agent:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: flat_store_byte v[0:1], v2
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: flat_store_b8_agent:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: flat_store_b8 v[0:1], v2
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ call void @llvm.amdgcn.av.store.b8.p0(ptr %p, i8 %v, metadata !2)
+ ret void
+}
+
+; scope: system
+define void @flat_store_b8_system(ptr %p, i8 %v) {
+; GFX942-SDAG-LABEL: flat_store_b8_system:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: flat_store_byte v[0:1], v2 sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: flat_store_b8_system:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: flat_store_byte v[0:1], v2
+; GFX1012-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: flat_store_b8_system:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: flat_store_b8 v[0:1], v2
+; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: flat_store_b8_system:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: flat_store_b8 v[0:1], v2 scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: flat_store_b8_system:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: flat_store_byte v[0:1], v2
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: flat_store_b8_system:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: flat_store_b8 v[0:1], v2
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: flat_store_b8_system:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: flat_store_byte v[0:1], v2 sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: flat_store_b8_system:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: flat_store_byte v[0:1], v2
+; GFX1012-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: flat_store_b8_system:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: flat_store_b8 v[0:1], v2
+; GFX1100-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: flat_store_b8_system:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: flat_store_b8 v[0:1], v2 scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_wait_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: flat_store_b8_system:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: flat_store_byte v[0:1], v2
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: flat_store_b8_system:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: flat_store_b8 v[0:1], v2
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ call void @llvm.amdgcn.av.store.b8.p0(ptr %p, i8 %v, metadata !3)
+ ret void
+}
+
+; scope: singlethread
+define void @flat_store_b8_singlethread(ptr %p, i8 %v) {
+; GFX942-SDAG-LABEL: flat_store_b8_singlethread:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: flat_store_byte v[0:1], v2
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: flat_store_b8_singlethread:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: flat_store_byte v[0:1], v2
+; GFX1012-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: flat_store_b8_singlethread:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: flat_store_b8 v[0:1], v2
+; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: flat_store_b8_singlethread:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: flat_store_b8 v[0:1], v2
+; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: flat_store_b8_singlethread:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: flat_store_byte v[0:1], v2
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: flat_store_b8_singlethread:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: flat_store_b8 v[0:1], v2
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: flat_store_b8_singlethread:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: flat_store_byte v[0:1], v2
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: flat_store_b8_singlethread:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: flat_store_byte v[0:1], v2
+; GFX1012-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: flat_store_b8_singlethread:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: flat_store_b8 v[0:1], v2
+; GFX1100-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: flat_store_b8_singlethread:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: flat_store_b8 v[0:1], v2
+; GFX1250-ISEL-NEXT: s_wait_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: flat_store_b8_singlethread:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: flat_store_byte v[0:1], v2
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: flat_store_b8_singlethread:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: flat_store_b8 v[0:1], v2
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ call void @llvm.amdgcn.av.store.b8.p0(ptr %p, i8 %v, metadata !4)
+ ret void
+}
+
+; scope: cluster
+define void @flat_store_b8_cluster(ptr %p, i8 %v) {
+; GFX942-SDAG-LABEL: flat_store_b8_cluster:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: flat_store_byte v[0:1], v2 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: flat_store_b8_cluster:
+; GFX1012-SDAG: ; %bb.0:
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: flat_store_byte v[0:1], v2
+; GFX1012-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: flat_store_b8_cluster:
+; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: flat_store_b8 v[0:1], v2
+; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: flat_store_b8_cluster:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: flat_store_b8 v[0:1], v2 scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-SDAG-LABEL: flat_store_b8_cluster:
+; GFX1012-WGP-SDAG: ; %bb.0:
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: flat_store_byte v[0:1], v2
+; GFX1012-WGP-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-SDAG-LABEL: flat_store_b8_cluster:
+; GFX1100-WGP-SDAG: ; %bb.0:
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: flat_store_b8 v[0:1], v2
+; GFX1100-WGP-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-WGP-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: flat_store_b8_cluster:
+; GFX942-ISEL: ; %bb.0:
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: flat_store_byte v[0:1], v2 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: flat_store_b8_cluster:
+; GFX1012-ISEL: ; %bb.0:
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: flat_store_byte v[0:1], v2
+; GFX1012-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: flat_store_b8_cluster:
+; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: flat_store_b8 v[0:1], v2
+; GFX1100-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: flat_store_b8_cluster:
+; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: flat_store_b8 v[0:1], v2 scope:SCOPE_SE
+; GFX1250-ISEL-NEXT: s_wait_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1012-WGP-ISEL-LABEL: flat_store_b8_cluster:
+; GFX1012-WGP-ISEL: ; %bb.0:
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: flat_store_byte v[0:1], v2
+; GFX1012-WGP-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1012-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-WGP-ISEL-LABEL: flat_store_b8_cluster:
+; GFX1100-WGP-ISEL: ; %bb.0:
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: flat_store_b8 v[0:1], v2
+; GFX1100-WGP-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
+ call void @llvm.amdgcn.av.store.b8.p0(ptr %p, i8 %v, metadata !5)
+ ret void
+}
+
; scope: system
define i16 @global_load_b16_system(ptr addrspace(1) %p) {
; GFX942-SDAG-LABEL: global_load_b16_system:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.load.b128.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.load.b128.ll
index 173ad8ce7463dd..1c25da5e68b5ae 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.load.b128.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.load.b128.ll
@@ -866,6 +866,218 @@ entry:
ret <4 x i32> %data
}
+;; vgpr pointer, system scope
+define i8 @av_global_load_b8_0_11(ptr addrspace(1) %addr) {
+; GFX906-SDAG-LABEL: av_global_load_b8_0_11:
+; GFX906-SDAG: ; %bb.0: ; %entry
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-SDAG-NEXT: global_load_ubyte v0, v[0:1], off glc
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX906-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-SDAG-LABEL: av_global_load_b8_0_11:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: global_load_ubyte v0, v[0:1], off sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: av_global_load_b8_0_11:
+; GFX1012-SDAG: ; %bb.0: ; %entry
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: global_load_ubyte v0, v[0:1], off glc dlc
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: av_global_load_b8_0_11:
+; GFX1100-SDAG: ; %bb.0: ; %entry
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: global_load_d16_u8 v0, v[0:1], off glc
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: av_global_load_b8_0_11:
+; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_load_u8 v0, v[0:1], off scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-SDAG-LABEL: av_global_load_b8_0_11:
+; GFX1310-SDAG: ; %bb.0: ; %entry
+; GFX1310-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1310-SDAG-NEXT: global_load_u8 v0, v[0:1], off scope:SCOPE_SYS
+; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX906-ISEL-LABEL: av_global_load_b8_0_11:
+; GFX906-ISEL: ; %bb.0: ; %entry
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-ISEL-NEXT: global_load_ubyte v0, v[0:1], off glc
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX906-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: av_global_load_b8_0_11:
+; GFX942-ISEL: ; %bb.0: ; %entry
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: global_load_ubyte v0, v[0:1], off sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: av_global_load_b8_0_11:
+; GFX1012-ISEL: ; %bb.0: ; %entry
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: global_load_ubyte v0, v[0:1], off glc dlc
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: av_global_load_b8_0_11:
+; GFX1100-ISEL: ; %bb.0: ; %entry
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: global_load_u8 v0, v[0:1], off glc
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: av_global_load_b8_0_11:
+; GFX1250-ISEL: ; %bb.0: ; %entry
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: global_load_u8 v0, v[0:1], off scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-ISEL-LABEL: av_global_load_b8_0_11:
+; GFX1310-ISEL: ; %bb.0: ; %entry
+; GFX1310-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_expcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1310-ISEL-NEXT: global_load_u8 v0, v[0:1], off scope:SCOPE_SYS
+; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %data = call i8 @llvm.amdgcn.av.load.b8.p1(ptr addrspace(1) %addr, metadata !3)
+ ret i8 %data
+}
+
+;; sgpr pointer, system scope
+define i8 @av_global_load_b8_saddr_0_03(ptr addrspace(1) inreg %addr) {
+; GFX906-SDAG-LABEL: av_global_load_b8_saddr_0_03:
+; GFX906-SDAG: ; %bb.0: ; %entry
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX906-SDAG-NEXT: global_load_ubyte v0, v0, s[16:17] glc
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX906-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-SDAG-LABEL: av_global_load_b8_saddr_0_03:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-SDAG-NEXT: global_load_ubyte v0, v0, s[0:1] sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: av_global_load_b8_saddr_0_03:
+; GFX1012-SDAG: ; %bb.0: ; %entry
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1012-SDAG-NEXT: global_load_ubyte v0, v0, s[16:17] glc dlc
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: av_global_load_b8_saddr_0_03:
+; GFX1100-SDAG: ; %bb.0: ; %entry
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1100-SDAG-NEXT: global_load_d16_u8 v0, v0, s[0:1] glc
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: av_global_load_b8_saddr_0_03:
+; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-SDAG-NEXT: global_load_u8 v0, v0, s[0:1] scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-SDAG-LABEL: av_global_load_b8_saddr_0_03:
+; GFX1310-SDAG: ; %bb.0: ; %entry
+; GFX1310-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1310-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1310-SDAG-NEXT: global_load_u8 v0, v0, s[0:1] scope:SCOPE_SYS
+; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX906-ISEL-LABEL: av_global_load_b8_saddr_0_03:
+; GFX906-ISEL: ; %bb.0: ; %entry
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX906-ISEL-NEXT: global_load_ubyte v0, v0, s[16:17] glc
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX906-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: av_global_load_b8_saddr_0_03:
+; GFX942-ISEL: ; %bb.0: ; %entry
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-ISEL-NEXT: global_load_ubyte v0, v0, s[0:1] sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: av_global_load_b8_saddr_0_03:
+; GFX1012-ISEL: ; %bb.0: ; %entry
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1012-ISEL-NEXT: global_load_ubyte v0, v0, s[16:17] glc dlc
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: av_global_load_b8_saddr_0_03:
+; GFX1100-ISEL: ; %bb.0: ; %entry
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1100-ISEL-NEXT: global_load_u8 v0, v0, s[0:1] glc
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: av_global_load_b8_saddr_0_03:
+; GFX1250-ISEL: ; %bb.0: ; %entry
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-ISEL-NEXT: global_load_u8 v0, v0, s[0:1] scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-ISEL-LABEL: av_global_load_b8_saddr_0_03:
+; GFX1310-ISEL: ; %bb.0: ; %entry
+; GFX1310-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_expcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1310-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1310-ISEL-NEXT: global_load_u8 v0, v0, s[0:1] scope:SCOPE_SYS
+; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %data = call i8 @llvm.amdgcn.av.load.b8.p1(ptr addrspace(1) %addr, metadata !3)
+ ret i8 %data
+}
+
;; vgpr pointer, system scope
define i16 @av_global_load_b16_0_11(ptr addrspace(1) %addr) {
; GFX906-SDAG-LABEL: av_global_load_b16_0_11:
@@ -8411,11 +8623,11 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX906-SDAG: ; %bb.0: ; %bb
; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX906-SDAG-NEXT: s_mov_b32 s4, -1
-; GFX906-SDAG-NEXT: .LBB66_1: ; %bb3
+; GFX906-SDAG-NEXT: .LBB68_1: ; %bb3
; GFX906-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX906-SDAG-NEXT: s_add_i32 s4, s4, 1
; GFX906-SDAG-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX906-SDAG-NEXT: s_cbranch_scc0 .LBB66_1
+; GFX906-SDAG-NEXT: s_cbranch_scc0 .LBB68_1
; GFX906-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX906-SDAG-NEXT: s_mov_b32 s5, 0
; GFX906-SDAG-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
@@ -8430,11 +8642,11 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX942-SDAG: ; %bb.0: ; %bb
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_mov_b32 s0, -1
-; GFX942-SDAG-NEXT: .LBB66_1: ; %bb3
+; GFX942-SDAG-NEXT: .LBB68_1: ; %bb3
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_i32 s0, s0, 1
; GFX942-SDAG-NEXT: s_cmpk_eq_i32 s0, 0xff
-; GFX942-SDAG-NEXT: s_cbranch_scc0 .LBB66_1
+; GFX942-SDAG-NEXT: s_cbranch_scc0 .LBB68_1
; GFX942-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX942-SDAG-NEXT: s_mov_b32 s1, 0
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[0:1], 2, v[0:1]
@@ -8446,11 +8658,11 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX1012-SDAG: ; %bb.0: ; %bb
; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1012-SDAG-NEXT: s_mov_b32 s4, -1
-; GFX1012-SDAG-NEXT: .LBB66_1: ; %bb3
+; GFX1012-SDAG-NEXT: .LBB68_1: ; %bb3
; GFX1012-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1012-SDAG-NEXT: s_add_i32 s4, s4, 1
; GFX1012-SDAG-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX1012-SDAG-NEXT: s_cbranch_scc0 .LBB66_1
+; GFX1012-SDAG-NEXT: s_cbranch_scc0 .LBB68_1
; GFX1012-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1012-SDAG-NEXT: s_mov_b32 s5, 0
; GFX1012-SDAG-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
@@ -8464,12 +8676,12 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX1100-SDAG: ; %bb.0: ; %bb
; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-SDAG-NEXT: s_mov_b32 s0, -1
-; GFX1100-SDAG-NEXT: .LBB66_1: ; %bb3
+; GFX1100-SDAG-NEXT: .LBB68_1: ; %bb3
; GFX1100-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1100-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1100-SDAG-NEXT: s_add_i32 s0, s0, 1
; GFX1100-SDAG-NEXT: s_cmpk_eq_i32 s0, 0xff
-; GFX1100-SDAG-NEXT: s_cbranch_scc0 .LBB66_1
+; GFX1100-SDAG-NEXT: s_cbranch_scc0 .LBB68_1
; GFX1100-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1100-SDAG-NEXT: s_mov_b32 s1, 0
; GFX1100-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -8486,12 +8698,12 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: s_mov_b32 s0, -1
-; GFX1250-SDAG-NEXT: .LBB66_1: ; %bb3
+; GFX1250-SDAG-NEXT: .LBB68_1: ; %bb3
; GFX1250-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-NEXT: s_add_co_i32 s0, s0, 1
; GFX1250-SDAG-NEXT: s_cmp_eq_u32 s0, 0xff
-; GFX1250-SDAG-NEXT: s_cbranch_scc0 .LBB66_1
+; GFX1250-SDAG-NEXT: s_cbranch_scc0 .LBB68_1
; GFX1250-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1250-SDAG-NEXT: s_mov_b32 s1, 0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -8508,12 +8720,12 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1310-SDAG-NEXT: s_mov_b32 s0, -1
-; GFX1310-SDAG-NEXT: .LBB66_1: ; %bb3
+; GFX1310-SDAG-NEXT: .LBB68_1: ; %bb3
; GFX1310-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1310-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1310-SDAG-NEXT: s_add_co_i32 s0, s0, 1
; GFX1310-SDAG-NEXT: s_cmp_eq_u32 s0, 0xff
-; GFX1310-SDAG-NEXT: s_cbranch_scc0 .LBB66_1
+; GFX1310-SDAG-NEXT: s_cbranch_scc0 .LBB68_1
; GFX1310-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1310-SDAG-NEXT: s_mov_b32 s1, 0
; GFX1310-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -8529,11 +8741,11 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX906-ISEL: ; %bb.0: ; %bb
; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX906-ISEL-NEXT: s_mov_b32 s4, -1
-; GFX906-ISEL-NEXT: .LBB66_1: ; %bb3
+; GFX906-ISEL-NEXT: .LBB68_1: ; %bb3
; GFX906-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX906-ISEL-NEXT: s_add_i32 s4, s4, 1
; GFX906-ISEL-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX906-ISEL-NEXT: s_cbranch_scc0 .LBB66_1
+; GFX906-ISEL-NEXT: s_cbranch_scc0 .LBB68_1
; GFX906-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX906-ISEL-NEXT: s_mov_b32 s5, 0
; GFX906-ISEL-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
@@ -8549,11 +8761,11 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX942-ISEL: ; %bb.0: ; %bb
; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-ISEL-NEXT: s_mov_b32 s0, -1
-; GFX942-ISEL-NEXT: .LBB66_1: ; %bb3
+; GFX942-ISEL-NEXT: .LBB68_1: ; %bb3
; GFX942-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-ISEL-NEXT: s_add_i32 s0, s0, 1
; GFX942-ISEL-NEXT: s_cmpk_eq_i32 s0, 0xff
-; GFX942-ISEL-NEXT: s_cbranch_scc0 .LBB66_1
+; GFX942-ISEL-NEXT: s_cbranch_scc0 .LBB68_1
; GFX942-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX942-ISEL-NEXT: s_mov_b32 s1, 0
; GFX942-ISEL-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
@@ -8569,11 +8781,11 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX1012-ISEL: ; %bb.0: ; %bb
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1012-ISEL-NEXT: s_mov_b32 s4, -1
-; GFX1012-ISEL-NEXT: .LBB66_1: ; %bb3
+; GFX1012-ISEL-NEXT: .LBB68_1: ; %bb3
; GFX1012-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1012-ISEL-NEXT: s_add_i32 s4, s4, 1
; GFX1012-ISEL-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX1012-ISEL-NEXT: s_cbranch_scc0 .LBB66_1
+; GFX1012-ISEL-NEXT: s_cbranch_scc0 .LBB68_1
; GFX1012-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1012-ISEL-NEXT: s_mov_b32 s5, 0
; GFX1012-ISEL-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
@@ -8589,12 +8801,12 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX1100-ISEL: ; %bb.0: ; %bb
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-ISEL-NEXT: s_mov_b32 s0, -1
-; GFX1100-ISEL-NEXT: .LBB66_1: ; %bb3
+; GFX1100-ISEL-NEXT: .LBB68_1: ; %bb3
; GFX1100-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1100-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1100-ISEL-NEXT: s_add_i32 s0, s0, 1
; GFX1100-ISEL-NEXT: s_cmpk_eq_i32 s0, 0xff
-; GFX1100-ISEL-NEXT: s_cbranch_scc0 .LBB66_1
+; GFX1100-ISEL-NEXT: s_cbranch_scc0 .LBB68_1
; GFX1100-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1100-ISEL-NEXT: s_mov_b32 s1, 0
; GFX1100-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -8612,12 +8824,12 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-ISEL-NEXT: s_mov_b32 s0, -1
-; GFX1250-ISEL-NEXT: .LBB66_1: ; %bb3
+; GFX1250-ISEL-NEXT: .LBB68_1: ; %bb3
; GFX1250-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-ISEL-NEXT: s_add_co_i32 s0, s0, 1
; GFX1250-ISEL-NEXT: s_cmp_eq_u32 s0, 0xff
-; GFX1250-ISEL-NEXT: s_cbranch_scc0 .LBB66_1
+; GFX1250-ISEL-NEXT: s_cbranch_scc0 .LBB68_1
; GFX1250-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1250-ISEL-NEXT: s_mov_b32 s1, 0
; GFX1250-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -8638,12 +8850,12 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
; GFX1310-ISEL-NEXT: s_mov_b32 s0, -1
-; GFX1310-ISEL-NEXT: .LBB66_1: ; %bb3
+; GFX1310-ISEL-NEXT: .LBB68_1: ; %bb3
; GFX1310-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1310-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1310-ISEL-NEXT: s_add_co_i32 s0, s0, 1
; GFX1310-ISEL-NEXT: s_cmp_eq_u32 s0, 0xff
-; GFX1310-ISEL-NEXT: s_cbranch_scc0 .LBB66_1
+; GFX1310-ISEL-NEXT: s_cbranch_scc0 .LBB68_1
; GFX1310-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1310-ISEL-NEXT: s_mov_b32 s1, 0
; GFX1310-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -8680,11 +8892,11 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX906-SDAG: ; %bb.0: ; %bb
; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX906-SDAG-NEXT: s_mov_b32 s4, -1
-; GFX906-SDAG-NEXT: .LBB67_1: ; %bb5
+; GFX906-SDAG-NEXT: .LBB69_1: ; %bb5
; GFX906-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX906-SDAG-NEXT: s_add_i32 s4, s4, 1
; GFX906-SDAG-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX906-SDAG-NEXT: s_cbranch_scc0 .LBB67_1
+; GFX906-SDAG-NEXT: s_cbranch_scc0 .LBB69_1
; GFX906-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX906-SDAG-NEXT: s_mov_b32 s5, 0
; GFX906-SDAG-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
@@ -8699,11 +8911,11 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX942-SDAG: ; %bb.0: ; %bb
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_mov_b32 s0, -1
-; GFX942-SDAG-NEXT: .LBB67_1: ; %bb5
+; GFX942-SDAG-NEXT: .LBB69_1: ; %bb5
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_i32 s0, s0, 1
; GFX942-SDAG-NEXT: s_cmpk_eq_i32 s0, 0xff
-; GFX942-SDAG-NEXT: s_cbranch_scc0 .LBB67_1
+; GFX942-SDAG-NEXT: s_cbranch_scc0 .LBB69_1
; GFX942-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX942-SDAG-NEXT: s_mov_b32 s1, 0
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[0:1], 2, v[0:1]
@@ -8715,11 +8927,11 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX1012-SDAG: ; %bb.0: ; %bb
; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1012-SDAG-NEXT: s_mov_b32 s4, -1
-; GFX1012-SDAG-NEXT: .LBB67_1: ; %bb5
+; GFX1012-SDAG-NEXT: .LBB69_1: ; %bb5
; GFX1012-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1012-SDAG-NEXT: s_add_i32 s4, s4, 1
; GFX1012-SDAG-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX1012-SDAG-NEXT: s_cbranch_scc0 .LBB67_1
+; GFX1012-SDAG-NEXT: s_cbranch_scc0 .LBB69_1
; GFX1012-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1012-SDAG-NEXT: s_mov_b32 s5, 0
; GFX1012-SDAG-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
@@ -8733,12 +8945,12 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX1100-SDAG: ; %bb.0: ; %bb
; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-SDAG-NEXT: s_mov_b32 s0, -1
-; GFX1100-SDAG-NEXT: .LBB67_1: ; %bb5
+; GFX1100-SDAG-NEXT: .LBB69_1: ; %bb5
; GFX1100-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1100-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1100-SDAG-NEXT: s_add_i32 s0, s0, 1
; GFX1100-SDAG-NEXT: s_cmpk_eq_i32 s0, 0xff
-; GFX1100-SDAG-NEXT: s_cbranch_scc0 .LBB67_1
+; GFX1100-SDAG-NEXT: s_cbranch_scc0 .LBB69_1
; GFX1100-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1100-SDAG-NEXT: s_mov_b32 s1, 0
; GFX1100-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -8755,12 +8967,12 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: s_mov_b32 s0, -1
-; GFX1250-SDAG-NEXT: .LBB67_1: ; %bb5
+; GFX1250-SDAG-NEXT: .LBB69_1: ; %bb5
; GFX1250-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-NEXT: s_add_co_i32 s0, s0, 1
; GFX1250-SDAG-NEXT: s_cmp_eq_u32 s0, 0xff
-; GFX1250-SDAG-NEXT: s_cbranch_scc0 .LBB67_1
+; GFX1250-SDAG-NEXT: s_cbranch_scc0 .LBB69_1
; GFX1250-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1250-SDAG-NEXT: s_mov_b32 s1, 0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -8777,12 +8989,12 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1310-SDAG-NEXT: s_mov_b32 s0, -1
-; GFX1310-SDAG-NEXT: .LBB67_1: ; %bb5
+; GFX1310-SDAG-NEXT: .LBB69_1: ; %bb5
; GFX1310-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1310-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1310-SDAG-NEXT: s_add_co_i32 s0, s0, 1
; GFX1310-SDAG-NEXT: s_cmp_eq_u32 s0, 0xff
-; GFX1310-SDAG-NEXT: s_cbranch_scc0 .LBB67_1
+; GFX1310-SDAG-NEXT: s_cbranch_scc0 .LBB69_1
; GFX1310-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1310-SDAG-NEXT: s_mov_b32 s1, 0
; GFX1310-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -8798,11 +9010,11 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX906-ISEL: ; %bb.0: ; %bb
; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX906-ISEL-NEXT: s_mov_b32 s4, -1
-; GFX906-ISEL-NEXT: .LBB67_1: ; %bb5
+; GFX906-ISEL-NEXT: .LBB69_1: ; %bb5
; GFX906-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX906-ISEL-NEXT: s_add_i32 s4, s4, 1
; GFX906-ISEL-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX906-ISEL-NEXT: s_cbranch_scc0 .LBB67_1
+; GFX906-ISEL-NEXT: s_cbranch_scc0 .LBB69_1
; GFX906-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX906-ISEL-NEXT: s_mov_b32 s5, 0
; GFX906-ISEL-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
@@ -8818,11 +9030,11 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX942-ISEL: ; %bb.0: ; %bb
; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-ISEL-NEXT: s_mov_b32 s0, -1
-; GFX942-ISEL-NEXT: .LBB67_1: ; %bb5
+; GFX942-ISEL-NEXT: .LBB69_1: ; %bb5
; GFX942-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-ISEL-NEXT: s_add_i32 s0, s0, 1
; GFX942-ISEL-NEXT: s_cmpk_eq_i32 s0, 0xff
-; GFX942-ISEL-NEXT: s_cbranch_scc0 .LBB67_1
+; GFX942-ISEL-NEXT: s_cbranch_scc0 .LBB69_1
; GFX942-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX942-ISEL-NEXT: s_mov_b32 s1, 0
; GFX942-ISEL-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
@@ -8838,11 +9050,11 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX1012-ISEL: ; %bb.0: ; %bb
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1012-ISEL-NEXT: s_mov_b32 s4, -1
-; GFX1012-ISEL-NEXT: .LBB67_1: ; %bb5
+; GFX1012-ISEL-NEXT: .LBB69_1: ; %bb5
; GFX1012-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1012-ISEL-NEXT: s_add_i32 s4, s4, 1
; GFX1012-ISEL-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX1012-ISEL-NEXT: s_cbranch_scc0 .LBB67_1
+; GFX1012-ISEL-NEXT: s_cbranch_scc0 .LBB69_1
; GFX1012-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1012-ISEL-NEXT: s_mov_b32 s5, 0
; GFX1012-ISEL-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
@@ -8858,12 +9070,12 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX1100-ISEL: ; %bb.0: ; %bb
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-ISEL-NEXT: s_mov_b32 s0, -1
-; GFX1100-ISEL-NEXT: .LBB67_1: ; %bb5
+; GFX1100-ISEL-NEXT: .LBB69_1: ; %bb5
; GFX1100-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1100-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1100-ISEL-NEXT: s_add_i32 s0, s0, 1
; GFX1100-ISEL-NEXT: s_cmpk_eq_i32 s0, 0xff
-; GFX1100-ISEL-NEXT: s_cbranch_scc0 .LBB67_1
+; GFX1100-ISEL-NEXT: s_cbranch_scc0 .LBB69_1
; GFX1100-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1100-ISEL-NEXT: s_mov_b32 s1, 0
; GFX1100-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -8881,12 +9093,12 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-ISEL-NEXT: s_mov_b32 s0, -1
-; GFX1250-ISEL-NEXT: .LBB67_1: ; %bb5
+; GFX1250-ISEL-NEXT: .LBB69_1: ; %bb5
; GFX1250-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-ISEL-NEXT: s_add_co_i32 s0, s0, 1
; GFX1250-ISEL-NEXT: s_cmp_eq_u32 s0, 0xff
-; GFX1250-ISEL-NEXT: s_cbranch_scc0 .LBB67_1
+; GFX1250-ISEL-NEXT: s_cbranch_scc0 .LBB69_1
; GFX1250-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1250-ISEL-NEXT: s_mov_b32 s1, 0
; GFX1250-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -8907,12 +9119,12 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
; GFX1310-ISEL-NEXT: s_mov_b32 s0, -1
-; GFX1310-ISEL-NEXT: .LBB67_1: ; %bb5
+; GFX1310-ISEL-NEXT: .LBB69_1: ; %bb5
; GFX1310-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1310-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1310-ISEL-NEXT: s_add_co_i32 s0, s0, 1
; GFX1310-ISEL-NEXT: s_cmp_eq_u32 s0, 0xff
-; GFX1310-ISEL-NEXT: s_cbranch_scc0 .LBB67_1
+; GFX1310-ISEL-NEXT: s_cbranch_scc0 .LBB69_1
; GFX1310-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1310-ISEL-NEXT: s_mov_b32 s1, 0
; GFX1310-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -17085,11 +17297,11 @@ define <4 x float> @global_saddr_64bit_lsr_iv(ptr addrspace(1) inreg %arg) {
; GFX906-SDAG: ; %bb.0: ; %bb
; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX906-SDAG-NEXT: s_mov_b32 s4, -1
-; GFX906-SDAG-NEXT: .LBB120_1: ; %bb3
+; GFX906-SDAG-NEXT: .LBB122_1: ; %bb3
; GFX906-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX906-SDAG-NEXT: s_add_i32 s4, s4, 1
; GFX906-SDAG-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX906-SDAG-NEXT: s_cbranch_scc0 .LBB120_1
+; GFX906-SDAG-NEXT: s_cbranch_scc0 .LBB122_1
; GFX906-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX906-SDAG-NEXT: s_mov_b32 s5, 0
; GFX906-SDAG-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
@@ -17104,11 +17316,11 @@ define <4 x float> @global_saddr_64bit_lsr_iv(ptr addrspace(1) inreg %arg) {
; GFX942-SDAG: ; %bb.0: ; %bb
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_mov_b32 s2, -1
-; GFX942-SDAG-NEXT: .LBB120_1: ; %bb3
+; GFX942-SDAG-NEXT: .LBB122_1: ; %bb3
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_i32 s2, s2, 1
; GFX942-SDAG-NEXT: s_cmpk_eq_i32 s2, 0xff
-; GFX942-SDAG-NEXT: s_cbranch_scc0 .LBB120_1
+; GFX942-SDAG-NEXT: s_cbranch_scc0 .LBB122_1
; GFX942-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX942-SDAG-NEXT: s_mov_b32 s3, 0
; GFX942-SDAG-NEXT: s_lshl_b64 s[2:3], s[2:3], 2
@@ -17123,11 +17335,11 @@ define <4 x float> @global_saddr_64bit_lsr_iv(ptr addrspace(1) inreg %arg) {
; GFX1012-SDAG: ; %bb.0: ; %bb
; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1012-SDAG-NEXT: s_mov_b32 s4, -1
-; GFX1012-SDAG-NEXT: .LBB120_1: ; %bb3
+; GFX1012-SDAG-NEXT: .LBB122_1: ; %bb3
; GFX1012-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1012-SDAG-NEXT: s_add_i32 s4, s4, 1
; GFX1012-SDAG-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX1012-SDAG-NEXT: s_cbranch_scc0 .LBB120_1
+; GFX1012-SDAG-NEXT: s_cbranch_scc0 .LBB122_1
; GFX1012-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1012-SDAG-NEXT: s_mov_b32 s5, 0
; GFX1012-SDAG-NEXT: v_mov_b32_e32 v0, 0
@@ -17142,12 +17354,12 @@ define <4 x float> @global_saddr_64bit_lsr_iv(ptr addrspace(1) inreg %arg) {
; GFX1100-SDAG: ; %bb.0: ; %bb
; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-SDAG-NEXT: s_mov_b32 s2, -1
-; GFX1100-SDAG-NEXT: .LBB120_1: ; %bb3
+; GFX1100-SDAG-NEXT: .LBB122_1: ; %bb3
; GFX1100-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1100-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1100-SDAG-NEXT: s_add_i32 s2, s2, 1
; GFX1100-SDAG-NEXT: s_cmpk_eq_i32 s2, 0xff
-; GFX1100-SDAG-NEXT: s_cbranch_scc0 .LBB120_1
+; GFX1100-SDAG-NEXT: s_cbranch_scc0 .LBB122_1
; GFX1100-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1100-SDAG-NEXT: s_mov_b32 s3, 0
; GFX1100-SDAG-NEXT: v_mov_b32_e32 v0, 0
@@ -17164,12 +17376,12 @@ define <4 x float> @global_saddr_64bit_lsr_iv(ptr addrspace(1) inreg %arg) {
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: s_mov_b32 s2, -1
-; GFX1250-SDAG-NEXT: .LBB120_1: ; %bb3
+; GFX1250-SDAG-NEXT: .LBB122_1: ; %bb3
; GFX1250-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-NEXT: s_add_co_i32 s2, s2, 1
; GFX1250-SDAG-NEXT: s_cmp_eq_u32 s2, 0xff
-; GFX1250-SDAG-NEXT: s_cbranch_scc0 .LBB120_1
+; GFX1250-SDAG-NEXT: s_cbranch_scc0 .LBB122_1
; GFX1250-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1250-SDAG-NEXT: s_mov_b32 s3, 0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
@@ -17188,12 +17400,12 @@ define <4 x float> @global_saddr_64bit_lsr_iv(ptr addrspace(1) inreg %arg) {
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1310-SDAG-NEXT: s_mov_b32 s2, -1
-; GFX1310-SDAG-NEXT: .LBB120_1: ; %bb3
+; GFX1310-SDAG-NEXT: .LBB122_1: ; %bb3
; GFX1310-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1310-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1310-SDAG-NEXT: s_add_co_i32 s2, s2, 1
; GFX1310-SDAG-NEXT: s_cmp_eq_u32 s2, 0xff
-; GFX1310-SDAG-NEXT: s_cbranch_scc0 .LBB120_1
+; GFX1310-SDAG-NEXT: s_cbranch_scc0 .LBB122_1
; GFX1310-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1310-SDAG-NEXT: s_mov_b32 s3, 0
; GFX1310-SDAG-NEXT: v_mov_b32_e32 v0, 0
@@ -17208,11 +17420,11 @@ define <4 x float> @global_saddr_64bit_lsr_iv(ptr addrspace(1) inreg %arg) {
; GFX906-ISEL: ; %bb.0: ; %bb
; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX906-ISEL-NEXT: s_mov_b32 s4, -1
-; GFX906-ISEL-NEXT: .LBB120_1: ; %bb3
+; GFX906-ISEL-NEXT: .LBB122_1: ; %bb3
; GFX906-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX906-ISEL-NEXT: s_add_i32 s4, s4, 1
; GFX906-ISEL-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX906-ISEL-NEXT: s_cbranch_scc0 .LBB120_1
+; GFX906-ISEL-NEXT: s_cbranch_scc0 .LBB122_1
; GFX906-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX906-ISEL-NEXT: s_mov_b32 s5, 0
; GFX906-ISEL-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
@@ -17235,11 +17447,11 @@ define <4 x float> @global_saddr_64bit_lsr_iv(ptr addrspace(1) inreg %arg) {
; GFX942-ISEL: ; %bb.0: ; %bb
; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-ISEL-NEXT: s_mov_b32 s2, -1
-; GFX942-ISEL-NEXT: .LBB120_1: ; %bb3
+; GFX942-ISEL-NEXT: .LBB122_1: ; %bb3
; GFX942-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-ISEL-NEXT: s_add_i32 s2, s2, 1
; GFX942-ISEL-NEXT: s_cmpk_eq_i32 s2, 0xff
-; GFX942-ISEL-NEXT: s_cbranch_scc0 .LBB120_1
+; GFX942-ISEL-NEXT: s_cbranch_scc0 .LBB122_1
; GFX942-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX942-ISEL-NEXT: s_mov_b32 s3, 0
; GFX942-ISEL-NEXT: s_lshl_b64 s[2:3], s[2:3], 2
@@ -17262,11 +17474,11 @@ define <4 x float> @global_saddr_64bit_lsr_iv(ptr addrspace(1) inreg %arg) {
; GFX1012-ISEL: ; %bb.0: ; %bb
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1012-ISEL-NEXT: s_mov_b32 s4, -1
-; GFX1012-ISEL-NEXT: .LBB120_1: ; %bb3
+; GFX1012-ISEL-NEXT: .LBB122_1: ; %bb3
; GFX1012-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1012-ISEL-NEXT: s_add_i32 s4, s4, 1
; GFX1012-ISEL-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX1012-ISEL-NEXT: s_cbranch_scc0 .LBB120_1
+; GFX1012-ISEL-NEXT: s_cbranch_scc0 .LBB122_1
; GFX1012-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1012-ISEL-NEXT: s_mov_b32 s5, 0
; GFX1012-ISEL-NEXT: v_mov_b32_e32 v0, 0
@@ -17289,12 +17501,12 @@ define <4 x float> @global_saddr_64bit_lsr_iv(ptr addrspace(1) inreg %arg) {
; GFX1100-ISEL: ; %bb.0: ; %bb
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-ISEL-NEXT: s_mov_b32 s2, -1
-; GFX1100-ISEL-NEXT: .LBB120_1: ; %bb3
+; GFX1100-ISEL-NEXT: .LBB122_1: ; %bb3
; GFX1100-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1100-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1100-ISEL-NEXT: s_add_i32 s2, s2, 1
; GFX1100-ISEL-NEXT: s_cmpk_eq_i32 s2, 0xff
-; GFX1100-ISEL-NEXT: s_cbranch_scc0 .LBB120_1
+; GFX1100-ISEL-NEXT: s_cbranch_scc0 .LBB122_1
; GFX1100-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1100-ISEL-NEXT: s_mov_b32 s3, 0
; GFX1100-ISEL-NEXT: v_mov_b32_e32 v0, 0
@@ -17318,12 +17530,12 @@ define <4 x float> @global_saddr_64bit_lsr_iv(ptr addrspace(1) inreg %arg) {
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-ISEL-NEXT: s_mov_b32 s2, -1
-; GFX1250-ISEL-NEXT: .LBB120_1: ; %bb3
+; GFX1250-ISEL-NEXT: .LBB122_1: ; %bb3
; GFX1250-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-ISEL-NEXT: s_add_co_i32 s2, s2, 1
; GFX1250-ISEL-NEXT: s_cmp_eq_u32 s2, 0xff
-; GFX1250-ISEL-NEXT: s_cbranch_scc0 .LBB120_1
+; GFX1250-ISEL-NEXT: s_cbranch_scc0 .LBB122_1
; GFX1250-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1250-ISEL-NEXT: s_mov_b32 s3, 0
; GFX1250-ISEL-NEXT: v_mov_b32_e32 v0, 0
@@ -17351,12 +17563,12 @@ define <4 x float> @global_saddr_64bit_lsr_iv(ptr addrspace(1) inreg %arg) {
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
; GFX1310-ISEL-NEXT: s_mov_b32 s2, -1
-; GFX1310-ISEL-NEXT: .LBB120_1: ; %bb3
+; GFX1310-ISEL-NEXT: .LBB122_1: ; %bb3
; GFX1310-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1310-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1310-ISEL-NEXT: s_add_co_i32 s2, s2, 1
; GFX1310-ISEL-NEXT: s_cmp_eq_u32 s2, 0xff
-; GFX1310-ISEL-NEXT: s_cbranch_scc0 .LBB120_1
+; GFX1310-ISEL-NEXT: s_cbranch_scc0 .LBB122_1
; GFX1310-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1310-ISEL-NEXT: s_mov_b32 s3, 0
; GFX1310-ISEL-NEXT: v_mov_b32_e32 v0, 0
@@ -17399,11 +17611,11 @@ define <4 x float> @global_saddr_64bit_lsr_iv_multiload(ptr addrspace(1) inreg %
; GFX906-SDAG: ; %bb.0: ; %bb
; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX906-SDAG-NEXT: s_mov_b32 s4, -1
-; GFX906-SDAG-NEXT: .LBB121_1: ; %bb5
+; GFX906-SDAG-NEXT: .LBB123_1: ; %bb5
; GFX906-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX906-SDAG-NEXT: s_add_i32 s4, s4, 1
; GFX906-SDAG-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX906-SDAG-NEXT: s_cbranch_scc0 .LBB121_1
+; GFX906-SDAG-NEXT: s_cbranch_scc0 .LBB123_1
; GFX906-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX906-SDAG-NEXT: s_mov_b32 s5, 0
; GFX906-SDAG-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
@@ -17418,11 +17630,11 @@ define <4 x float> @global_saddr_64bit_lsr_iv_multiload(ptr addrspace(1) inreg %
; GFX942-SDAG: ; %bb.0: ; %bb
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_mov_b32 s2, -1
-; GFX942-SDAG-NEXT: .LBB121_1: ; %bb5
+; GFX942-SDAG-NEXT: .LBB123_1: ; %bb5
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_i32 s2, s2, 1
; GFX942-SDAG-NEXT: s_cmpk_eq_i32 s2, 0xff
-; GFX942-SDAG-NEXT: s_cbranch_scc0 .LBB121_1
+; GFX942-SDAG-NEXT: s_cbranch_scc0 .LBB123_1
; GFX942-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX942-SDAG-NEXT: s_mov_b32 s3, 0
; GFX942-SDAG-NEXT: s_lshl_b64 s[2:3], s[2:3], 2
@@ -17437,11 +17649,11 @@ define <4 x float> @global_saddr_64bit_lsr_iv_multiload(ptr addrspace(1) inreg %
; GFX1012-SDAG: ; %bb.0: ; %bb
; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1012-SDAG-NEXT: s_mov_b32 s4, -1
-; GFX1012-SDAG-NEXT: .LBB121_1: ; %bb5
+; GFX1012-SDAG-NEXT: .LBB123_1: ; %bb5
; GFX1012-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1012-SDAG-NEXT: s_add_i32 s4, s4, 1
; GFX1012-SDAG-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX1012-SDAG-NEXT: s_cbranch_scc0 .LBB121_1
+; GFX1012-SDAG-NEXT: s_cbranch_scc0 .LBB123_1
; GFX1012-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1012-SDAG-NEXT: s_mov_b32 s5, 0
; GFX1012-SDAG-NEXT: v_mov_b32_e32 v0, 0
@@ -17456,12 +17668,12 @@ define <4 x float> @global_saddr_64bit_lsr_iv_multiload(ptr addrspace(1) inreg %
; GFX1100-SDAG: ; %bb.0: ; %bb
; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-SDAG-NEXT: s_mov_b32 s2, -1
-; GFX1100-SDAG-NEXT: .LBB121_1: ; %bb5
+; GFX1100-SDAG-NEXT: .LBB123_1: ; %bb5
; GFX1100-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1100-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1100-SDAG-NEXT: s_add_i32 s2, s2, 1
; GFX1100-SDAG-NEXT: s_cmpk_eq_i32 s2, 0xff
-; GFX1100-SDAG-NEXT: s_cbranch_scc0 .LBB121_1
+; GFX1100-SDAG-NEXT: s_cbranch_scc0 .LBB123_1
; GFX1100-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1100-SDAG-NEXT: s_mov_b32 s3, 0
; GFX1100-SDAG-NEXT: v_mov_b32_e32 v0, 0
@@ -17478,12 +17690,12 @@ define <4 x float> @global_saddr_64bit_lsr_iv_multiload(ptr addrspace(1) inreg %
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: s_mov_b32 s2, -1
-; GFX1250-SDAG-NEXT: .LBB121_1: ; %bb5
+; GFX1250-SDAG-NEXT: .LBB123_1: ; %bb5
; GFX1250-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-NEXT: s_add_co_i32 s2, s2, 1
; GFX1250-SDAG-NEXT: s_cmp_eq_u32 s2, 0xff
-; GFX1250-SDAG-NEXT: s_cbranch_scc0 .LBB121_1
+; GFX1250-SDAG-NEXT: s_cbranch_scc0 .LBB123_1
; GFX1250-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1250-SDAG-NEXT: s_mov_b32 s3, 0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
@@ -17502,12 +17714,12 @@ define <4 x float> @global_saddr_64bit_lsr_iv_multiload(ptr addrspace(1) inreg %
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1310-SDAG-NEXT: s_mov_b32 s2, -1
-; GFX1310-SDAG-NEXT: .LBB121_1: ; %bb5
+; GFX1310-SDAG-NEXT: .LBB123_1: ; %bb5
; GFX1310-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1310-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1310-SDAG-NEXT: s_add_co_i32 s2, s2, 1
; GFX1310-SDAG-NEXT: s_cmp_eq_u32 s2, 0xff
-; GFX1310-SDAG-NEXT: s_cbranch_scc0 .LBB121_1
+; GFX1310-SDAG-NEXT: s_cbranch_scc0 .LBB123_1
; GFX1310-SDAG-NEXT: ; %bb.2: ; %bb2
; GFX1310-SDAG-NEXT: s_mov_b32 s3, 0
; GFX1310-SDAG-NEXT: v_mov_b32_e32 v0, 0
@@ -17522,11 +17734,11 @@ define <4 x float> @global_saddr_64bit_lsr_iv_multiload(ptr addrspace(1) inreg %
; GFX906-ISEL: ; %bb.0: ; %bb
; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX906-ISEL-NEXT: s_mov_b32 s4, -1
-; GFX906-ISEL-NEXT: .LBB121_1: ; %bb5
+; GFX906-ISEL-NEXT: .LBB123_1: ; %bb5
; GFX906-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX906-ISEL-NEXT: s_add_i32 s4, s4, 1
; GFX906-ISEL-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX906-ISEL-NEXT: s_cbranch_scc0 .LBB121_1
+; GFX906-ISEL-NEXT: s_cbranch_scc0 .LBB123_1
; GFX906-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX906-ISEL-NEXT: s_mov_b32 s5, 0
; GFX906-ISEL-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
@@ -17549,11 +17761,11 @@ define <4 x float> @global_saddr_64bit_lsr_iv_multiload(ptr addrspace(1) inreg %
; GFX942-ISEL: ; %bb.0: ; %bb
; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-ISEL-NEXT: s_mov_b32 s2, -1
-; GFX942-ISEL-NEXT: .LBB121_1: ; %bb5
+; GFX942-ISEL-NEXT: .LBB123_1: ; %bb5
; GFX942-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-ISEL-NEXT: s_add_i32 s2, s2, 1
; GFX942-ISEL-NEXT: s_cmpk_eq_i32 s2, 0xff
-; GFX942-ISEL-NEXT: s_cbranch_scc0 .LBB121_1
+; GFX942-ISEL-NEXT: s_cbranch_scc0 .LBB123_1
; GFX942-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX942-ISEL-NEXT: s_mov_b32 s3, 0
; GFX942-ISEL-NEXT: s_lshl_b64 s[2:3], s[2:3], 2
@@ -17576,11 +17788,11 @@ define <4 x float> @global_saddr_64bit_lsr_iv_multiload(ptr addrspace(1) inreg %
; GFX1012-ISEL: ; %bb.0: ; %bb
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1012-ISEL-NEXT: s_mov_b32 s4, -1
-; GFX1012-ISEL-NEXT: .LBB121_1: ; %bb5
+; GFX1012-ISEL-NEXT: .LBB123_1: ; %bb5
; GFX1012-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1012-ISEL-NEXT: s_add_i32 s4, s4, 1
; GFX1012-ISEL-NEXT: s_cmpk_eq_i32 s4, 0xff
-; GFX1012-ISEL-NEXT: s_cbranch_scc0 .LBB121_1
+; GFX1012-ISEL-NEXT: s_cbranch_scc0 .LBB123_1
; GFX1012-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1012-ISEL-NEXT: s_mov_b32 s5, 0
; GFX1012-ISEL-NEXT: v_mov_b32_e32 v0, 0
@@ -17603,12 +17815,12 @@ define <4 x float> @global_saddr_64bit_lsr_iv_multiload(ptr addrspace(1) inreg %
; GFX1100-ISEL: ; %bb.0: ; %bb
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-ISEL-NEXT: s_mov_b32 s2, -1
-; GFX1100-ISEL-NEXT: .LBB121_1: ; %bb5
+; GFX1100-ISEL-NEXT: .LBB123_1: ; %bb5
; GFX1100-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1100-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1100-ISEL-NEXT: s_add_i32 s2, s2, 1
; GFX1100-ISEL-NEXT: s_cmpk_eq_i32 s2, 0xff
-; GFX1100-ISEL-NEXT: s_cbranch_scc0 .LBB121_1
+; GFX1100-ISEL-NEXT: s_cbranch_scc0 .LBB123_1
; GFX1100-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1100-ISEL-NEXT: s_mov_b32 s3, 0
; GFX1100-ISEL-NEXT: v_mov_b32_e32 v0, 0
@@ -17632,12 +17844,12 @@ define <4 x float> @global_saddr_64bit_lsr_iv_multiload(ptr addrspace(1) inreg %
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-ISEL-NEXT: s_mov_b32 s2, -1
-; GFX1250-ISEL-NEXT: .LBB121_1: ; %bb5
+; GFX1250-ISEL-NEXT: .LBB123_1: ; %bb5
; GFX1250-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-ISEL-NEXT: s_add_co_i32 s2, s2, 1
; GFX1250-ISEL-NEXT: s_cmp_eq_u32 s2, 0xff
-; GFX1250-ISEL-NEXT: s_cbranch_scc0 .LBB121_1
+; GFX1250-ISEL-NEXT: s_cbranch_scc0 .LBB123_1
; GFX1250-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1250-ISEL-NEXT: s_mov_b32 s3, 0
; GFX1250-ISEL-NEXT: v_mov_b32_e32 v0, 0
@@ -17665,12 +17877,12 @@ define <4 x float> @global_saddr_64bit_lsr_iv_multiload(ptr addrspace(1) inreg %
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
; GFX1310-ISEL-NEXT: s_mov_b32 s2, -1
-; GFX1310-ISEL-NEXT: .LBB121_1: ; %bb5
+; GFX1310-ISEL-NEXT: .LBB123_1: ; %bb5
; GFX1310-ISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1310-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1310-ISEL-NEXT: s_add_co_i32 s2, s2, 1
; GFX1310-ISEL-NEXT: s_cmp_eq_u32 s2, 0xff
-; GFX1310-ISEL-NEXT: s_cbranch_scc0 .LBB121_1
+; GFX1310-ISEL-NEXT: s_cbranch_scc0 .LBB123_1
; GFX1310-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1310-ISEL-NEXT: s_mov_b32 s3, 0
; GFX1310-ISEL-NEXT: v_mov_b32_e32 v0, 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll
index 6afa6b8a5c481a..9962551027e27f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll
@@ -386,6 +386,202 @@ entry:
ret void
}
+;; vgpr pointer, system scope
+define void @av_global_store_b8_0_11(ptr addrspace(1) %addr, i8 %data) {
+; GFX906-SDAG-LABEL: av_global_store_b8_0_11:
+; GFX906-SDAG: ; %bb.0: ; %entry
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-SDAG-NEXT: global_store_byte v[0:1], v2, off
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX906-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-SDAG-LABEL: av_global_store_b8_0_11:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: global_store_byte v[0:1], v2, off sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: av_global_store_b8_0_11:
+; GFX1012-SDAG: ; %bb.0: ; %entry
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: global_store_byte v[0:1], v2, off
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: av_global_store_b8_0_11:
+; GFX1100-SDAG: ; %bb.0: ; %entry
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: av_global_store_b8_0_11:
+; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_store_b8 v[0:1], v2, off scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-SDAG-LABEL: av_global_store_b8_0_11:
+; GFX1310-SDAG: ; %bb.0: ; %entry
+; GFX1310-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1310-SDAG-NEXT: global_store_b8 v[0:1], v2, off scope:SCOPE_SYS
+; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX906-ISEL-LABEL: av_global_store_b8_0_11:
+; GFX906-ISEL: ; %bb.0: ; %entry
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-ISEL-NEXT: global_store_byte v[0:1], v2, off
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX906-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: av_global_store_b8_0_11:
+; GFX942-ISEL: ; %bb.0: ; %entry
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: global_store_byte v[0:1], v2, off sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: av_global_store_b8_0_11:
+; GFX1012-ISEL: ; %bb.0: ; %entry
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: global_store_byte v[0:1], v2, off
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: av_global_store_b8_0_11:
+; GFX1100-ISEL: ; %bb.0: ; %entry
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: global_store_b8 v[0:1], v2, off
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: av_global_store_b8_0_11:
+; GFX1250-ISEL: ; %bb.0: ; %entry
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: global_store_b8 v[0:1], v2, off scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-ISEL-LABEL: av_global_store_b8_0_11:
+; GFX1310-ISEL: ; %bb.0: ; %entry
+; GFX1310-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_expcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1310-ISEL-NEXT: global_store_b8 v[0:1], v2, off scope:SCOPE_SYS
+; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ call void @llvm.amdgcn.av.store.b8.p1(ptr addrspace(1) %addr, i8 %data, metadata !3)
+ ret void
+}
+
+;; sgpr pointer, system scope
+define void @av_global_store_b8_saddr_0_11(ptr addrspace(1) inreg %addr, i8 %data) {
+; GFX906-SDAG-LABEL: av_global_store_b8_saddr_0_11:
+; GFX906-SDAG: ; %bb.0: ; %entry
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX906-SDAG-NEXT: global_store_byte v1, v0, s[16:17]
+; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX906-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-SDAG-LABEL: av_global_store_b8_saddr_0_11:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX942-SDAG-NEXT: global_store_byte v1, v0, s[0:1] sc0 sc1
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-SDAG-LABEL: av_global_store_b8_saddr_0_11:
+; GFX1012-SDAG: ; %bb.0: ; %entry
+; GFX1012-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1012-SDAG-NEXT: global_store_byte v1, v0, s[16:17]
+; GFX1012-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: av_global_store_b8_saddr_0_11:
+; GFX1100-SDAG: ; %bb.0: ; %entry
+; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1100-SDAG-NEXT: global_store_b8 v1, v0, s[0:1]
+; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: av_global_store_b8_saddr_0_11:
+; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: global_store_b8 v1, v0, s[0:1] scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-SDAG-LABEL: av_global_store_b8_saddr_0_11:
+; GFX1310-SDAG: ; %bb.0: ; %entry
+; GFX1310-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1310-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1310-SDAG-NEXT: global_store_b8 v1, v0, s[0:1] scope:SCOPE_SYS
+; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX906-ISEL-LABEL: av_global_store_b8_saddr_0_11:
+; GFX906-ISEL: ; %bb.0: ; %entry
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX906-ISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX906-ISEL-NEXT: global_store_byte v1, v0, s[16:17]
+; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX906-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-ISEL-LABEL: av_global_store_b8_saddr_0_11:
+; GFX942-ISEL: ; %bb.0: ; %entry
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX942-ISEL-NEXT: global_store_byte v1, v0, s[0:1] sc0 sc1
+; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1012-ISEL-LABEL: av_global_store_b8_saddr_0_11:
+; GFX1012-ISEL: ; %bb.0: ; %entry
+; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1012-ISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1012-ISEL-NEXT: global_store_byte v1, v0, s[16:17]
+; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-ISEL-LABEL: av_global_store_b8_saddr_0_11:
+; GFX1100-ISEL: ; %bb.0: ; %entry
+; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-ISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1100-ISEL-NEXT: global_store_b8 v1, v0, s[0:1]
+; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-ISEL-LABEL: av_global_store_b8_saddr_0_11:
+; GFX1250-ISEL: ; %bb.0: ; %entry
+; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-ISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-ISEL-NEXT: global_store_b8 v1, v0, s[0:1] scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-ISEL-LABEL: av_global_store_b8_saddr_0_11:
+; GFX1310-ISEL: ; %bb.0: ; %entry
+; GFX1310-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_expcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1310-ISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1310-ISEL-NEXT: global_store_b8 v1, v0, s[0:1] scope:SCOPE_SYS
+; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ call void @llvm.amdgcn.av.store.b8.p1(ptr addrspace(1) %addr, i8 %data, metadata !3)
+ ret void
+}
+
;; vgpr pointer, system scope
define void @av_global_store_b16_0_11(ptr addrspace(1) %addr, i16 %data) {
; GFX906-SDAG-LABEL: av_global_store_b16_0_11:
diff --git a/llvm/test/CodeGen/AMDGPU/unsupported-av-load.ll b/llvm/test/CodeGen/AMDGPU/unsupported-av-load.ll
index 5539b8526fece3..f3858dc970291e 100644
--- a/llvm/test/CodeGen/AMDGPU/unsupported-av-load.ll
+++ b/llvm/test/CodeGen/AMDGPU/unsupported-av-load.ll
@@ -7,15 +7,18 @@
; RUN: not llc -global-isel=1 -mtriple=amdgpu8.10 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR %s
define void @av_load(ptr addrspace(1) %addr, ptr addrspace(1) %out) {
+; ERR: error: {{.*}}: in function @av_load {{.*}}: llvm.amdgcn.av.load.b8 requires target feature 'flat-global-insts'
; ERR: error: {{.*}}: in function @av_load {{.*}}: llvm.amdgcn.av.load.b16 requires target feature 'flat-global-insts'
; ERR: error: {{.*}}: in function @av_load {{.*}}: llvm.amdgcn.av.load.b32 requires target feature 'flat-global-insts'
; ERR: error: {{.*}}: in function @av_load {{.*}}: llvm.amdgcn.av.load.b64 requires target feature 'flat-global-insts'
; ERR: error: {{.*}}: in function @av_load {{.*}}: llvm.amdgcn.av.load.b128 requires target feature 'flat-global-insts'
entry:
+ %data8 = call i8 @llvm.amdgcn.av.load.b8.p1(ptr addrspace(1) %addr, metadata !0)
%data16 = call i16 @llvm.amdgcn.av.load.b16.p1(ptr addrspace(1) %addr, metadata !0)
%data32 = call i32 @llvm.amdgcn.av.load.b32.p1(ptr addrspace(1) %addr, metadata !0)
%data64 = call i64 @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) %addr, metadata !0)
%data128 = call <4 x i32> @llvm.amdgcn.av.load.b128.p1(ptr addrspace(1) %addr, metadata !0)
+ store volatile i8 %data8, ptr addrspace(1) %out
store volatile i16 %data16, ptr addrspace(1) %out
store volatile i32 %data32, ptr addrspace(1) %out
store volatile i64 %data64, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/unsupported-av-store.ll b/llvm/test/CodeGen/AMDGPU/unsupported-av-store.ll
index 03543c53daaab9..03d2ceb695ee00 100644
--- a/llvm/test/CodeGen/AMDGPU/unsupported-av-store.ll
+++ b/llvm/test/CodeGen/AMDGPU/unsupported-av-store.ll
@@ -6,12 +6,14 @@
; RUN: not llc -global-isel=1 -mtriple=amdgpu7.05 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR %s
; RUN: not llc -global-isel=1 -mtriple=amdgpu8.10 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR %s
-define void @av_store(ptr addrspace(1) %addr, i16 %data16, i32 %data32, i64 %data64, <4 x i32> %data128) {
+define void @av_store(ptr addrspace(1) %addr, i8 %data8, i16 %data16, i32 %data32, i64 %data64, <4 x i32> %data128) {
+; ERR: error: {{.*}}: in function @av_store {{.*}}: llvm.amdgcn.av.store.b8 requires target feature 'flat-global-insts'
; ERR: error: {{.*}}: in function @av_store {{.*}}: llvm.amdgcn.av.store.b16 requires target feature 'flat-global-insts'
; ERR: error: {{.*}}: in function @av_store {{.*}}: llvm.amdgcn.av.store.b32 requires target feature 'flat-global-insts'
; ERR: error: {{.*}}: in function @av_store {{.*}}: llvm.amdgcn.av.store.b64 requires target feature 'flat-global-insts'
; ERR: error: {{.*}}: in function @av_store {{.*}}: llvm.amdgcn.av.store.b128 requires target feature 'flat-global-insts'
entry:
+ call void @llvm.amdgcn.av.store.b8.p1(ptr addrspace(1) %addr, i8 %data8, metadata !0)
call void @llvm.amdgcn.av.store.b16.p1(ptr addrspace(1) %addr, i16 %data16, metadata !0)
call void @llvm.amdgcn.av.store.b32.p1(ptr addrspace(1) %addr, i32 %data32, metadata !0)
call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) %addr, i64 %data64, metadata !0)
diff --git a/llvm/test/Verifier/AMDGPU/intrinsics-av.ll b/llvm/test/Verifier/AMDGPU/intrinsics-av.ll
index 67626843c6588b..4e171eda549080 100644
--- a/llvm/test/Verifier/AMDGPU/intrinsics-av.ll
+++ b/llvm/test/Verifier/AMDGPU/intrinsics-av.ll
@@ -28,6 +28,20 @@ define void @av_flat_store_b128_bad_metadata(ptr %addr, <4 x i32> %data) {
ret void
}
+; CHECK: the last argument to av load/store intrinsics must be a metadata string
+; CHECK-NEXT: call i8 @llvm.amdgcn.av.load.b8.p1({{.*}})
+define i8 @av_global_load_b8_bad_metadata(ptr addrspace(1) %addr) {
+ %data = call i8 @llvm.amdgcn.av.load.b8.p1(ptr addrspace(1) %addr, metadata i32 0)
+ ret i8 %data
+}
+
+; CHECK: the last argument to av load/store intrinsics must be a metadata string
+; CHECK-NEXT: call void @llvm.amdgcn.av.store.b8.p1({{.*}})
+define void @av_global_store_b8_bad_metadata(ptr addrspace(1) %addr, i8 %data) {
+ call void @llvm.amdgcn.av.store.b8.p1(ptr addrspace(1) %addr, i8 %data, metadata i32 0)
+ ret void
+}
+
; CHECK: the last argument to av load/store intrinsics must be a metadata string
; CHECK-NEXT: call i16 @llvm.amdgcn.av.load.b16.p1({{.*}})
define i16 @av_global_load_b16_bad_metadata(ptr addrspace(1) %addr) {
>From 1e8bf6088a1b92c2293ade12672dc0bd5d9922bc Mon Sep 17 00:00:00 2001
From: pvanhout <pierre.vanhoutryve at amd.com>
Date: Wed, 7 Oct 2026 12:12:46 +0200
Subject: [PATCH 3/3] Use <2 x i32> for b64
---
clang/docs/LanguageExtensions.md | 5 +-
clang/include/clang/Basic/BuiltinsAMDGPU.td | 4 +-
.../builtins-amdgcn-global-load-store.cl | 124 +++++++++---------
clang/test/SemaHIP/amdgpu-av-load-store.hip | 10 +-
...builtins-amdgcn-global-load-store-error.cl | 14 +-
...s-amdgcn-global-load-store-target-error.cl | 6 +-
llvm/docs/AMDGPUUsage.rst | 2 +-
llvm/include/llvm/IR/IntrinsicsAMDGPU.td | 4 +-
llvm/test/CodeGen/AMDGPU/amdgcn-av-scopes.ll | 28 ++--
.../AMDGPU/llvm.amdgcn.av.load.b128.ll | 12 +-
.../AMDGPU/llvm.amdgcn.av.store.b128.ll | 20 +--
.../CodeGen/AMDGPU/unsupported-av-load.ll | 4 +-
.../CodeGen/AMDGPU/unsupported-av-store.ll | 4 +-
llvm/test/Verifier/AMDGPU/intrinsics-av.ll | 12 +-
14 files changed, 125 insertions(+), 124 deletions(-)
diff --git a/clang/docs/LanguageExtensions.md b/clang/docs/LanguageExtensions.md
index 88cd29412ea255..a01881d0fb9f08 100644
--- a/clang/docs/LanguageExtensions.md
+++ b/clang/docs/LanguageExtensions.md
@@ -5501,18 +5501,19 @@ the given mask has the same value for all active lanes of the current wave.
Signature:
```c
+typedef __attribute__((__vector_size__(2 * sizeof(unsigned int)))) unsigned int v2u;
typedef __attribute__((__vector_size__(4 * sizeof(unsigned int)))) unsigned int v4u;
unsigned char __builtin_amdgcn_av_load_b8(unsigned char *src, int scope);
unsigned short __builtin_amdgcn_av_load_b16(unsigned short *src, int scope);
unsigned int __builtin_amdgcn_av_load_b32(unsigned int *src, int scope);
-uint64_t __builtin_amdgcn_av_load_b64(uint64_t *src, int scope);
+v2u __builtin_amdgcn_av_load_b64(v2u *src, int scope);
v4u __builtin_amdgcn_av_load_b128(v4u *src, int scope);
void __builtin_amdgcn_av_store_b8(unsigned char *dst, unsigned char data, int scope);
void __builtin_amdgcn_av_store_b16(unsigned short *dst, unsigned short data, int scope);
void __builtin_amdgcn_av_store_b32(unsigned int *dst, unsigned int data, int scope);
-void __builtin_amdgcn_av_store_b64(uint64_t *dst, uint64_t data, int scope);
+void __builtin_amdgcn_av_store_b64(v2u *dst, v2u data, int scope);
void __builtin_amdgcn_av_store_b128(v4u *dst, v4u data, int scope);
```
diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.td b/clang/include/clang/Basic/BuiltinsAMDGPU.td
index 4cd8bef934912b..45c4838a488c3f 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.td
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.td
@@ -349,12 +349,12 @@ def __builtin_amdgcn_av_store_b32
let ArgNames = ["ptr", "data", "scope"];
}
def __builtin_amdgcn_av_load_b64
- : AMDGPUBuiltin<"uint64_t(uint64_t *, int)", [], "flat-global-insts"> {
+ : AMDGPUBuiltin<"_ExtVector<2, unsigned int>(_ExtVector<2, unsigned int> *, int)", [], "flat-global-insts"> {
let Documentation = [DocAVLoadB64];
let ArgNames = ["ptr", "scope"];
}
def __builtin_amdgcn_av_store_b64
- : AMDGPUBuiltin<"void(uint64_t *, uint64_t, int)", [], "flat-global-insts"> {
+ : AMDGPUBuiltin<"void(_ExtVector<2, unsigned int> *, _ExtVector<2, unsigned int>, int)", [], "flat-global-insts"> {
let Documentation = [DocAVStoreB64];
let ArgNames = ["ptr", "data", "scope"];
}
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-store.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-store.cl
index e5fc5453f02f94..49baab6eb4d0c0 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-store.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-store.cl
@@ -12,11 +12,11 @@ typedef v4u32 __global *global_ptr_to_v4u32;
typedef unsigned char u8;
typedef unsigned short u16;
typedef unsigned int u32;
-typedef unsigned long u64;
+typedef __attribute__((__vector_size__(2 * sizeof(unsigned int)))) unsigned int v2u32;
typedef u8 __global *global_ptr_to_u8;
typedef u16 __global *global_ptr_to_u16;
typedef u32 __global *global_ptr_to_u32;
-typedef u64 __global *global_ptr_to_u64;
+typedef v2u32 __global *global_ptr_to_v2u32;
//------------------------------------------------------------------------------
// Global Load
@@ -240,55 +240,55 @@ u32 test_amdgcn_global_load_b32_cluster(global_ptr_to_u32 ptr) {
// CHECK-LABEL: @test_amdgcn_global_load_b64_wave(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META8]])
-// CHECK-NEXT: ret i64 [[TMP0]]
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META8]])
+// CHECK-NEXT: ret <2 x i32> [[TMP0]]
//
-u64 test_amdgcn_global_load_b64_wave(global_ptr_to_u64 ptr) {
+v2u32 test_amdgcn_global_load_b64_wave(global_ptr_to_v2u32 ptr) {
return __builtin_amdgcn_av_load_b64(ptr, __MEMORY_SCOPE_WVFRNT);
}
// CHECK-LABEL: @test_amdgcn_global_load_b64_workgroup(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META9]])
-// CHECK-NEXT: ret i64 [[TMP0]]
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META9]])
+// CHECK-NEXT: ret <2 x i32> [[TMP0]]
//
-u64 test_amdgcn_global_load_b64_workgroup(global_ptr_to_u64 ptr) {
+v2u32 test_amdgcn_global_load_b64_workgroup(global_ptr_to_v2u32 ptr) {
return __builtin_amdgcn_av_load_b64(ptr, __MEMORY_SCOPE_WRKGRP);
}
// CHECK-LABEL: @test_amdgcn_global_load_b64_device(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META10]])
-// CHECK-NEXT: ret i64 [[TMP0]]
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META10]])
+// CHECK-NEXT: ret <2 x i32> [[TMP0]]
//
-u64 test_amdgcn_global_load_b64_device(global_ptr_to_u64 ptr) {
+v2u32 test_amdgcn_global_load_b64_device(global_ptr_to_v2u32 ptr) {
return __builtin_amdgcn_av_load_b64(ptr, __MEMORY_SCOPE_DEVICE);
}
// CHECK-LABEL: @test_amdgcn_global_load_b64_system(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META11]])
-// CHECK-NEXT: ret i64 [[TMP0]]
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META11]])
+// CHECK-NEXT: ret <2 x i32> [[TMP0]]
//
-u64 test_amdgcn_global_load_b64_system(global_ptr_to_u64 ptr) {
+v2u32 test_amdgcn_global_load_b64_system(global_ptr_to_v2u32 ptr) {
return __builtin_amdgcn_av_load_b64(ptr, __MEMORY_SCOPE_SYSTEM);
}
// CHECK-LABEL: @test_amdgcn_global_load_b64_single(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META12]])
-// CHECK-NEXT: ret i64 [[TMP0]]
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META12]])
+// CHECK-NEXT: ret <2 x i32> [[TMP0]]
//
-u64 test_amdgcn_global_load_b64_single(global_ptr_to_u64 ptr) {
+v2u32 test_amdgcn_global_load_b64_single(global_ptr_to_v2u32 ptr) {
return __builtin_amdgcn_av_load_b64(ptr, __MEMORY_SCOPE_SINGLE);
}
// CHECK-LABEL: @test_amdgcn_global_load_b64_cluster(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META13]])
-// CHECK-NEXT: ret i64 [[TMP0]]
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) [[PTR:%.*]], metadata [[META13]])
+// CHECK-NEXT: ret <2 x i32> [[TMP0]]
//
-u64 test_amdgcn_global_load_b64_cluster(global_ptr_to_u64 ptr) {
+v2u32 test_amdgcn_global_load_b64_cluster(global_ptr_to_v2u32 ptr) {
return __builtin_amdgcn_av_load_b64(ptr, __MEMORY_SCOPE_CLUSTR);
}
@@ -514,55 +514,55 @@ void test_amdgcn_global_store_b32_cluster(global_ptr_to_u32 ptr, u32 data) {
// CHECK-LABEL: @test_amdgcn_global_store_b64_wave(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) [[PTR:%.*]], i64 [[DATA:%.*]], metadata [[META8]])
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) [[PTR:%.*]], <2 x i32> [[DATA:%.*]], metadata [[META8]])
// CHECK-NEXT: ret void
//
-void test_amdgcn_global_store_b64_wave(global_ptr_to_u64 ptr, u64 data) {
+void test_amdgcn_global_store_b64_wave(global_ptr_to_v2u32 ptr, v2u32 data) {
__builtin_amdgcn_av_store_b64(ptr, data, __MEMORY_SCOPE_WVFRNT);
}
// CHECK-LABEL: @test_amdgcn_global_store_b64_workgroup(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) [[PTR:%.*]], i64 [[DATA:%.*]], metadata [[META9]])
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) [[PTR:%.*]], <2 x i32> [[DATA:%.*]], metadata [[META9]])
// CHECK-NEXT: ret void
//
-void test_amdgcn_global_store_b64_workgroup(global_ptr_to_u64 ptr, u64 data) {
+void test_amdgcn_global_store_b64_workgroup(global_ptr_to_v2u32 ptr, v2u32 data) {
__builtin_amdgcn_av_store_b64(ptr, data, __MEMORY_SCOPE_WRKGRP);
}
// CHECK-LABEL: @test_amdgcn_global_store_b64_device(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) [[PTR:%.*]], i64 [[DATA:%.*]], metadata [[META10]])
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) [[PTR:%.*]], <2 x i32> [[DATA:%.*]], metadata [[META10]])
// CHECK-NEXT: ret void
//
-void test_amdgcn_global_store_b64_device(global_ptr_to_u64 ptr, u64 data) {
+void test_amdgcn_global_store_b64_device(global_ptr_to_v2u32 ptr, v2u32 data) {
__builtin_amdgcn_av_store_b64(ptr, data, __MEMORY_SCOPE_DEVICE);
}
// CHECK-LABEL: @test_amdgcn_global_store_b64_system(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) [[PTR:%.*]], i64 [[DATA:%.*]], metadata [[META11]])
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) [[PTR:%.*]], <2 x i32> [[DATA:%.*]], metadata [[META11]])
// CHECK-NEXT: ret void
//
-void test_amdgcn_global_store_b64_system(global_ptr_to_u64 ptr, u64 data) {
+void test_amdgcn_global_store_b64_system(global_ptr_to_v2u32 ptr, v2u32 data) {
__builtin_amdgcn_av_store_b64(ptr, data, __MEMORY_SCOPE_SYSTEM);
}
// CHECK-LABEL: @test_amdgcn_global_store_b64_single(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) [[PTR:%.*]], i64 [[DATA:%.*]], metadata [[META12]])
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) [[PTR:%.*]], <2 x i32> [[DATA:%.*]], metadata [[META12]])
// CHECK-NEXT: ret void
//
-void test_amdgcn_global_store_b64_single(global_ptr_to_u64 ptr, u64 data) {
+void test_amdgcn_global_store_b64_single(global_ptr_to_v2u32 ptr, v2u32 data) {
__builtin_amdgcn_av_store_b64(ptr, data, __MEMORY_SCOPE_SINGLE);
}
// CHECK-LABEL: @test_amdgcn_global_store_b64_cluster(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) [[PTR:%.*]], i64 [[DATA:%.*]], metadata [[META13]])
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) [[PTR:%.*]], <2 x i32> [[DATA:%.*]], metadata [[META13]])
// CHECK-NEXT: ret void
//
-void test_amdgcn_global_store_b64_cluster(global_ptr_to_u64 ptr, u64 data) {
+void test_amdgcn_global_store_b64_cluster(global_ptr_to_v2u32 ptr, v2u32 data) {
__builtin_amdgcn_av_store_b64(ptr, data, __MEMORY_SCOPE_CLUSTR);
}
@@ -788,55 +788,55 @@ u32 test_amdgcn_flat_load_b32_cluster(u32 * ptr) {
// CHECK-LABEL: @test_amdgcn_flat_load_b64_wave(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.amdgcn.av.load.b64.p0(ptr [[PTR:%.*]], metadata [[META8]])
-// CHECK-NEXT: ret i64 [[TMP0]]
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.av.load.b64.p0(ptr [[PTR:%.*]], metadata [[META8]])
+// CHECK-NEXT: ret <2 x i32> [[TMP0]]
//
-u64 test_amdgcn_flat_load_b64_wave(u64 * ptr) {
+v2u32 test_amdgcn_flat_load_b64_wave(v2u32 * ptr) {
return __builtin_amdgcn_av_load_b64(ptr, __MEMORY_SCOPE_WVFRNT);
}
// CHECK-LABEL: @test_amdgcn_flat_load_b64_workgroup(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.amdgcn.av.load.b64.p0(ptr [[PTR:%.*]], metadata [[META9]])
-// CHECK-NEXT: ret i64 [[TMP0]]
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.av.load.b64.p0(ptr [[PTR:%.*]], metadata [[META9]])
+// CHECK-NEXT: ret <2 x i32> [[TMP0]]
//
-u64 test_amdgcn_flat_load_b64_workgroup(u64 * ptr) {
+v2u32 test_amdgcn_flat_load_b64_workgroup(v2u32 * ptr) {
return __builtin_amdgcn_av_load_b64(ptr, __MEMORY_SCOPE_WRKGRP);
}
// CHECK-LABEL: @test_amdgcn_flat_load_b64_device(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.amdgcn.av.load.b64.p0(ptr [[PTR:%.*]], metadata [[META10]])
-// CHECK-NEXT: ret i64 [[TMP0]]
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.av.load.b64.p0(ptr [[PTR:%.*]], metadata [[META10]])
+// CHECK-NEXT: ret <2 x i32> [[TMP0]]
//
-u64 test_amdgcn_flat_load_b64_device(u64 * ptr) {
+v2u32 test_amdgcn_flat_load_b64_device(v2u32 * ptr) {
return __builtin_amdgcn_av_load_b64(ptr, __MEMORY_SCOPE_DEVICE);
}
// CHECK-LABEL: @test_amdgcn_flat_load_b64_system(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.amdgcn.av.load.b64.p0(ptr [[PTR:%.*]], metadata [[META11]])
-// CHECK-NEXT: ret i64 [[TMP0]]
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.av.load.b64.p0(ptr [[PTR:%.*]], metadata [[META11]])
+// CHECK-NEXT: ret <2 x i32> [[TMP0]]
//
-u64 test_amdgcn_flat_load_b64_system(u64 * ptr) {
+v2u32 test_amdgcn_flat_load_b64_system(v2u32 * ptr) {
return __builtin_amdgcn_av_load_b64(ptr, __MEMORY_SCOPE_SYSTEM);
}
// CHECK-LABEL: @test_amdgcn_flat_load_b64_single(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.amdgcn.av.load.b64.p0(ptr [[PTR:%.*]], metadata [[META12]])
-// CHECK-NEXT: ret i64 [[TMP0]]
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.av.load.b64.p0(ptr [[PTR:%.*]], metadata [[META12]])
+// CHECK-NEXT: ret <2 x i32> [[TMP0]]
//
-u64 test_amdgcn_flat_load_b64_single(u64 * ptr) {
+v2u32 test_amdgcn_flat_load_b64_single(v2u32 * ptr) {
return __builtin_amdgcn_av_load_b64(ptr, __MEMORY_SCOPE_SINGLE);
}
// CHECK-LABEL: @test_amdgcn_flat_load_b64_cluster(
// CHECK-NEXT: entry:
-// CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.amdgcn.av.load.b64.p0(ptr [[PTR:%.*]], metadata [[META13]])
-// CHECK-NEXT: ret i64 [[TMP0]]
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.av.load.b64.p0(ptr [[PTR:%.*]], metadata [[META13]])
+// CHECK-NEXT: ret <2 x i32> [[TMP0]]
//
-u64 test_amdgcn_flat_load_b64_cluster(u64 * ptr) {
+v2u32 test_amdgcn_flat_load_b64_cluster(v2u32 * ptr) {
return __builtin_amdgcn_av_load_b64(ptr, __MEMORY_SCOPE_CLUSTR);
}
@@ -1062,55 +1062,55 @@ void test_amdgcn_flat_store_b32_cluster(u32 * ptr, u32 data) {
// CHECK-LABEL: @test_amdgcn_flat_store_b64_wave(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p0(ptr [[PTR:%.*]], i64 [[DATA:%.*]], metadata [[META8]])
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p0(ptr [[PTR:%.*]], <2 x i32> [[DATA:%.*]], metadata [[META8]])
// CHECK-NEXT: ret void
//
-void test_amdgcn_flat_store_b64_wave(u64 * ptr, u64 data) {
+void test_amdgcn_flat_store_b64_wave(v2u32 * ptr, v2u32 data) {
__builtin_amdgcn_av_store_b64(ptr, data, __MEMORY_SCOPE_WVFRNT);
}
// CHECK-LABEL: @test_amdgcn_flat_store_b64_workgroup(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p0(ptr [[PTR:%.*]], i64 [[DATA:%.*]], metadata [[META9]])
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p0(ptr [[PTR:%.*]], <2 x i32> [[DATA:%.*]], metadata [[META9]])
// CHECK-NEXT: ret void
//
-void test_amdgcn_flat_store_b64_workgroup(u64 * ptr, u64 data) {
+void test_amdgcn_flat_store_b64_workgroup(v2u32 * ptr, v2u32 data) {
__builtin_amdgcn_av_store_b64(ptr, data, __MEMORY_SCOPE_WRKGRP);
}
// CHECK-LABEL: @test_amdgcn_flat_store_b64_device(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p0(ptr [[PTR:%.*]], i64 [[DATA:%.*]], metadata [[META10]])
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p0(ptr [[PTR:%.*]], <2 x i32> [[DATA:%.*]], metadata [[META10]])
// CHECK-NEXT: ret void
//
-void test_amdgcn_flat_store_b64_device(u64 * ptr, u64 data) {
+void test_amdgcn_flat_store_b64_device(v2u32 * ptr, v2u32 data) {
__builtin_amdgcn_av_store_b64(ptr, data, __MEMORY_SCOPE_DEVICE);
}
// CHECK-LABEL: @test_amdgcn_flat_store_b64_system(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p0(ptr [[PTR:%.*]], i64 [[DATA:%.*]], metadata [[META11]])
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p0(ptr [[PTR:%.*]], <2 x i32> [[DATA:%.*]], metadata [[META11]])
// CHECK-NEXT: ret void
//
-void test_amdgcn_flat_store_b64_system(u64 * ptr, u64 data) {
+void test_amdgcn_flat_store_b64_system(v2u32 * ptr, v2u32 data) {
__builtin_amdgcn_av_store_b64(ptr, data, __MEMORY_SCOPE_SYSTEM);
}
// CHECK-LABEL: @test_amdgcn_flat_store_b64_single(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p0(ptr [[PTR:%.*]], i64 [[DATA:%.*]], metadata [[META12]])
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p0(ptr [[PTR:%.*]], <2 x i32> [[DATA:%.*]], metadata [[META12]])
// CHECK-NEXT: ret void
//
-void test_amdgcn_flat_store_b64_single(u64 * ptr, u64 data) {
+void test_amdgcn_flat_store_b64_single(v2u32 * ptr, v2u32 data) {
__builtin_amdgcn_av_store_b64(ptr, data, __MEMORY_SCOPE_SINGLE);
}
// CHECK-LABEL: @test_amdgcn_flat_store_b64_cluster(
// CHECK-NEXT: entry:
-// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p0(ptr [[PTR:%.*]], i64 [[DATA:%.*]], metadata [[META13]])
+// CHECK-NEXT: tail call void @llvm.amdgcn.av.store.b64.p0(ptr [[PTR:%.*]], <2 x i32> [[DATA:%.*]], metadata [[META13]])
// CHECK-NEXT: ret void
//
-void test_amdgcn_flat_store_b64_cluster(u64 * ptr, u64 data) {
+void test_amdgcn_flat_store_b64_cluster(v2u32 * ptr, v2u32 data) {
__builtin_amdgcn_av_store_b64(ptr, data, __MEMORY_SCOPE_CLUSTR);
}
//.
diff --git a/clang/test/SemaHIP/amdgpu-av-load-store.hip b/clang/test/SemaHIP/amdgpu-av-load-store.hip
index b354dba388aff2..1a308c4e0c27ed 100644
--- a/clang/test/SemaHIP/amdgpu-av-load-store.hip
+++ b/clang/test/SemaHIP/amdgpu-av-load-store.hip
@@ -11,7 +11,7 @@ typedef __attribute__((__vector_size__(4 * sizeof(unsigned int)))) unsigned int
typedef unsigned char u8;
typedef unsigned short u16;
typedef unsigned int u32;
-typedef unsigned long u64;
+typedef __attribute__((__vector_size__(2 * sizeof(unsigned int)))) unsigned int v2u32;
__device__ void test_av_load_store_device(v4u32 *ptr, v4u32 data) {
v4u32 res = __builtin_amdgcn_av_load_b128(ptr, 0);
@@ -53,12 +53,12 @@ __global__ void test_av_load_store_b32_kernel(u32 *ptr, u32 data) {
__builtin_amdgcn_av_store_b32(ptr, data, 0);
}
-__device__ void test_av_load_store_b64_device(u64 *ptr, u64 data) {
- u64 res = __builtin_amdgcn_av_load_b64(ptr, 0);
+__device__ void test_av_load_store_b64_device(v2u32 *ptr, v2u32 data) {
+ v2u32 res = __builtin_amdgcn_av_load_b64(ptr, 0);
__builtin_amdgcn_av_store_b64(ptr, data, 0);
}
-__global__ void test_av_load_store_b64_kernel(u64 *ptr, u64 data) {
- u64 res = __builtin_amdgcn_av_load_b64(ptr, 0);
+__global__ void test_av_load_store_b64_kernel(v2u32 *ptr, v2u32 data) {
+ v2u32 res = __builtin_amdgcn_av_load_b64(ptr, 0);
__builtin_amdgcn_av_store_b64(ptr, data, 0);
}
diff --git a/clang/test/SemaOpenCL/builtins-amdgcn-global-load-store-error.cl b/clang/test/SemaOpenCL/builtins-amdgcn-global-load-store-error.cl
index dba8e7fbe2b067..bcd5a0a18404d2 100644
--- a/clang/test/SemaOpenCL/builtins-amdgcn-global-load-store-error.cl
+++ b/clang/test/SemaOpenCL/builtins-amdgcn-global-load-store-error.cl
@@ -4,7 +4,7 @@
typedef unsigned char u8;
typedef unsigned short u16;
typedef unsigned int u32;
-typedef unsigned long u64;
+typedef __attribute__((__vector_size__(2 * sizeof(unsigned int)))) unsigned int v2u32;
typedef __attribute__((__vector_size__(4 * sizeof(unsigned int)))) unsigned int v4u32;
typedef v4u32 __global *global_ptr_to_v4u32;
@@ -15,8 +15,8 @@ typedef u16 __global *global_ptr_to_u16;
typedef u16 __private *private_ptr_to_u16;
typedef u32 __global *global_ptr_to_u32;
typedef u32 __private *private_ptr_to_u32;
-typedef u64 __global *global_ptr_to_u64;
-typedef u64 __private *private_ptr_to_u64;
+typedef v2u32 __global *global_ptr_to_v2u32;
+typedef v2u32 __private *private_ptr_to_v2u32;
void test_amdgcn_av_store_b128_bad_ptr(private_ptr_to_v4u32 ptr, v4u32 data) {
__builtin_amdgcn_av_store_b128(ptr, data, __MEMORY_SCOPE_SYSTEM); //expected-error{{builtin requires a global or generic pointer}}
@@ -82,18 +82,18 @@ u32 test_amdgcn_av_load_b32_bad_scope(global_ptr_to_u32 ptr) {
return __builtin_amdgcn_av_load_b32(ptr, 42); //expected-error{{synchronization scope argument to atomic operation is invalid}}
}
-void test_amdgcn_av_store_b64_bad_ptr(private_ptr_to_u64 ptr, u64 data) {
+void test_amdgcn_av_store_b64_bad_ptr(private_ptr_to_v2u32 ptr, v2u32 data) {
__builtin_amdgcn_av_store_b64(ptr, data, __MEMORY_SCOPE_SYSTEM); //expected-error{{builtin requires a global or generic pointer}}
}
-void test_amdgcn_av_store_b64_bad_scope(global_ptr_to_u64 ptr, u64 data) {
+void test_amdgcn_av_store_b64_bad_scope(global_ptr_to_v2u32 ptr, v2u32 data) {
__builtin_amdgcn_av_store_b64(ptr, data, 42); //expected-error{{synchronization scope argument to atomic operation is invalid}}
}
-u64 test_amdgcn_av_load_b64_bad_ptr(private_ptr_to_u64 ptr) {
+v2u32 test_amdgcn_av_load_b64_bad_ptr(private_ptr_to_v2u32 ptr) {
return __builtin_amdgcn_av_load_b64(ptr, __MEMORY_SCOPE_SYSTEM); //expected-error{{builtin requires a global or generic pointer}}
}
-u64 test_amdgcn_av_load_b64_bad_scope(global_ptr_to_u64 ptr) {
+v2u32 test_amdgcn_av_load_b64_bad_scope(global_ptr_to_v2u32 ptr) {
return __builtin_amdgcn_av_load_b64(ptr, 42); //expected-error{{synchronization scope argument to atomic operation is invalid}}
}
diff --git a/clang/test/SemaOpenCL/builtins-amdgcn-global-load-store-target-error.cl b/clang/test/SemaOpenCL/builtins-amdgcn-global-load-store-target-error.cl
index fab983a57de1e2..a3a15b6ebad7ac 100644
--- a/clang/test/SemaOpenCL/builtins-amdgcn-global-load-store-target-error.cl
+++ b/clang/test/SemaOpenCL/builtins-amdgcn-global-load-store-target-error.cl
@@ -8,16 +8,16 @@ typedef v4u32 __global *global_ptr_to_v4u32;
typedef unsigned char u8;
typedef unsigned short u16;
typedef unsigned int u32;
-typedef unsigned long u64;
+typedef __attribute__((__vector_size__(2 * sizeof(unsigned int)))) unsigned int v2u32;
typedef u8 __global *global_ptr_to_u8;
typedef u16 __global *global_ptr_to_u16;
typedef u32 __global *global_ptr_to_u32;
-typedef u64 __global *global_ptr_to_u64;
+typedef v2u32 __global *global_ptr_to_v2u32;
void test_amdgcn_av_load_store_target(global_ptr_to_u8 ptr8, u8 data8,
global_ptr_to_u16 ptr16, u16 data16,
global_ptr_to_u32 ptr32, u32 data32,
- global_ptr_to_u64 ptr64, u64 data64,
+ global_ptr_to_v2u32 ptr64, v2u32 data64,
global_ptr_to_v4u32 ptr128, v4u32 data128) {
__builtin_amdgcn_av_load_b8(ptr8, 0); // expected-error{{'__builtin_amdgcn_av_load_b8' needs target feature flat-global-insts}}
__builtin_amdgcn_av_load_b16(ptr16, 0); // expected-error{{'__builtin_amdgcn_av_load_b16' needs target feature flat-global-insts}}
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index a6070cf082ff81..22fff241643bf3 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -2263,7 +2263,7 @@ These intrinsics exist for the following data types:
8 bits ``i8`` ``llvm.amdgcn.av.load.b8`` ``llvm.amdgcn.av.store.b8``
16 bits ``i16`` ``llvm.amdgcn.av.load.b16`` ``llvm.amdgcn.av.store.b16``
32 bits ``i32`` ``llvm.amdgcn.av.load.b32`` ``llvm.amdgcn.av.store.b32``
- 64 bits ``i64`` ``llvm.amdgcn.av.load.b64`` ``llvm.amdgcn.av.store.b64``
+ 64 bits ``<2 x i32>`` ``llvm.amdgcn.av.load.b64`` ``llvm.amdgcn.av.store.b64``
128 bits ``<4 x i32>`` ``llvm.amdgcn.av.load.b128`` ``llvm.amdgcn.av.store.b128``
========== ============= ====================================== =======================================
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index ca6ff65f9da4ef..8a4c672573af50 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -1025,8 +1025,8 @@ def int_amdgcn_av_load_b16 : AMDGPUAVLoad<llvm_i16_ty>;
def int_amdgcn_av_store_b32 : AMDGPUAVStore<llvm_i32_ty>;
def int_amdgcn_av_load_b32 : AMDGPUAVLoad<llvm_i32_ty>;
-def int_amdgcn_av_store_b64 : AMDGPUAVStore<llvm_i64_ty>;
-def int_amdgcn_av_load_b64 : AMDGPUAVLoad<llvm_i64_ty>;
+def int_amdgcn_av_store_b64 : AMDGPUAVStore<llvm_v2i32_ty>;
+def int_amdgcn_av_load_b64 : AMDGPUAVLoad<llvm_v2i32_ty>;
def int_amdgcn_av_store_b128 : AMDGPUAVStore<llvm_v4i32_ty>;
def int_amdgcn_av_load_b128 : AMDGPUAVLoad<llvm_v4i32_ty>;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn-av-scopes.ll b/llvm/test/CodeGen/AMDGPU/amdgcn-av-scopes.ll
index d59b5059246803..e972048dad6330 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn-av-scopes.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn-av-scopes.ll
@@ -4973,7 +4973,7 @@ define void @flat_store_b32_system(ptr %p, i32 %v) {
}
; scope: system
-define i64 @global_load_b64_system(ptr addrspace(1) %p) {
+define <2 x i32> @global_load_b64_system(ptr addrspace(1) %p) {
; GFX942-SDAG-LABEL: global_load_b64_system:
; GFX942-SDAG: ; %bb.0:
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5059,16 +5059,16 @@ define i64 @global_load_b64_system(ptr addrspace(1) %p) {
; GFX1100-WGP-ISEL-NEXT: global_load_b64 v[0:1], v[0:1], off glc
; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
- %v = call i64 @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) %p, metadata !3)
- ret i64 %v
+ %v = call <2 x i32> @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) %p, metadata !3)
+ ret <2 x i32> %v
}
; scope: system
-define void @global_store_b64_system(ptr addrspace(1) %p, i64 %v) {
+define void @global_store_b64_system(ptr addrspace(1) %p, <2 x i32> %v) {
; GFX942-SDAG-LABEL: global_store_b64_system:
; GFX942-SDAG: ; %bb.0:
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-SDAG-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX942-SDAG-NEXT: global_store_dwordx2 v[0:1], v[2:3], off sc0 sc1
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -5088,7 +5088,7 @@ define void @global_store_b64_system(ptr addrspace(1) %p, i64 %v) {
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX1250-SDAG-NEXT: global_store_b64 v[0:1], v[2:3], off scope:SCOPE_SYS
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1012-WGP-SDAG-LABEL: global_store_b64_system:
@@ -5140,12 +5140,12 @@ define void @global_store_b64_system(ptr addrspace(1) %p, i64 %v) {
; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-WGP-ISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
- call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) %p, i64 %v, metadata !3)
+ call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) %p, <2 x i32> %v, metadata !3)
ret void
}
; scope: system
-define i64 @flat_load_b64_system(ptr %p) {
+define <2 x i32> @flat_load_b64_system(ptr %p) {
; GFX942-SDAG-LABEL: flat_load_b64_system:
; GFX942-SDAG: ; %bb.0:
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5231,16 +5231,16 @@ define i64 @flat_load_b64_system(ptr %p) {
; GFX1100-WGP-ISEL-NEXT: flat_load_b64 v[0:1], v[0:1] glc
; GFX1100-WGP-ISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
- %v = call i64 @llvm.amdgcn.av.load.b64.p0(ptr %p, metadata !3)
- ret i64 %v
+ %v = call <2 x i32> @llvm.amdgcn.av.load.b64.p0(ptr %p, metadata !3)
+ ret <2 x i32> %v
}
; scope: system
-define void @flat_store_b64_system(ptr %p, i64 %v) {
+define void @flat_store_b64_system(ptr %p, <2 x i32> %v) {
; GFX942-SDAG-LABEL: flat_store_b64_system:
; GFX942-SDAG: ; %bb.0:
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; GFX942-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3] sc0 sc1
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -5262,7 +5262,7 @@ define void @flat_store_b64_system(ptr %p, i64 %v) {
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: flat_store_b64 v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: flat_store_b64 v[0:1], v[2:3] scope:SCOPE_SYS
; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -5322,7 +5322,7 @@ define void @flat_store_b64_system(ptr %p, i64 %v) {
; GFX1100-WGP-ISEL-NEXT: flat_store_b64 v[0:1], v[2:3]
; GFX1100-WGP-ISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-WGP-ISEL-NEXT: s_setpc_b64 s[30:31]
- call void @llvm.amdgcn.av.store.b64.p0(ptr %p, i64 %v, metadata !3)
+ call void @llvm.amdgcn.av.store.b64.p0(ptr %p, <2 x i32> %v, metadata !3)
ret void
}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.load.b128.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.load.b128.ll
index 1c25da5e68b5ae..82228698ad2213 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.load.b128.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.load.b128.ll
@@ -1503,7 +1503,7 @@ entry:
}
;; vgpr pointer, system scope
-define i64 @av_global_load_b64_0_11(ptr addrspace(1) %addr) {
+define <2 x i32> @av_global_load_b64_0_11(ptr addrspace(1) %addr) {
; GFX906-SDAG-LABEL: av_global_load_b64_0_11:
; GFX906-SDAG: ; %bb.0: ; %entry
; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1598,12 +1598,12 @@ define i64 @av_global_load_b64_0_11(ptr addrspace(1) %addr) {
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
entry:
- %data = call i64 @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) %addr, metadata !3)
- ret i64 %data
+ %data = call <2 x i32> @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) %addr, metadata !3)
+ ret <2 x i32> %data
}
;; sgpr pointer, system scope
-define i64 @av_global_load_b64_saddr_0_03(ptr addrspace(1) inreg %addr) {
+define <2 x i32> @av_global_load_b64_saddr_0_03(ptr addrspace(1) inreg %addr) {
; GFX906-SDAG-LABEL: av_global_load_b64_saddr_0_03:
; GFX906-SDAG: ; %bb.0: ; %entry
; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1710,8 +1710,8 @@ define i64 @av_global_load_b64_saddr_0_03(ptr addrspace(1) inreg %addr) {
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
entry:
- %data = call i64 @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) %addr, metadata !3)
- ret i64 %data
+ %data = call <2 x i32> @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) %addr, metadata !3)
+ ret <2 x i32> %data
}
;;==============================================================================
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll
index 9962551027e27f..485ba9249a69e8 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll
@@ -975,7 +975,7 @@ entry:
}
;; vgpr pointer, system scope
-define void @av_global_store_b64_0_11(ptr addrspace(1) %addr, i64 %data) {
+define void @av_global_store_b64_0_11(ptr addrspace(1) %addr, <2 x i32> %data) {
; GFX906-SDAG-LABEL: av_global_store_b64_0_11:
; GFX906-SDAG: ; %bb.0: ; %entry
; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -986,7 +986,7 @@ define void @av_global_store_b64_0_11(ptr addrspace(1) %addr, i64 %data) {
; GFX942-SDAG-LABEL: av_global_store_b64_0_11:
; GFX942-SDAG: ; %bb.0: ; %entry
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-SDAG-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX942-SDAG-NEXT: global_store_dwordx2 v[0:1], v[2:3], off sc0 sc1
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -1006,7 +1006,7 @@ define void @av_global_store_b64_0_11(ptr addrspace(1) %addr, i64 %data) {
; GFX1250-SDAG: ; %bb.0: ; %entry
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX1250-SDAG-NEXT: global_store_b64 v[0:1], v[2:3], off scope:SCOPE_SYS
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1310-SDAG-LABEL: av_global_store_b64_0_11:
@@ -1016,7 +1016,7 @@ define void @av_global_store_b64_0_11(ptr addrspace(1) %addr, i64 %data) {
; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX1310-SDAG-NEXT: global_store_b64 v[0:1], v[2:3], off scope:SCOPE_SYS
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GFX906-ISEL-LABEL: av_global_store_b64_0_11:
@@ -1062,12 +1062,12 @@ define void @av_global_store_b64_0_11(ptr addrspace(1) %addr, i64 %data) {
; GFX1310-ISEL-NEXT: global_store_b64 v[0:1], v[2:3], off scope:SCOPE_SYS
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
entry:
- call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) %addr, i64 %data, metadata !3)
+ call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) %addr, <2 x i32> %data, metadata !3)
ret void
}
;; sgpr pointer, system scope
-define void @av_global_store_b64_saddr_0_11(ptr addrspace(1) inreg %addr, i64 %data) {
+define void @av_global_store_b64_saddr_0_11(ptr addrspace(1) inreg %addr, <2 x i32> %data) {
; GFX906-SDAG-LABEL: av_global_store_b64_saddr_0_11:
; GFX906-SDAG: ; %bb.0: ; %entry
; GFX906-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1080,7 +1080,7 @@ define void @av_global_store_b64_saddr_0_11(ptr addrspace(1) inreg %addr, i64 %d
; GFX942-SDAG: ; %bb.0: ; %entry
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX942-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX942-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -1103,7 +1103,7 @@ define void @av_global_store_b64_saddr_0_11(ptr addrspace(1) inreg %addr, i64 %d
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1] scope:SCOPE_SYS
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1310-SDAG-LABEL: av_global_store_b64_saddr_0_11:
@@ -1114,7 +1114,7 @@ define void @av_global_store_b64_saddr_0_11(ptr addrspace(1) inreg %addr, i64 %d
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1310-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX1310-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1310-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1] scope:SCOPE_SYS
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GFX906-ISEL-LABEL: av_global_store_b64_saddr_0_11:
@@ -1166,7 +1166,7 @@ define void @av_global_store_b64_saddr_0_11(ptr addrspace(1) inreg %addr, i64 %d
; GFX1310-ISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1] scope:SCOPE_SYS
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
entry:
- call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) %addr, i64 %data, metadata !3)
+ call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) %addr, <2 x i32> %data, metadata !3)
ret void
}
diff --git a/llvm/test/CodeGen/AMDGPU/unsupported-av-load.ll b/llvm/test/CodeGen/AMDGPU/unsupported-av-load.ll
index f3858dc970291e..351e92f2db42ba 100644
--- a/llvm/test/CodeGen/AMDGPU/unsupported-av-load.ll
+++ b/llvm/test/CodeGen/AMDGPU/unsupported-av-load.ll
@@ -16,12 +16,12 @@ entry:
%data8 = call i8 @llvm.amdgcn.av.load.b8.p1(ptr addrspace(1) %addr, metadata !0)
%data16 = call i16 @llvm.amdgcn.av.load.b16.p1(ptr addrspace(1) %addr, metadata !0)
%data32 = call i32 @llvm.amdgcn.av.load.b32.p1(ptr addrspace(1) %addr, metadata !0)
- %data64 = call i64 @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) %addr, metadata !0)
+ %data64 = call <2 x i32> @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) %addr, metadata !0)
%data128 = call <4 x i32> @llvm.amdgcn.av.load.b128.p1(ptr addrspace(1) %addr, metadata !0)
store volatile i8 %data8, ptr addrspace(1) %out
store volatile i16 %data16, ptr addrspace(1) %out
store volatile i32 %data32, ptr addrspace(1) %out
- store volatile i64 %data64, ptr addrspace(1) %out
+ store volatile <2 x i32> %data64, ptr addrspace(1) %out
store volatile <4 x i32> %data128, ptr addrspace(1) %out
ret void
}
diff --git a/llvm/test/CodeGen/AMDGPU/unsupported-av-store.ll b/llvm/test/CodeGen/AMDGPU/unsupported-av-store.ll
index 03d2ceb695ee00..bc49cd82509971 100644
--- a/llvm/test/CodeGen/AMDGPU/unsupported-av-store.ll
+++ b/llvm/test/CodeGen/AMDGPU/unsupported-av-store.ll
@@ -6,7 +6,7 @@
; RUN: not llc -global-isel=1 -mtriple=amdgpu7.05 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR %s
; RUN: not llc -global-isel=1 -mtriple=amdgpu8.10 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR %s
-define void @av_store(ptr addrspace(1) %addr, i8 %data8, i16 %data16, i32 %data32, i64 %data64, <4 x i32> %data128) {
+define void @av_store(ptr addrspace(1) %addr, i8 %data8, i16 %data16, i32 %data32, <2 x i32> %data64, <4 x i32> %data128) {
; ERR: error: {{.*}}: in function @av_store {{.*}}: llvm.amdgcn.av.store.b8 requires target feature 'flat-global-insts'
; ERR: error: {{.*}}: in function @av_store {{.*}}: llvm.amdgcn.av.store.b16 requires target feature 'flat-global-insts'
; ERR: error: {{.*}}: in function @av_store {{.*}}: llvm.amdgcn.av.store.b32 requires target feature 'flat-global-insts'
@@ -16,7 +16,7 @@ entry:
call void @llvm.amdgcn.av.store.b8.p1(ptr addrspace(1) %addr, i8 %data8, metadata !0)
call void @llvm.amdgcn.av.store.b16.p1(ptr addrspace(1) %addr, i16 %data16, metadata !0)
call void @llvm.amdgcn.av.store.b32.p1(ptr addrspace(1) %addr, i32 %data32, metadata !0)
- call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) %addr, i64 %data64, metadata !0)
+ call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) %addr, <2 x i32> %data64, metadata !0)
call void @llvm.amdgcn.av.store.b128.p1(ptr addrspace(1) %addr, <4 x i32> %data128, metadata !0)
ret void
}
diff --git a/llvm/test/Verifier/AMDGPU/intrinsics-av.ll b/llvm/test/Verifier/AMDGPU/intrinsics-av.ll
index 4e171eda549080..b1e88070ec9cc9 100644
--- a/llvm/test/Verifier/AMDGPU/intrinsics-av.ll
+++ b/llvm/test/Verifier/AMDGPU/intrinsics-av.ll
@@ -71,15 +71,15 @@ define void @av_global_store_b32_bad_metadata(ptr addrspace(1) %addr, i32 %data)
}
; CHECK: the last argument to av load/store intrinsics must be a metadata string
-; CHECK-NEXT: call i64 @llvm.amdgcn.av.load.b64.p1({{.*}})
-define i64 @av_global_load_b64_bad_metadata(ptr addrspace(1) %addr) {
- %data = call i64 @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) %addr, metadata i32 0)
- ret i64 %data
+; CHECK-NEXT: call <2 x i32> @llvm.amdgcn.av.load.b64.p1({{.*}})
+define <2 x i32> @av_global_load_b64_bad_metadata(ptr addrspace(1) %addr) {
+ %data = call <2 x i32> @llvm.amdgcn.av.load.b64.p1(ptr addrspace(1) %addr, metadata i32 0)
+ ret <2 x i32> %data
}
; CHECK: the last argument to av load/store intrinsics must be a metadata string
; CHECK-NEXT: call void @llvm.amdgcn.av.store.b64.p1({{.*}})
-define void @av_global_store_b64_bad_metadata(ptr addrspace(1) %addr, i64 %data) {
- call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) %addr, i64 %data, metadata i32 0)
+define void @av_global_store_b64_bad_metadata(ptr addrspace(1) %addr, <2 x i32> %data) {
+ call void @llvm.amdgcn.av.store.b64.p1(ptr addrspace(1) %addr, <2 x i32> %data, metadata i32 0)
ret void
}
More information about the cfe-commits
mailing list