[llvm-branch-commits] [clang] [clang-tools-extra] [compiler-rt] [flang] [libc] [libcxx] [lld] [lldb] [llvm] [mlir] [polly] [AMDGPU] Split large raw buffer offsets to SOFFSET where allowed (PR #207796)
Krzysztof Drewniak via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Jul 6 11:50:29 PDT 2026
=?utf-8?q?Ömer_Sinan_Ağacan?= <omeragacan at gmail.com>,Nathan Corbyn
<n_corbyn at apple.com>,Christopher Ferris
<cferris1000 at users.noreply.github.com>,Matt Arsenault
<Matthew.Arsenault at amd.com>,Kamlesh Kumar <kamleshbhalui at gmail.com>,Aliaksei
Urbanski <aliaksei.urbanski at gmail.com>,Alex Strelnikov <strel at google.com>,Louis
Dionne <ldionne.2 at gmail.com>,Michael Jones <michaelrj at google.com>,Alex
Langford <alangford at apple.com>,Andrei Elovikov <andrei.elovikov at sifive.com>,Krzysztof
Drewniak <Krzysztof.Drewniak at amd.com>,Matt Arsenault
<Matthew.Arsenault at amd.com>,Krzysztof Drewniak <Krzysztof.Drewniak at amd.com>,Zhen
Wang <zhenw at nvidia.com>,Ken Matsui
<26405363+ken-matsui at users.noreply.github.com>,Simon Pilgrim
<llvm-dev at redking.me.uk>,Quentin Colombet <quentin.colombet at gmail.com>,Krzysztof
Drewniak <Krzysztof.Drewniak at amd.com>,Krzysztof Drewniak
<Krzysztof.Drewniak at amd.com>
Message-ID:
In-Reply-To: <llvm.org/llvm/llvm-project/pull/207796 at github.com>
https://github.com/krzysz00 updated https://github.com/llvm/llvm-project/pull/207796
>From 6575e395e3f37f4bea17092f612b37649d319041 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Mon, 6 Jul 2026 08:22:22 -0700
Subject: [PATCH 01/46] [ELF] Reuse SHT_GROUP selection verdicts in
initializeSections. NFC (#207437)
For each SHT_GROUP section, the parallel initializeSections re-derives
what the serial parse() already decided
(https://reviews.llvm.org/D130810).
Record the kept group section indices during parse() so that we can save
the work (xxh3 string hash and DenseMap lookup) in the parallel
initializeSections().
In a clang-relassert --threads=8 benchmark, "Initialize sections"
decreases from 39.0 to 30.6.
---
lld/ELF/InputFiles.cpp | 18 ++++++++++--------
lld/ELF/InputFiles.h | 4 ++++
2 files changed, 14 insertions(+), 8 deletions(-)
diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp
index 2f544f0fe0958..12d09213c9d2d 100644
--- a/lld/ELF/InputFiles.cpp
+++ b/lld/ELF/InputFiles.cpp
@@ -604,6 +604,7 @@ template <class ELFT> void ObjFile<ELFT>::parse(bool ignoreComdats) {
.try_emplace(CachedHashStringRef(signature), this)
.second;
if (keepGroup) {
+ keptGroups.push_back(i);
if (!ctx.arg.resolveGroups)
sections[i] = createInputSection(
i, sec, check(obj.getSectionName(sec, shstrtab)));
@@ -769,6 +770,8 @@ void ObjFile<ELFT>::initializeSections(bool ignoreComdats,
StringRef shstrtab = CHECK2(obj.getSectionStringTable(objSections), this);
uint64_t size = objSections.size();
SmallVector<ArrayRef<Elf_Word>, 0> selectedGroups;
+ ArrayRef<uint32_t> keptGroups = this->keptGroups;
+ size_t keptIdx = 0;
AArch64BuildAttrSubsections aarch64BAsubSections;
bool hasAArch64BuildAttributes = false;
for (size_t i = 0; i != size; ++i) {
@@ -826,14 +829,13 @@ void ObjFile<ELFT>::initializeSections(bool ignoreComdats,
case SHT_GROUP: {
if (!ctx.arg.relocatable)
sections[i] = &InputSection::discarded;
- StringRef signature =
- cantFail(this->getELFSyms<ELFT>()[sec.sh_info].getName(stringTable));
- ArrayRef<Elf_Word> entries =
- cantFail(obj.template getSectionContentsAsArray<Elf_Word>(sec));
- if ((entries[0] & GRP_COMDAT) == 0 || ignoreComdats ||
- ctx.symtab->comdatGroups.find(CachedHashStringRef(signature))
- ->second == this)
- selectedGroups.push_back(entries);
+ // Use the verdict parse() recorded for this group instead of repeating
+ // the signature hashing and comdatGroups lookup.
+ while (keptIdx != keptGroups.size() && keptGroups[keptIdx] < i)
+ ++keptIdx;
+ if (keptIdx != keptGroups.size() && keptGroups[keptIdx] == i)
+ selectedGroups.push_back(
+ cantFail(obj.template getSectionContentsAsArray<Elf_Word>(sec)));
break;
}
case SHT_SYMTAB_SHNDX:
diff --git a/lld/ELF/InputFiles.h b/lld/ELF/InputFiles.h
index aef599102ecfc..0ded9b2fa38e2 100644
--- a/lld/ELF/InputFiles.h
+++ b/lld/ELF/InputFiles.h
@@ -315,6 +315,10 @@ template <class ELFT> class ObjFile : public ELFFileBase {
// The following variable contains the contents of .symtab_shndx.
// If the section does not exist (which is common), the array is empty.
ArrayRef<Elf_Word> shndxTable;
+
+ // Section indices of kept SHT_GROUP sections, recorded by parse() in
+ // ascending order, to be used by the parallel initializeSections().
+ SmallVector<uint32_t, 0> keptGroups;
};
class BitcodeFile : public InputFile {
>From 39505025e42479ebfde4600df00a80f2fe3625f7 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn at outlook.com>
Date: Mon, 6 Jul 2026 10:23:13 -0500
Subject: [PATCH 02/46] [ClangLinkerWrapper] Use discrete steps in verbose mode
(#204186)
Summary:
One persistent problem with the linker wrapper flow is that it was more
difficult to reuse as a script than the previous flow. This is because
it did a lot of work internally. In the past we moved al ot of this into
dedicated LLVM tools, so now it's possible to simply use these tools
instead.
This PR changes the verbose mode handling to defer steps to tools rather
than doing it internally. This allows users to use verbose printing and
can copy/paste the results to re-run the steps.
---
clang/docs/ClangLinkerWrapper.rst | 51 +++--
.../linker-wrapper-verbose.c | 94 ++++++++
.../ClangLinkerWrapper.cpp | 203 +++++++++++++++++-
3 files changed, 318 insertions(+), 30 deletions(-)
create mode 100644 clang/test/OffloadTools/clang-linker-wrapper/linker-wrapper-verbose.c
diff --git a/clang/docs/ClangLinkerWrapper.rst b/clang/docs/ClangLinkerWrapper.rst
index 3637bdb848273..555d123372f29 100644
--- a/clang/docs/ClangLinkerWrapper.rst
+++ b/clang/docs/ClangLinkerWrapper.rst
@@ -57,7 +57,9 @@ only for the linker wrapper will be forwarded to the wrapped linker job.
--save-temps Save intermediate results
--sysroot<value> Set the system root
--verbose Verbose output from tools
- --v Display the version number and exit
+ -v
+ --wrapper-verbose Verbose output from the linker-wrapper
+ --version Display the version number and exit
-- The separator for the wrapped linker arguments
The linker wrapper will generate the appropriate runtime calls to register the
@@ -97,34 +99,41 @@ The linker wrapper performs a lot of steps internally, such as input matching,
symbol resolution, and image registration. This makes it difficult to debug in
some scenarios. The behavior of the linker-wrapper is controlled mostly through
metadata, described in `clang documentation
-<https://clang.llvm.org/docs/OffloadingDesign.html>`_. Intermediate output can
-be obtained from the linker-wrapper using the ``--save-temps`` flag. These files
-can then be modified.
+<https://clang.llvm.org/docs/OffloadingDesign.html>`_.
+
+The individual tool invocations the wrapper performs can be printed with the
+``--wrapper-verbose`` flag, and the intermediate files they operate on can be
+kept with ``--save-temps``. When both are enabled the wrapper emits a
+self-contained sequence of commands that reproduce its output. The example below
+shows the sequence for a single OpenMP image.
.. code-block:: sh
$> clang openmp.c -fopenmp --offload-arch=gfx90a -c
- $> clang openmp.o -fopenmp --offload-arch=gfx90a -Wl,--save-temps
- $> ; Modify temp files.
- $> llvm-objcopy --update-section=.llvm.offloading=out.bc openmp.o
+ $> clang openmp.o -fopenmp --offload-arch=gfx90a -Wl,--wrapper-verbose -Wl,--save-temps
-Doing this will allow you to override one of the input files by replacing its
-embedded offloading metadata with a user-modified version. However, this will be
-more difficult when there are multiple input files. For a very large hammer, the
-``--override-image=<kind>=<file>`` flag can be used.
+ # 1. Extract each embedded device image from the host object.
+ llvm-offload-binary openmp.o --image=kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx90a,file=openmp.gfx90a.o
-In the following example, we use the ``--save-temps`` to obtain the LLVM-IR just
-before running the backend. We then modify it to test altered behavior, and then
-compile it to a binary. This can then be passed to the linker-wrapper which will
-then ignore all embedded metadata and use the provided image as if it were the
-result of the device linking phase.
+ # 2. Link the extracted image for the device target.
+ clang --target=amdgcn-amd-amdhsa -mcpu=gfx90a openmp.gfx90a.o -o openmp.gfx90a.img <...>
-.. code-block:: sh
+ # 3. Bundle the linked image back into the offloading binary format.
+ llvm-offload-binary -o openmp.gfx90a.offload --image=file=openmp.gfx90a.img,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx90a
+
+ # 4. Generate the host runtime registration code for the bundled images.
+ llvm-offload-wrapper --kind=openmp --triple=x86_64-unknown-linux-gnu -o openmp.wrapper.bc openmp.gfx90a.offload
+
+ # 5. Compile the registration code into a host object.
+ clang --target=x86_64-unknown-linux-gnu -c -fPIC -o openmp.wrapper.o openmp.wrapper.bc
+
+ # 6. Link the host objects with the registration code into the executable.
+ ld.lld openmp.host.o openmp.wrapper.o -o a.out <...>
+
+To replace the output of a single stage, edit the relevant intermediate file and
+re-run the remaining commands. To bypass the device link entirely and substitute
+a pre-built image, use the ``--override-image=<kind>=<file>`` flag.
- $> clang openmp.c -fopenmp --offload-arch=gfx90a -Wl,--save-temps
- $> ; Modify temp files.
- $> clang --target=amdgcn-amd-amdhsa -mcpu=gfx90a -nogpulib out.bc -o a.out
- $> clang openmp.c -fopenmp --offload-arch=gfx90a -Wl,--override-image=openmp=a.out
Example
=======
diff --git a/clang/test/OffloadTools/clang-linker-wrapper/linker-wrapper-verbose.c b/clang/test/OffloadTools/clang-linker-wrapper/linker-wrapper-verbose.c
new file mode 100644
index 0000000000000..54bd856746fd3
--- /dev/null
+++ b/clang/test/OffloadTools/clang-linker-wrapper/linker-wrapper-verbose.c
@@ -0,0 +1,94 @@
+// REQUIRES: x86-registered-target
+
+// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.elf.o
+
+//
+// For OpenMP everything goes through the LLVM offloading binary type.
+//
+// RUN: llvm-offload-binary -o %t.out \
+// RUN: --image=file=%t.elf.o,kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_70 \
+// RUN: --image=file=%t.elf.o,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx90a
+// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out
+// RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --wrapper-verbose --save-temps --dry-run \
+// RUN: --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=OPENMP
+
+// OPENMP: llvm-offload-binary{{.*}} {{.*}}.o --image=kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_70,file={{.*}}.o --image=kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx90a,file={{.*}}.o
+// OPENMP: clang{{.*}} --target=nvptx64-nvidia-cuda -march=sm_70
+// OPENMP: clang{{.*}} --target=amdgcn-amd-amdhsa -mcpu=gfx90a
+// OPENMP: llvm-offload-binary{{.*}} -o {{.*}}.offload --image=file={{.*}}.img,kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_70
+// OPENMP: llvm-offload-binary{{.*}} -o {{.*}}.offload --image=file={{.*}}.img,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx90a
+// OPENMP: llvm-offload-wrapper{{.*}} --kind=openmp --triple=x86_64-unknown-linux-gnu -o [[BC:.*]].bc {{.*}}.offload {{.*}}.offload
+// OPENMP: clang{{.*}} --no-default-config --target=x86_64-unknown-linux-gnu -c -fPIC -o {{.*}}.openmp.image.wrapper{{.*}}.o [[BC]].bc
+
+//
+// The '--relocatable' flag is forwarded to the wrapper tool for OpenMP.
+//
+// RUN: llvm-offload-binary -o %t.out \
+// RUN: --image=file=%t.elf.o,kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_70
+// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out
+// RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --wrapper-verbose --save-temps --dry-run \
+// RUN: --linker-path=/usr/bin/ld -r %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=RELOCATABLE
+
+// RELOCATABLE: llvm-offload-wrapper{{.*}} --kind=openmp --triple=x86_64-unknown-linux-gnu -o {{.*}}.bc --relocatable {{.*}}.offload
+
+//
+// For CUDA the device images are combined with 'fatbinary'.
+//
+// RUN: llvm-offload-binary -o %t.out \
+// RUN: --image=file=%t.elf.o,kind=cuda,triple=nvptx64-nvidia-cuda,arch=sm_70 \
+// RUN: --image=file=%t.elf.o,kind=cuda,triple=nvptx64-nvidia-cuda,arch=sm_52
+// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out
+// RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --wrapper-verbose --save-temps --dry-run \
+// RUN: --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=CUDA
+
+// CUDA: llvm-offload-binary{{.*}} {{.*}}.o --image=kind=cuda,triple=nvptx64-nvidia-cuda,arch=sm_70,file={{.*}}.o --image=kind=cuda,triple=nvptx64-nvidia-cuda,arch=sm_52,file={{.*}}.o
+// CUDA: clang{{.*}} --target=nvptx64-nvidia-cuda -march=sm_70
+// CUDA: clang{{.*}} --target=nvptx64-nvidia-cuda -march=sm_52
+// CUDA: fatbinary{{.*}}--create [[FB:.*]].fatbin {{.*}}--image3=kind=elf,sm=70{{.*}}--image3=kind=elf,sm=52
+// CUDA: llvm-offload-wrapper{{.*}} --kind=cuda --triple=x86_64-unknown-linux-gnu -o [[BC:.*]].bc [[FB]].fatbin
+// CUDA: clang{{.*}} --no-default-config --target=x86_64-unknown-linux-gnu -c -fPIC -o {{.*}}.cuda.image.wrapper{{.*}}.o [[BC]].bc
+
+//
+// For HIP the device images are combined with 'clang-offload-bundler'.
+//
+// RUN: llvm-offload-binary -o %t.out \
+// RUN: --image=file=%t.elf.o,kind=hip,triple=amdgcn-amd-amdhsa,arch=gfx90a \
+// RUN: --image=file=%t.elf.o,kind=hip,triple=amdgcn-amd-amdhsa,arch=gfx908
+// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out
+// RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --wrapper-verbose --save-temps --dry-run \
+// RUN: --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=HIP
+
+// HIP: llvm-offload-binary{{.*}} {{.*}}.o --image=kind=hip,triple=amdgcn-amd-amdhsa,arch=gfx90a,file={{.*}}.o --image=kind=hip,triple=amdgcn-amd-amdhsa,arch=gfx908,file={{.*}}.o
+// HIP: clang{{.*}} --target=amdgcn-amd-amdhsa -mcpu=gfx90a
+// HIP: clang{{.*}} --target=amdgcn-amd-amdhsa -mcpu=gfx908
+// HIP: clang-offload-bundler{{.*}}-targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa--gfx90a,hip-amdgcn-amd-amdhsa--gfx908{{.*}}-output=[[FB:.*]].hipfb
+// HIP: llvm-offload-wrapper{{.*}} --kind=hip --triple=x86_64-unknown-linux-gnu -o [[BC:.*]].bc [[FB]].hipfb
+// HIP: clang{{.*}} --no-default-config --target=x86_64-unknown-linux-gnu -c -fPIC -o {{.*}}.hip.image.wrapper{{.*}}.o [[BC]].bc
+
+//
+// For SYCL the device image is linked with 'clang --sycl-link' and wrapped
+// directly with 'llvm-offload-wrapper --kind=sycl'.
+//
+// RUN: llvm-offload-binary -o %t.out \
+// RUN: --image=file=%t.elf.o,kind=sycl,triple=spirv64-unknown-unknown,arch=generic
+// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out
+// RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --wrapper-verbose --save-temps --dry-run \
+// RUN: --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=SYCL
+
+// SYCL: llvm-offload-binary{{.*}} {{.*}}.o --image=kind=sycl,triple=spirv64-unknown-unknown,arch=generic,file={{.*}}.o
+// SYCL: clang{{.*}} --target=spirv64-unknown-unknown {{.*}} --sycl-link {{.*}}-triple=spirv64-unknown-unknown{{.*}}-arch=
+// SYCL: llvm-offload-wrapper{{.*}} --kind=sycl --triple=x86_64-unknown-linux-gnu -o [[BC:.*]].bc {{.*}}.img
+// SYCL: clang{{.*}} --no-default-config --target=x86_64-unknown-linux-gnu -c -fPIC -o {{.*}}.sycl.image.wrapper{{.*}}.o [[BC]].bc
+
+//
+// Images pulled from a static archive are extracted from the archive path and
+// singled out by their member name so the replayed command is reproducible.
+//
+// RUN: llvm-offload-binary -o %t.out \
+// RUN: --image=file=%t.elf.o,kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_70
+// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out
+// RUN: rm -f %t.a && llvm-ar rcs %t.a %t.o
+// RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --wrapper-verbose --save-temps --dry-run \
+// RUN: --should-extract=sm_70 --linker-path=/usr/bin/ld %t.a -o a.out 2>&1 | FileCheck %s --check-prefix=ARCHIVE
+
+// ARCHIVE: llvm-offload-binary{{.*}} {{.*}}.a --image=kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_70,member={{.*}}.o,file={{.*}}.o
diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
index 76be0f776e561..5abda37ada7fb 100644
--- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
+++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
@@ -710,6 +710,67 @@ Expected<StringRef> compileModule(Module &M, OffloadKind Kind) {
return *TempFileOrErr;
}
+/// Performs the wrapping stage with individual tool invocations for verbose
+/// printing.
+Expected<StringRef>
+wrapDeviceImagesVerbose(ArrayRef<std::unique_ptr<MemoryBuffer>> Buffers,
+ const ArgList &Args, OffloadKind Kind) {
+ Expected<std::string> WrapperPath = findProgram(
+ "llvm-offload-wrapper", {getExecutableDir("llvm-offload-wrapper")});
+ if (!WrapperPath)
+ return WrapperPath.takeError();
+
+ llvm::Triple Triple(
+ Args.getLastArgValue(OPT_host_triple_EQ, sys::getDefaultTargetTriple()));
+
+ // Generate the runtime registration bitcode from the bundled images.
+ auto BitcodeOrErr = createOutputFile(
+ ExecutableName + "." + getOffloadKindName(Kind) + ".image.wrapper", "bc");
+ if (!BitcodeOrErr)
+ return BitcodeOrErr.takeError();
+
+ SmallVector<StringRef> WrapperArgs = {
+ *WrapperPath,
+ Args.MakeArgString("--kind=" + getOffloadKindName(Kind)),
+ Args.MakeArgString("--triple=" + Triple.getTriple()),
+ "-o",
+ *BitcodeOrErr,
+ };
+ if (Kind == OFK_OpenMP && Args.hasArg(OPT_relocatable))
+ WrapperArgs.push_back("--relocatable");
+ for (const auto &Buffer : Buffers)
+ WrapperArgs.push_back(Buffer->getBufferIdentifier());
+
+ if (Error Err = executeCommands(*WrapperPath, WrapperArgs))
+ return std::move(Err);
+
+ // Compile the generated registration bitcode into a host object.
+ Expected<std::string> ClangPath =
+ findProgram("clang", {getExecutableDir("clang")});
+ if (!ClangPath)
+ return ClangPath.takeError();
+
+ auto ObjectOrErr = createOutputFile(
+ ExecutableName + "." + getOffloadKindName(Kind) + ".image.wrapper", "o");
+ if (!ObjectOrErr)
+ return ObjectOrErr.takeError();
+
+ const StringRef ClangArgs[] = {
+ *ClangPath,
+ "--no-default-config",
+ Args.MakeArgString("--target=" + Triple.getTriple()),
+ "-c",
+ "-fPIC",
+ "-o",
+ *ObjectOrErr,
+ *BitcodeOrErr,
+ };
+ if (Error Err = executeCommands(*ClangPath, ClangArgs))
+ return std::move(Err);
+
+ return *ObjectOrErr;
+}
+
/// Creates the object file containing the device image and runtime
/// registration code from the device images stored in \p Images.
Expected<StringRef>
@@ -717,6 +778,10 @@ wrapDeviceImages(ArrayRef<std::unique_ptr<MemoryBuffer>> Buffers,
const ArgList &Args, OffloadKind Kind) {
llvm::TimeTraceScope TimeScope("Wrap bundled images");
+ // We use the discrete tools if we are in verbose mode with '--save-temps'.
+ if (Verbose && SaveTemps && !Args.hasArg(OPT_print_wrapped_module))
+ return wrapDeviceImagesVerbose(Buffers, Args, Kind);
+
SmallVector<ArrayRef<char>, 4> BuffersToWrap;
for (const auto &Buffer : Buffers)
BuffersToWrap.emplace_back(
@@ -779,6 +844,50 @@ wrapDeviceImages(ArrayRef<std::unique_ptr<MemoryBuffer>> Buffers,
return *FileOrErr;
}
+/// Perform the OpenMP bundling with 'llvm-offload-binary' in verbose mode.
+Expected<SmallVector<std::unique_ptr<MemoryBuffer>>>
+bundleOpenMPVerbose(ArrayRef<OffloadingImage> Images) {
+ Expected<std::string> OffloadBinaryPath = findProgram(
+ "llvm-offload-binary", {getExecutableDir("llvm-offload-binary")});
+ if (!OffloadBinaryPath)
+ return OffloadBinaryPath.takeError();
+
+ BumpPtrAllocator Alloc;
+ StringSaver Saver(Alloc);
+ SmallVector<std::unique_ptr<MemoryBuffer>> Buffers;
+ for (const OffloadingImage &Image : Images) {
+ StringRef ImageFile = Image.Image->getBufferIdentifier();
+ auto BinaryOrErr =
+ createOutputFile(sys::path::stem(ImageFile) + "." +
+ getOffloadKindName(Image.TheOffloadKind),
+ "offload");
+ if (!BinaryOrErr)
+ return BinaryOrErr.takeError();
+
+ std::string ImageArg = ("--image=file=" + ImageFile +
+ ",kind=" + getOffloadKindName(Image.TheOffloadKind))
+ .str();
+ for (const auto &[Key, Value] : Image.StringData)
+ ImageArg += ("," + Key + "=" + Value).str();
+
+ SmallVector<StringRef> CmdArgs = {*OffloadBinaryPath, "-o", *BinaryOrErr,
+ Saver.save(ImageArg)};
+ if (Error Err = executeCommands(*OffloadBinaryPath, CmdArgs))
+ return std::move(Err);
+
+ auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(*BinaryOrErr);
+ if (std::error_code EC = BufferOrErr.getError()) {
+ if (DryRun)
+ BufferOrErr = MemoryBuffer::getMemBuffer("", *BinaryOrErr);
+ else
+ return createFileError(*BinaryOrErr, EC);
+ }
+ Buffers.emplace_back(std::move(*BufferOrErr));
+ }
+
+ return std::move(Buffers);
+}
+
Expected<SmallVector<std::unique_ptr<MemoryBuffer>>>
bundleOpenMP(ArrayRef<OffloadingImage> Images) {
SmallVector<std::unique_ptr<MemoryBuffer>> Buffers;
@@ -796,7 +905,8 @@ bundleSYCL(ArrayRef<OffloadingImage> Images) {
// clang-sycl-linker packs outputs into one binary blob. Therefore, it is
// passed to Offload Wrapper as is.
StringRef S(Image.Image->getBufferStart(), Image.Image->getBufferSize());
- Buffers.emplace_back(MemoryBuffer::getMemBufferCopy(S));
+ Buffers.emplace_back(
+ MemoryBuffer::getMemBufferCopy(S, Image.Image->getBufferIdentifier()));
}
return std::move(Buffers);
@@ -817,8 +927,12 @@ bundleCuda(ArrayRef<OffloadingImage> Images, const ArgList &Args) {
llvm::MemoryBuffer::getFileOrSTDIN(*FileOrErr);
SmallVector<std::unique_ptr<MemoryBuffer>> Buffers;
- if (std::error_code EC = ImageOrError.getError())
- return createFileError(*FileOrErr, EC);
+ if (std::error_code EC = ImageOrError.getError()) {
+ if (DryRun)
+ ImageOrError = MemoryBuffer::getMemBuffer("", *FileOrErr);
+ else
+ return createFileError(*FileOrErr, EC);
+ }
Buffers.emplace_back(std::move(*ImageOrError));
return std::move(Buffers);
@@ -840,8 +954,12 @@ bundleHIP(ArrayRef<OffloadingImage> Images, const ArgList &Args) {
llvm::MemoryBuffer::getFileOrSTDIN(*FileOrErr);
SmallVector<std::unique_ptr<MemoryBuffer>> Buffers;
- if (std::error_code EC = ImageOrError.getError())
- return createFileError(*FileOrErr, EC);
+ if (std::error_code EC = ImageOrError.getError()) {
+ if (DryRun)
+ ImageOrError = MemoryBuffer::getMemBuffer("", *FileOrErr);
+ else
+ return createFileError(*FileOrErr, EC);
+ }
Buffers.emplace_back(std::move(*ImageOrError));
return std::move(Buffers);
@@ -855,7 +973,8 @@ bundleLinkedOutput(ArrayRef<OffloadingImage> Images, const ArgList &Args,
llvm::TimeTraceScope TimeScope("Bundle linked output");
switch (Kind) {
case OFK_OpenMP:
- return bundleOpenMP(Images);
+ return (Verbose && SaveTemps) ? bundleOpenMPVerbose(Images)
+ : bundleOpenMP(Images);
case OFK_SYCL:
return bundleSYCL(Images);
case OFK_Cuda:
@@ -1027,7 +1146,7 @@ linkAndWrapDeviceFiles(ArrayRef<SmallVector<OffloadFile>> LinkerInputFiles,
llvm::MemoryBuffer::getFileOrSTDIN(*OutputOrErr);
if (std::error_code EC = FileOrErr.getError()) {
if (DryRun)
- FileOrErr = MemoryBuffer::getMemBuffer("");
+ FileOrErr = MemoryBuffer::getMemBuffer("", *OutputOrErr);
else
return createFileError(*OutputOrErr, EC);
}
@@ -1151,6 +1270,63 @@ std::optional<std::string> searchLibrary(StringRef Input, StringRef Root,
return searchLibraryBaseName(Input, Root, SearchPaths);
}
+/// In verbose mode we need to replay the extracted files so the user can
+/// reproduce the generated. This only prints the steps that would result in the
+/// same output files given the input.
+Error emitExtractCommands(
+ ArrayRef<SmallVector<OffloadFile>> InputsForTarget,
+ const DenseMap<StringRef, StringRef> &SourceForImage) {
+ Expected<std::string> OffloadBinaryPath = findProgram(
+ "llvm-offload-binary", {getExecutableDir("llvm-offload-binary")});
+ if (!OffloadBinaryPath)
+ return OffloadBinaryPath.takeError();
+
+ BumpPtrAllocator Alloc;
+ StringSaver Saver(Alloc);
+ MapVector<StringRef, SmallVector<StringRef>> Commands;
+ DenseSet<StringRef> Seen;
+ for (const auto &Input : InputsForTarget) {
+ for (const OffloadFile &File : Input) {
+ const OffloadBinary &Binary = *File.getBinary();
+ StringRef Identifier = Binary.getMemoryBufferRef().getBufferIdentifier();
+ StringRef Source = SourceForImage.lookup(Identifier);
+ if (Source.empty())
+ Source = Identifier;
+
+ auto OutputOrErr =
+ createOutputFile(sys::path::stem(Identifier) + "-" +
+ Binary.getTriple() + "-" + Binary.getArch(),
+ "o");
+ if (!OutputOrErr)
+ return OutputOrErr.takeError();
+
+ std::string ImageArg =
+ ("--image=kind=" + getOffloadKindName(Binary.getOffloadKind()) +
+ ",triple=" + Binary.getTriple())
+ .str();
+ if (!Binary.getArch().empty())
+ ImageArg += (",arch=" + Binary.getArch()).str();
+ file_magic Magic;
+ if (!identify_magic(Source, Magic) && Magic == file_magic::archive)
+ ImageArg += (",member=" + sys::path::filename(Identifier)).str();
+ ImageArg += (",file=" + *OutputOrErr).str();
+
+ // Shared images only need to be extracted once per source.
+ StringRef SavedImage = Saver.save(ImageArg);
+ if (!Seen.insert(Saver.save(Source + "\x01" + SavedImage)).second)
+ continue;
+ Commands[Source].push_back(SavedImage);
+ }
+ }
+
+ for (const auto &[Source, Images] : Commands) {
+ SmallVector<StringRef> CmdArgs = {*OffloadBinaryPath, Source};
+ llvm::append_range(CmdArgs, Images);
+ printCommands(CmdArgs);
+ }
+ return Error::success();
+}
+
/// Search the input files and libraries for embedded device offloading code
/// and add it to the list of files to be linked. Files coming from static
/// libraries are only added to the input if they are used by an existing
@@ -1175,6 +1351,7 @@ getDeviceInput(const ArgList &Args) {
bool WholeArchive = Args.hasArg(OPT_wholearchive_flag);
SmallVector<OffloadFile> ObjectFilesToExtract;
SmallVector<OffloadFile> ArchiveFilesToExtract;
+ DenseMap<StringRef, StringRef> SourceForImage;
for (const opt::Arg *Arg : Args.filtered(
OPT_INPUT, OPT_library, OPT_whole_archive, OPT_no_whole_archive)) {
if (Arg->getOption().matches(OPT_whole_archive) ||
@@ -1209,6 +1386,10 @@ getDeviceInput(const ArgList &Args) {
return std::move(Err);
for (auto &Binary : Binaries) {
+ if (Verbose && SaveTemps)
+ SourceForImage.try_emplace(
+ Binary.getBinary()->getMemoryBufferRef().getBufferIdentifier(),
+ Saver.save(StringRef(*Filename)));
if (identify_magic(Buffer.getBuffer()) == file_magic::archive &&
!WholeArchive)
ArchiveFilesToExtract.emplace_back(std::move(Binary));
@@ -1238,8 +1419,8 @@ getDeviceInput(const ArgList &Args) {
}
llvm::DenseSet<StringRef> ShouldExtract;
- for (auto &Arg : Args.getAllArgValues(OPT_should_extract))
- ShouldExtract.insert(Arg);
+ for (StringRef Arg : Args.getAllArgValues(OPT_should_extract))
+ ShouldExtract.insert(Saver.save(Arg));
// We only extract archive members from the fat binary if we find a used or
// requested target. Unlike normal static archive handling, we just extract
@@ -1271,6 +1452,10 @@ getDeviceInput(const ArgList &Args) {
for (auto &[ID, Input] : InputFiles)
InputsForTarget.emplace_back(std::move(Input));
+ if (Verbose && SaveTemps)
+ if (Error Err = emitExtractCommands(InputsForTarget, SourceForImage))
+ return std::move(Err);
+
return std::move(InputsForTarget);
}
>From b8c1776668c217a3ddfa18a7668a75245d08b528 Mon Sep 17 00:00:00 2001
From: Mohsen Rahmati <mohsenrahmati at icloud.com>
Date: Mon, 6 Jul 2026 12:26:40 -0300
Subject: [PATCH 03/46] [Clang] Add constant evaluation support for x86 psadbw
(#169253)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This patch adds constant evaluation support for the x86 PSADBW and PSADBW128 intrinsics in Clang’s constexpr interpreter.
---
clang/include/clang/Basic/BuiltinsX86.td | 17 +++------
clang/lib/AST/ByteCode/InterpBuiltin.cpp | 44 ++++++++++++++++++++++
clang/lib/AST/ExprConstant.cpp | 32 ++++++++++++++++
clang/lib/Headers/avx2intrin.h | 7 ++--
clang/lib/Headers/avx512bwintrin.h | 8 ++--
clang/lib/Headers/emmintrin.h | 6 +--
clang/lib/Headers/xmmintrin.h | 9 ++---
clang/test/CodeGen/X86/avx2-builtins.c | 9 +++++
clang/test/CodeGen/X86/avx512bw-builtins.c | 22 ++++++++++-
clang/test/CodeGen/X86/mmx-builtins.c | 3 ++
clang/test/CodeGen/X86/sse2-builtins.c | 5 +++
clang/test/CodeGen/builtins-x86.c | 3 +-
12 files changed, 133 insertions(+), 32 deletions(-)
diff --git a/clang/include/clang/Basic/BuiltinsX86.td b/clang/include/clang/Basic/BuiltinsX86.td
index d9fa0e14556b9..9b7abefb75049 100644
--- a/clang/include/clang/Basic/BuiltinsX86.td
+++ b/clang/include/clang/Basic/BuiltinsX86.td
@@ -173,13 +173,13 @@ let Features = "sse2", Attributes = [NoThrow] in {
let Features = "sse2", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<128>] in {
def cvtpd2ps : X86Builtin<"_Vector<4, float>(_Vector<2, double>)">;
def cvtsd2ss : X86Builtin<"_Vector<4, float>(_Vector<4, float>, _Vector<2, double>)">;
+ def psadbw128 : X86Builtin<"_Vector<2, long long int>(_Vector<16, unsigned char>, _Vector<16, unsigned char>)">;
}
let Features = "avx512f", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<128>] in {
def cvtsd2ss_round_mask : X86Builtin<"_Vector<4, float>(_Vector<4, float>, _Vector<2, double>, _Vector<4, float>, unsigned char, _Constant int)">;
}
let Features = "sse2", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
- def psadbw128 : X86Builtin<"_Vector<2, long long int>(_Vector<16, char>, _Vector<16, char>)">;
def cvtpd2dq : X86Builtin<"_Vector<4, int>(_Vector<2, double>)">;
def cvttpd2dq : X86Builtin<"_Vector<4, int>(_Vector<2, double>)">;
def cvtsd2si : X86Builtin<"int(_Vector<2, double>)">;
@@ -574,15 +574,11 @@ let Features = "avx", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWid
def vec_set_v8si : X86Builtin<"_Vector<8, int>(_Vector<8, int>, int, _Constant int)">;
}
-let Features = "avx2", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
- def psadbw256
- : X86Builtin<
- "_Vector<4, long long int>(_Vector<32, char>, _Vector<32, char>)">;
-}
-
let Features = "avx2", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<256>] in {
def mpsadbw256 : X86Builtin<"_Vector<16, short>(_Vector<32, char>, _Vector<32, char>, _Constant char)">;
-
+ def psadbw256
+ : X86Builtin<
+ "_Vector<4, long long int>(_Vector<32, unsigned char>, _Vector<32, unsigned char>)">;
def permdf256
: X86Builtin<"_Vector<4, double>(_Vector<4, double>, _Constant int)">;
def permdi256 : X86Builtin<"_Vector<4, long long int>(_Vector<4, long long "
@@ -3178,6 +3174,7 @@ let Features = "avx512bw", Attributes = [NoThrow, Const, Constexpr] in {
let Features = "avx512bw", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<512>] in {
def palignr512 : X86Builtin<"_Vector<64, char>(_Vector<64, char>, _Vector<64, char>, _Constant int)">;
+ def psadbw512 : X86Builtin<"_Vector<8, long long int>(_Vector<64, unsigned char>, _Vector<64, unsigned char>)">;
}
let Features = "avx512bw,avx512vl", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<128>] in {
@@ -3192,10 +3189,6 @@ let Features = "avx512bw", Attributes = [NoThrow, Const, Constexpr, RequiredVect
def dbpsadbw512 : X86Builtin<"_Vector<32, short>(_Vector<64, char>, _Vector<64, char>, _Constant int)">;
}
-let Features = "avx512bw", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
- def psadbw512 : X86Builtin<"_Vector<8, long long int>(_Vector<64, char>, _Vector<64, char>)">;
-}
-
let Features = "avx512f", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<512>] in {
def compressdf512_mask : X86Builtin<"_Vector<8, double>(_Vector<8, double>, _Vector<8, double>, unsigned char)">;
def compressdi512_mask : X86Builtin<"_Vector<8, long long int>(_Vector<8, long long int>, _Vector<8, long long int>, unsigned char)">;
diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index be4ef53f04cef..70717720c592d 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -2949,6 +2949,45 @@ static bool interp__builtin_ia32_pmul(
return true;
}
+static bool interp__builtin_ia32_psadbw(InterpState &S, CodePtr OpPC,
+ const CallExpr *Call) {
+ assert(Call->getNumArgs() == 2);
+
+ const Pointer &RHS = S.Stk.pop<Pointer>();
+ const Pointer &LHS = S.Stk.pop<Pointer>();
+ const Pointer &Dst = S.Stk.peek<Pointer>();
+
+ const auto *SrcVT = Call->getArg(0)->getType()->castAs<VectorType>();
+ PrimType SrcElemT = *S.getContext().classify(SrcVT->getElementType());
+ unsigned SourceLen = SrcVT->getNumElements();
+ assert((SourceLen % 8) == 0);
+
+ const auto *DestVT = Call->getType()->castAs<VectorType>();
+ PrimType DestElemT = *S.getContext().classify(DestVT->getElementType());
+ bool DestUnsigned =
+ DestVT->getElementType()->isUnsignedIntegerOrEnumerationType();
+
+ unsigned DstElem = 0;
+ for (unsigned Lane = 0; Lane != SourceLen; Lane += 8) {
+ APInt Sum(64, 0);
+ for (unsigned I = 0; I != 8; ++I) {
+ INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
+ APSInt L = LHS.elem<T>(Lane + I).toAPSInt();
+ APSInt R = RHS.elem<T>(Lane + I).toAPSInt();
+ Sum += llvm::APIntOps::abdu(L.extOrTrunc(8), R.extOrTrunc(8)).zext(64);
+ });
+ }
+
+ INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
+ Dst.elem<T>(DstElem) = static_cast<T>(APSInt(Sum, DestUnsigned));
+ });
+ ++DstElem;
+ }
+
+ Dst.initializeAllElements();
+ return true;
+}
+
static bool interp__builtin_ia32_dbpsadbw(InterpState &S, CodePtr OpPC,
const CallExpr *Call) {
assert(Call->getNumArgs() == 3);
@@ -5514,6 +5553,11 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
(HiLHS.sext(BitWidth) * HiRHS.sext(BitWidth));
});
+ case clang::X86::BI__builtin_ia32_psadbw128:
+ case clang::X86::BI__builtin_ia32_psadbw256:
+ case clang::X86::BI__builtin_ia32_psadbw512:
+ return interp__builtin_ia32_psadbw(S, OpPC, Call);
+
case clang::X86::BI__builtin_ia32_dbpsadbw128:
case clang::X86::BI__builtin_ia32_dbpsadbw256:
case clang::X86::BI__builtin_ia32_dbpsadbw512:
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 3e24721b14535..109dc627133f2 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -12688,6 +12688,38 @@ bool VectorExprEvaluator::VisitCallExpr(const CallExpr *E) {
.extractBits(16, 1);
});
+ case clang::X86::BI__builtin_ia32_psadbw128:
+ case clang::X86::BI__builtin_ia32_psadbw256:
+ case clang::X86::BI__builtin_ia32_psadbw512: {
+ APValue SourceLHS, SourceRHS;
+ if (!EvaluateAsRValue(Info, E->getArg(0), SourceLHS) ||
+ !EvaluateAsRValue(Info, E->getArg(1), SourceRHS))
+ return false;
+
+ assert(SourceLHS.isVector() && SourceRHS.isVector());
+ unsigned SourceLen = SourceLHS.getVectorLength();
+ assert(SourceLen == SourceRHS.getVectorLength());
+ assert((SourceLen % 8) == 0);
+
+ auto *DestTy = E->getType()->castAs<VectorType>();
+ QualType DestEltTy = DestTy->getElementType();
+ bool DestUnsigned = DestEltTy->isUnsignedIntegerOrEnumerationType();
+ SmallVector<APValue, 8> ResultElements;
+ ResultElements.reserve(SourceLen / 8);
+
+ for (unsigned Lane = 0; Lane != SourceLen; Lane += 8) {
+ APInt Sum(64, 0);
+ for (unsigned I = 0; I != 8; ++I) {
+ APInt LHS = SourceLHS.getVectorElt(Lane + I).getInt().extOrTrunc(8);
+ APInt RHS = SourceRHS.getVectorElt(Lane + I).getInt().extOrTrunc(8);
+ Sum += llvm::APIntOps::abdu(LHS, RHS).zext(64);
+ }
+ ResultElements.push_back(APValue(APSInt(Sum, DestUnsigned)));
+ }
+
+ return Success(APValue(ResultElements.data(), ResultElements.size()), E);
+ }
+
case clang::X86::BI__builtin_ia32_pmaddubsw128:
case clang::X86::BI__builtin_ia32_pmaddubsw256:
case clang::X86::BI__builtin_ia32_pmaddubsw512:
diff --git a/clang/lib/Headers/avx2intrin.h b/clang/lib/Headers/avx2intrin.h
index d3ceb2327ac62..8eb22fe519e7d 100644
--- a/clang/lib/Headers/avx2intrin.h
+++ b/clang/lib/Headers/avx2intrin.h
@@ -1810,10 +1810,9 @@ _mm256_or_si256(__m256i __a, __m256i __b)
/// \param __b
/// A 256-bit integer vector.
/// \returns A 256-bit integer vector containing the result.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_sad_epu8(__m256i __a, __m256i __b)
-{
- return __builtin_ia32_psadbw256((__v32qi)__a, (__v32qi)__b);
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_sad_epu8(__m256i __a, __m256i __b) {
+ return __builtin_ia32_psadbw256((__v32qu)__a, (__v32qu)__b);
}
/// Shuffles 8-bit integers in the 256-bit integer vector \a __a according
diff --git a/clang/lib/Headers/avx512bwintrin.h b/clang/lib/Headers/avx512bwintrin.h
index 83cabc0dfb5ac..5917eeaaa38fe 100644
--- a/clang/lib/Headers/avx512bwintrin.h
+++ b/clang/lib/Headers/avx512bwintrin.h
@@ -1872,11 +1872,9 @@ _mm512_mask_permutexvar_epi16(__m512i __W, __mmask32 __M, __m512i __A,
(__v32hi)_mm512_dbsad_epu8((A), (B), (imm)), \
(__v32hi)_mm512_setzero_si512()))
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_sad_epu8 (__m512i __A, __m512i __B)
-{
- return (__m512i) __builtin_ia32_psadbw512 ((__v64qi) __A,
- (__v64qi) __B);
+static __inline__ __m512i
+ __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_sad_epu8(__m512i __A, __m512i __B) {
+ return (__m512i)__builtin_ia32_psadbw512((__v64qu)__A, (__v64qu)__B);
}
#undef __DEFAULT_FN_ATTRS512
diff --git a/clang/lib/Headers/emmintrin.h b/clang/lib/Headers/emmintrin.h
index 43c93263f015a..3f039edf87e81 100644
--- a/clang/lib/Headers/emmintrin.h
+++ b/clang/lib/Headers/emmintrin.h
@@ -2481,9 +2481,9 @@ _mm_mul_epu32(__m128i __a, __m128i __b) {
/// A 128-bit integer vector containing one of the source operands.
/// \returns A [2 x i64] vector containing the sums of the sets of absolute
/// differences between both operands.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sad_epu8(__m128i __a,
- __m128i __b) {
- return __builtin_ia32_psadbw128((__v16qi)__a, (__v16qi)__b);
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_sad_epu8(__m128i __a, __m128i __b) {
+ return __builtin_ia32_psadbw128((__v16qu)__a, (__v16qu)__b);
}
/// Subtracts the corresponding 8-bit integer values in the operands.
diff --git a/clang/lib/Headers/xmmintrin.h b/clang/lib/Headers/xmmintrin.h
index efc0e6ce47e7d..73eab9e460ca5 100644
--- a/clang/lib/Headers/xmmintrin.h
+++ b/clang/lib/Headers/xmmintrin.h
@@ -2571,11 +2571,10 @@ _mm_avg_pu16(__m64 __a, __m64 __b) {
/// \returns A 64-bit integer vector whose lower 16 bits contain the sums of the
/// sets of absolute differences between both operands. The upper bits are
/// cleared.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
-_mm_sad_pu8(__m64 __a, __m64 __b)
-{
- return __trunc64(__builtin_ia32_psadbw128((__v16qi)__zext128(__a),
- (__v16qi)__zext128(__b)));
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
+_mm_sad_pu8(__m64 __a, __m64 __b) {
+ return __trunc64(__builtin_ia32_psadbw128((__v16qu)__zext128(__a),
+ (__v16qu)__zext128(__b)));
}
#if defined(__cplusplus)
diff --git a/clang/test/CodeGen/X86/avx2-builtins.c b/clang/test/CodeGen/X86/avx2-builtins.c
index cb14d1aafedde..a8df1a279ec37 100644
--- a/clang/test/CodeGen/X86/avx2-builtins.c
+++ b/clang/test/CodeGen/X86/avx2-builtins.c
@@ -1185,6 +1185,15 @@ __m256i test_mm256_sad_epu8(__m256i x, __m256i y) {
// CHECK: call {{.*}}<4 x i64> @llvm.x86.avx2.psad.bw(<32 x i8> %{{.*}}, <32 x i8> %{{.*}})
return _mm256_sad_epu8(x, y);
}
+TEST_CONSTEXPR(match_m256i(_mm256_sad_epu8((__m256i)(__v32qu){0, 1, 2, 3, 4, 5, 6, 7,
+ 8, 9, 10, 11, 12, 13, 14, 15,
+ 16, 17, 18, 19, 20, 21, 22, 23,
+ 24, 25, 26, 27, 28, 29, 30, 31},
+ (__m256i)(__v32qu){31, 30, 29, 28, 27, 26, 25, 24,
+ 23, 22, 21, 20, 19, 18, 17, 16,
+ 15, 14, 13, 12, 11, 10, 9, 8,
+ 7, 6, 5, 4, 3, 2, 1, 0}),
+ 192ULL, 64ULL, 64ULL, 192ULL));
__m256i test_mm256_shuffle_epi8(__m256i a, __m256i b) {
// CHECK-LABEL: test_mm256_shuffle_epi8
diff --git a/clang/test/CodeGen/X86/avx512bw-builtins.c b/clang/test/CodeGen/X86/avx512bw-builtins.c
index 945789b18dd8a..427efbbb99af8 100644
--- a/clang/test/CodeGen/X86/avx512bw-builtins.c
+++ b/clang/test/CodeGen/X86/avx512bw-builtins.c
@@ -3295,8 +3295,26 @@ TEST_CONSTEXPR(match_v32hu(_mm512_maskz_dbsad_epu8((__mmask32)0xAAAAAAAA,
__m512i test_mm512_sad_epu8(__m512i __A, __m512i __B) {
// CHECK-LABEL: test_mm512_sad_epu8
// CHECK: @llvm.x86.avx512.psad.bw.512
- return _mm512_sad_epu8(__A, __B);
-}
+ return _mm512_sad_epu8(__A, __B);
+}
+TEST_CONSTEXPR(match_m512i(_mm512_sad_epu8((__m512i)(__v64qu){0, 1, 2, 3, 4, 5, 6, 7,
+ 8, 9, 10, 11, 12, 13, 14, 15,
+ 16, 17, 18, 19, 20, 21, 22, 23,
+ 24, 25, 26, 27, 28, 29, 30, 31,
+ 32, 33, 34, 35, 36, 37, 38, 39,
+ 40, 41, 42, 43, 44, 45, 46, 47,
+ 48, 49, 50, 51, 52, 53, 54, 55,
+ 56, 57, 58, 59, 60, 61, 62, 63},
+ (__m512i)(__v64qu){63, 62, 61, 60, 59, 58, 57, 56,
+ 55, 54, 53, 52, 51, 50, 49, 48,
+ 47, 46, 45, 44, 43, 42, 41, 40,
+ 39, 38, 37, 36, 35, 34, 33, 32,
+ 31, 30, 29, 28, 27, 26, 25, 24,
+ 23, 22, 21, 20, 19, 18, 17, 16,
+ 15, 14, 13, 12, 11, 10, 9, 8,
+ 7, 6, 5, 4, 3, 2, 1, 0}),
+ 448ULL, 320ULL, 192ULL, 64ULL,
+ 64ULL, 192ULL, 320ULL, 448ULL));
__mmask32 test_mm512_movepi16_mask(__m512i __A) {
// CHECK-LABEL: test_mm512_movepi16_mask
diff --git a/clang/test/CodeGen/X86/mmx-builtins.c b/clang/test/CodeGen/X86/mmx-builtins.c
index 37d6306ecdb7d..90b42ba3cf099 100644
--- a/clang/test/CodeGen/X86/mmx-builtins.c
+++ b/clang/test/CodeGen/X86/mmx-builtins.c
@@ -486,6 +486,9 @@ __m64 test_mm_sad_pu8(__m64 a, __m64 b) {
// CHECK: call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8>
return _mm_sad_pu8(a, b);
}
+TEST_CONSTEXPR(match_m64(_mm_sad_pu8((__m64)(__v8qu){0, 1, 2, 3, 4, 5, 6, 7},
+ (__m64)(__v8qu){7, 6, 5, 4, 3, 2, 1, 0}),
+ 32ULL));
__m64 test_mm_set_pi8(char a, char b, char c, char d, char e, char f, char g, char h) {
// CHECK-LABEL: test_mm_set_pi8
diff --git a/clang/test/CodeGen/X86/sse2-builtins.c b/clang/test/CodeGen/X86/sse2-builtins.c
index 2993b8bb719d6..3e36c0047baf0 100644
--- a/clang/test/CodeGen/X86/sse2-builtins.c
+++ b/clang/test/CodeGen/X86/sse2-builtins.c
@@ -1075,6 +1075,11 @@ __m128i test_mm_sad_epu8(__m128i A, __m128i B) {
// CHECK: call {{.*}}<2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> %{{.*}}, <16 x i8> %{{.*}})
return _mm_sad_epu8(A, B);
}
+TEST_CONSTEXPR(match_m128i(_mm_sad_epu8((__m128i)(__v16qu){0, 1, 2, 3, 4, 5, 6, 7,
+ 255, 254, 253, 252, 251, 250, 249, 248},
+ (__m128i)(__v16qu){7, 6, 5, 4, 3, 2, 1, 0,
+ 248, 249, 250, 251, 252, 253, 254, 255}),
+ 32ULL, 32ULL));
__m128i test_mm_set_epi8(char A, char B, char C, char D,
char E, char F, char G, char H,
diff --git a/clang/test/CodeGen/builtins-x86.c b/clang/test/CodeGen/builtins-x86.c
index 0f66d8c4e3120..f67ad13d97208 100644
--- a/clang/test/CodeGen/builtins-x86.c
+++ b/clang/test/CodeGen/builtins-x86.c
@@ -21,6 +21,7 @@ typedef float V2f __attribute__((vector_size(8)));
// 128-bit
typedef char V16c __attribute__((vector_size(16)));
+typedef unsigned char V16Uc __attribute__((vector_size(16)));
typedef signed short V8s __attribute__((vector_size(16)));
typedef unsigned short V8u __attribute__((vector_size(16)));
typedef signed int V4i __attribute__((vector_size(16)));
@@ -289,7 +290,7 @@ void f0(void) {
#ifdef USE_64
(void) __builtin_ia32_movnti64(tmp_LLip, tmp_LLi);
#endif
- tmp_V2LLi = __builtin_ia32_psadbw128(tmp_V16c, tmp_V16c);
+ tmp_V2LLi = __builtin_ia32_psadbw128((V16Uc)tmp_V16c, (V16Uc)tmp_V16c);
tmp_V4i = __builtin_ia32_cvtpd2dq(tmp_V2d);
tmp_V4f = __builtin_ia32_cvtpd2ps(tmp_V2d);
tmp_V4i = __builtin_ia32_cvttpd2dq(tmp_V2d);
>From 3ed7f90ec978ce4670779adf30fac17e87cba4fb Mon Sep 17 00:00:00 2001
From: Lucas Mellone <github.snugness349 at passinbox.com>
Date: Mon, 6 Jul 2026 17:31:30 +0200
Subject: [PATCH 04/46] [libc++][ranges] Applied [[nodiscard]] to
`elements_view` (#206589)
[[nodiscard]] should be applied to functions where discarding the return
value is most likely a correctness issue.
- https://libcxx.llvm.org/CodingGuidelines.html
- https://wg21.link/range.elements
Towards https://github.com/llvm/llvm-project/issues/172124
---------
Co-authored-by: A. Jiang <de34 at live.cn>
---
libcxx/include/__ranges/elements_view.h | 41 ++++---
.../range.elements/nodiscard.verify.cpp | 116 ++++++++++++++++++
2 files changed, 137 insertions(+), 20 deletions(-)
create mode 100644 libcxx/test/libcxx/ranges/range.adaptors/range.elements/nodiscard.verify.cpp
diff --git a/libcxx/include/__ranges/elements_view.h b/libcxx/include/__ranges/elements_view.h
index b1419f2a1dd91..33c152bbd03b4 100644
--- a/libcxx/include/__ranges/elements_view.h
+++ b/libcxx/include/__ranges/elements_view.h
@@ -77,57 +77,57 @@ class elements_view : public view_interface<elements_view<_View, _Np>> {
_LIBCPP_HIDE_FROM_ABI constexpr explicit elements_view(_View __base) : __base_(std::move(__base)) {}
- _LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
+ [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
requires copy_constructible<_View>
{
return __base_;
}
- _LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
+ [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
- _LIBCPP_HIDE_FROM_ABI constexpr auto begin()
+ [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto begin()
requires(!__simple_view<_View>)
{
return __iterator</*_Const=*/false>(ranges::begin(__base_));
}
- _LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
+ [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
requires range<const _View>
{
return __iterator</*_Const=*/true>(ranges::begin(__base_));
}
- _LIBCPP_HIDE_FROM_ABI constexpr auto end()
+ [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto end()
requires(!__simple_view<_View> && !common_range<_View>)
{
return __sentinel</*_Const=*/false>{ranges::end(__base_)};
}
- _LIBCPP_HIDE_FROM_ABI constexpr auto end()
+ [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto end()
requires(!__simple_view<_View> && common_range<_View>)
{
return __iterator</*_Const=*/false>{ranges::end(__base_)};
}
- _LIBCPP_HIDE_FROM_ABI constexpr auto end() const
+ [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto end() const
requires range<const _View>
{
return __sentinel</*_Const=*/true>{ranges::end(__base_)};
}
- _LIBCPP_HIDE_FROM_ABI constexpr auto end() const
+ [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto end() const
requires common_range<const _View>
{
return __iterator</*_Const=*/true>{ranges::end(__base_)};
}
- _LIBCPP_HIDE_FROM_ABI constexpr auto size()
+ [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto size()
requires sized_range<_View>
{
return ranges::size(__base_);
}
- _LIBCPP_HIDE_FROM_ABI constexpr auto size() const
+ [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto size() const
requires sized_range<const _View>
{
return ranges::size(__base_);
@@ -211,11 +211,11 @@ class elements_view<_View, _Np>::__iterator
requires _Const && convertible_to<iterator_t<_View>, iterator_t<_Base>>
: __current_(std::move(__i.__current_)) {}
- _LIBCPP_HIDE_FROM_ABI constexpr const iterator_t<_Base>& base() const& noexcept { return __current_; }
+ [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const iterator_t<_Base>& base() const& noexcept { return __current_; }
- _LIBCPP_HIDE_FROM_ABI constexpr iterator_t<_Base> base() && { return std::move(__current_); }
+ [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr iterator_t<_Base> base() && { return std::move(__current_); }
- _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator*() const { return __get_element(__current_); }
+ [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator*() const { return __get_element(__current_); }
_LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator++() {
++__current_;
@@ -261,7 +261,7 @@ class elements_view<_View, _Np>::__iterator
return *this;
}
- _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator[](difference_type __n) const
+ [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator[](difference_type __n) const
requires random_access_range<_Base>
{
return __get_element(__current_ + __n);
@@ -303,25 +303,26 @@ class elements_view<_View, _Np>::__iterator
return __x.__current_ <=> __y.__current_;
}
- _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(const __iterator& __x, difference_type __y)
+ [[nodiscard]] _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(const __iterator& __x, difference_type __y)
requires random_access_range<_Base>
{
return __iterator{__x} += __y;
}
- _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(difference_type __x, const __iterator& __y)
+ [[nodiscard]] _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(difference_type __x, const __iterator& __y)
requires random_access_range<_Base>
{
return __y + __x;
}
- _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator-(const __iterator& __x, difference_type __y)
+ [[nodiscard]] _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator-(const __iterator& __x, difference_type __y)
requires random_access_range<_Base>
{
return __iterator{__x} -= __y;
}
- _LIBCPP_HIDE_FROM_ABI friend constexpr difference_type operator-(const __iterator& __x, const __iterator& __y)
+ [[nodiscard]] _LIBCPP_HIDE_FROM_ABI friend constexpr difference_type
+ operator-(const __iterator& __x, const __iterator& __y)
requires sized_sentinel_for<iterator_t<_Base>, iterator_t<_Base>>
{
return __x.__current_ - __y.__current_;
@@ -365,14 +366,14 @@ class elements_view<_View, _Np>::__sentinel {
template <bool _OtherConst>
requires sized_sentinel_for<sentinel_t<_Base>, iterator_t<__maybe_const<_OtherConst, _View>>>
- _LIBCPP_HIDE_FROM_ABI friend constexpr range_difference_t<__maybe_const<_OtherConst, _View>>
+ [[nodiscard]] _LIBCPP_HIDE_FROM_ABI friend constexpr range_difference_t<__maybe_const<_OtherConst, _View>>
operator-(const __iterator<_OtherConst>& __x, const __sentinel& __y) {
return __get_current(__x) - __y.__end_;
}
template <bool _OtherConst>
requires sized_sentinel_for<sentinel_t<_Base>, iterator_t<__maybe_const<_OtherConst, _View>>>
- _LIBCPP_HIDE_FROM_ABI friend constexpr range_difference_t<__maybe_const<_OtherConst, _View>>
+ [[nodiscard]] _LIBCPP_HIDE_FROM_ABI friend constexpr range_difference_t<__maybe_const<_OtherConst, _View>>
operator-(const __sentinel& __x, const __iterator<_OtherConst>& __y) {
return __x.__end_ - __get_current(__y);
}
diff --git a/libcxx/test/libcxx/ranges/range.adaptors/range.elements/nodiscard.verify.cpp b/libcxx/test/libcxx/ranges/range.adaptors/range.elements/nodiscard.verify.cpp
new file mode 100644
index 0000000000000..088c1dba5af9d
--- /dev/null
+++ b/libcxx/test/libcxx/ranges/range.adaptors/range.elements/nodiscard.verify.cpp
@@ -0,0 +1,116 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+
+// Test the libc++ extension that std::ranges::elements_view and std::views::elements are marked as [[nodiscard]].
+
+#include <functional>
+#include <map>
+#include <ranges>
+#include <utility>
+
+struct View : std::ranges::view_interface<View> {
+ std::tuple<int, int>* begin();
+ const std::tuple<int, int>* begin() const;
+ volatile std::tuple<int, int>* end();
+ const volatile std::tuple<int, int>* end() const;
+};
+static_assert(!std::ranges::common_range<View>);
+static_assert(!std::same_as<std::ranges::iterator_t<View>, std::ranges::iterator_t<const View>>);
+static_assert(!std::same_as<std::ranges::sentinel_t<View>, std::ranges::sentinel_t<const View>>);
+
+struct CommonView : std::ranges::view_interface<View> {
+ std::tuple<int, int>* begin();
+ const std::tuple<int, int>* begin() const;
+ std::tuple<int, int>* end();
+ const std::tuple<int, int>* end() const;
+};
+static_assert(std::ranges::common_range<CommonView>);
+static_assert(!std::same_as<std::ranges::iterator_t<CommonView>, std::ranges::iterator_t<const CommonView>>);
+static_assert(!std::same_as<std::ranges::sentinel_t<CommonView>, std::ranges::sentinel_t<const CommonView>>);
+
+void test() {
+ auto v = View{} | std::views::elements<1>;
+ auto common_v = CommonView{} | std::views::elements<1>;
+
+ // [range.elements.view]
+
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ std::as_const(v).base();
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ std::move(v).base();
+
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ v.begin();
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ std::as_const(v).begin();
+
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ v.end();
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ std::as_const(v).end();
+
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ common_v.end();
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ std::as_const(common_v).end();
+
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ v.size();
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ std::as_const(v).size();
+
+ // [range.elements.iterator]
+
+ auto it = v.begin();
+ auto c_it = std::as_const(v).begin();
+
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ c_it.base();
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ std::move(it).base();
+
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ *c_it;
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ c_it[0];
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ it + 0;
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ 0 + it;
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ it - 0;
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ it - it;
+
+ // [range.elements.sentinel]
+
+ auto st = v.end();
+
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ it - st;
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ st - it;
+
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ st - c_it;
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ c_it - st;
+
+ // [range.elements.overview]
+
+ auto historical_figures = std::map{
+ std::pair{"Lovelace", 1815}, std::pair{"Turing", 1912}, std::pair{"Babbage", 1791}, std::pair{"Hamilton", 1936}};
+
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ std::views::elements<0>(historical_figures);
+
+ // expected-warning at +1 {{ignoring return value of function declared with 'nodiscard' attribute}}
+ std::views::elements<1>(historical_figures);
+}
>From 0cb5782f7ae0e859d3a649327227a3119e69cc25 Mon Sep 17 00:00:00 2001
From: Scott Linder <scott.linder at amd.com>
Date: Mon, 6 Jul 2026 11:33:03 -0400
Subject: [PATCH 05/46] [docs][AMDGPU] Correct spelling of
DW_AT_LLVM_memory_space (#207745)
---
llvm/docs/AMDGPUDwarfExtensionsForHeterogeneousDebugging.rst | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/docs/AMDGPUDwarfExtensionsForHeterogeneousDebugging.rst b/llvm/docs/AMDGPUDwarfExtensionsForHeterogeneousDebugging.rst
index 6bcfadd121815..933f6dd72723c 100644
--- a/llvm/docs/AMDGPUDwarfExtensionsForHeterogeneousDebugging.rst
+++ b/llvm/docs/AMDGPUDwarfExtensionsForHeterogeneousDebugging.rst
@@ -3687,9 +3687,9 @@ constant may have the following attributes:
3. ``DW_AT_LLVM_memory_space``
- A ``DW_AT_memory_space`` attribute with a constant value representing a source
+ A ``DW_AT_LLVM_memory_space`` attribute with a constant value representing a source
language specific DWARF memory space (see 2.14 "Memory Spaces"). If omitted,
- defaults to ``DW_MSPACE_none``.
+ defaults to ``DW_MSPACE_LLVM_none``.
A.4.2 Common Block Entries
>From d930a16182aee3a541173c58bd2061c5265ffd44 Mon Sep 17 00:00:00 2001
From: 311Volt <41923095+311Volt at users.noreply.github.com>
Date: Mon, 6 Jul 2026 17:35:25 +0200
Subject: [PATCH 06/46] [Offload] run clang-format on olCreateProgram unit
tests (#207757)
Follow-up to: https://github.com/llvm/llvm-project/pull/206462
---
offload/unittests/OffloadAPI/program/olCreateProgram.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/offload/unittests/OffloadAPI/program/olCreateProgram.cpp b/offload/unittests/OffloadAPI/program/olCreateProgram.cpp
index 7bbd2308e1d18..9177f91359784 100644
--- a/offload/unittests/OffloadAPI/program/olCreateProgram.cpp
+++ b/offload/unittests/OffloadAPI/program/olCreateProgram.cpp
@@ -65,7 +65,8 @@ TEST_P(olCreateProgramTest, ZeroSizeBinary) {
ol_program_handle_t Program = nullptr;
- // backend rejection of a binary is not guaranteed to map to a specific ol_errc_t, so we ASSERT_ANY_ERROR for now
+ // backend rejection of a binary is not guaranteed to map to a specific
+ // ol_errc_t, so we ASSERT_ANY_ERROR for now
ASSERT_ANY_ERROR(
olCreateProgram(Device, DeviceBin->getBufferStart(), 0, &Program));
ASSERT_EQ(Program, nullptr);
>From 3b57d53a792cd9e9ba848cfe6fef151a0cb6f5d9 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Mon, 6 Jul 2026 08:39:27 -0700
Subject: [PATCH 07/46] [LoopInfo] Reuse DomTree's own DFS numbering in
analyze() (#207650)
analyze() discovers loop headers with a post_order(DomRoot) walk, then
tests each back-edge candidate with DomTree.dominates(). Initial queries
use `dominatedBySlowTreeWalk` and then switch to `updateDFSNumbers`
(Euler tour technique), requiring two tree walks in total.
Optimize this with an upfront updateDFSNumbers to avoid the post-order
walk. Both post-order and reverse pre-order satisfy the
child-before-parent (bottom-up) property. This does change the order in
which siblings are visited (reverse preorder visits the last child
first; post-order visits it last), but that has no observable effect:
the sub-loop and block vectors are only populated by the separate
PopulateLoopsDFS traversal afterward, not by discovery order.
`post_order` (and `depth_first`) are also inefficient for tree
traversals:
DomTreeNodeBase doesn't expose graph numbers, so they fall back to
tracking visited nodes in a SmallPtrSet. For a tree this bookkeeping is
unnecessary, since by definition every node has exactly one parent and
can never be reached twice.
DFS numbers, of [0, 2*N), have many nullptr gaps. Simplify
updateDFSNumbers() so DFSNumOut no longer consumes its own counter tick:
DFSNumIn becomes a dense [0, N) preorder rank.
---
llvm/include/llvm/Support/GenericDomTree.h | 14 ++++++++++--
.../llvm/Support/GenericDomTreeConstruction.h | 4 ++--
.../llvm/Support/GenericLoopInfoImpl.h | 15 +++++++++----
llvm/unittests/IR/DominatorTreeTest.cpp | 22 +++++++++----------
4 files changed, 36 insertions(+), 19 deletions(-)
diff --git a/llvm/include/llvm/Support/GenericDomTree.h b/llvm/include/llvm/Support/GenericDomTree.h
index 920983e7bd800..45bc6269177c7 100644
--- a/llvm/include/llvm/Support/GenericDomTree.h
+++ b/llvm/include/llvm/Support/GenericDomTree.h
@@ -46,6 +46,8 @@ namespace llvm {
template <typename NodeT, bool IsPostDom>
class DominatorTreeBase;
+template <class BlockT, class LoopT> class LoopInfoBase;
+
namespace DomTreeBuilder {
template <typename DomTreeT>
struct SemiNCAInfo;
@@ -316,8 +318,9 @@ class DominatorTreeBase {
unsigned BlockNumberEpoch = 0;
friend struct DomTreeBuilder::SemiNCAInfo<DominatorTreeBase>;
+ template <class BlockT, class LoopT> friend class LoopInfoBase;
- public:
+public:
DominatorTreeBase() = default;
DominatorTreeBase(const DominatorTreeBase &) = delete;
@@ -382,6 +385,13 @@ class DominatorTreeBase {
}
private:
+ // For LoopInfoBase's use in deriving a reverse-preorder traversal.
+ auto nodes() const {
+ return make_filter_range(DomTreeNodes, [](const DomTreeNodeBase<NodeT> *N) {
+ return N != nullptr;
+ });
+ }
+
std::optional<unsigned> getNodeIndex(const NodeT *BB) const {
if constexpr (GraphHasNodeNumbers<NodeT *>) {
assert(BlockNumberEpoch ==
@@ -850,7 +860,7 @@ class DominatorTreeBase {
// If we visited all of the children of this node, "recurse" back up the
// stack setting the DFOutNum.
if (ChildIt == Node->end()) {
- Node->DFSNumOut = DFSNum++;
+ Node->DFSNumOut = DFSNum;
WorkStack.pop_back();
} else {
// Otherwise, recursively visit this child.
diff --git a/llvm/include/llvm/Support/GenericDomTreeConstruction.h b/llvm/include/llvm/Support/GenericDomTreeConstruction.h
index 9ac5c5444c88a..80365e72e8d07 100644
--- a/llvm/include/llvm/Support/GenericDomTreeConstruction.h
+++ b/llvm/include/llvm/Support/GenericDomTreeConstruction.h
@@ -1444,13 +1444,13 @@ template <typename DomTreeT> struct SemiNCAInfo {
return false;
}
- if (Children.back()->getDFSNumOut() + 1 != Node->getDFSNumOut()) {
+ if (Children.back()->getDFSNumOut() != Node->getDFSNumOut()) {
PrintChildrenError(Children.back(), nullptr);
return false;
}
for (size_t i = 0, e = Children.size() - 1; i != e; ++i) {
- if (Children[i]->getDFSNumOut() + 1 != Children[i + 1]->getDFSNumIn()) {
+ if (Children[i]->getDFSNumOut() != Children[i + 1]->getDFSNumIn()) {
PrintChildrenError(Children[i], Children[i + 1]);
return false;
}
diff --git a/llvm/include/llvm/Support/GenericLoopInfoImpl.h b/llvm/include/llvm/Support/GenericLoopInfoImpl.h
index 4666f4dac9cb6..5d827c1cab8c2 100644
--- a/llvm/include/llvm/Support/GenericLoopInfoImpl.h
+++ b/llvm/include/llvm/Support/GenericLoopInfoImpl.h
@@ -560,8 +560,8 @@ void PopulateLoopsDFS<BlockT, LoopT>::insertIntoLoop(BlockT *Block) {
Subloop->addBlockEntry(Block);
}
-/// Analyze LoopInfo discovers loops during a postorder DominatorTree traversal
-/// interleaved with backward CFG traversals within each subloop
+/// Analyze LoopInfo discovers loops during a reverse preorder DominatorTree
+/// traversal interleaved with backward CFG traversals within each subloop
/// (discoverAndMapSubloop). The backward traversal skips inner subloops, so
/// this part of the algorithm is linear in the number of CFG edges. Subloop and
/// Block vectors are then populated during a single forward CFG traversal
@@ -576,7 +576,6 @@ void PopulateLoopsDFS<BlockT, LoopT>::insertIntoLoop(BlockT *Block) {
/// insertions per block.
template <class BlockT, class LoopT>
void LoopInfoBase<BlockT, LoopT>::analyze(const DomTreeBase<BlockT> &DomTree) {
- // Postorder traversal of the dominator tree.
const DomTreeNodeBase<BlockT> *DomRoot = DomTree.getRootNode();
if constexpr (GraphHasNodeNumbers<const BlockT *>) {
ParentPtr = DomRoot->getBlock()->getParent();
@@ -584,8 +583,16 @@ void LoopInfoBase<BlockT, LoopT>::analyze(const DomTreeBase<BlockT> &DomTree) {
unsigned Max = GraphTraits<ParentT>::getMaxNumber(ParentPtr);
BBMap.resize(Max);
}
- for (auto DomNode : post_order(DomRoot)) {
+ // Visit dominator tree nodes in reverse preorder: like postorder, this
+ // guarantees a sub-loop is discovered before the outer loop.
+ DomTree.updateDFSNumbers();
+ SmallVector<const DomTreeNodeBase<BlockT> *, 32> PreorderNodes(
+ DomRoot->getDFSNumOut());
+ for (const DomTreeNodeBase<BlockT> *Node : DomTree.nodes())
+ PreorderNodes[Node->getDFSNumIn()] = Node;
+
+ for (const DomTreeNodeBase<BlockT> *DomNode : llvm::reverse(PreorderNodes)) {
BlockT *Header = DomNode->getBlock();
SmallVector<BlockT *, 4> Backedges;
diff --git a/llvm/unittests/IR/DominatorTreeTest.cpp b/llvm/unittests/IR/DominatorTreeTest.cpp
index 042b7f449ba89..070a7325bf28e 100644
--- a/llvm/unittests/IR/DominatorTreeTest.cpp
+++ b/llvm/unittests/IR/DominatorTreeTest.cpp
@@ -253,13 +253,13 @@ TEST(DominatorTree, Unreachable) {
// Check DFS Numbers before
DT->updateDFSNumbers();
EXPECT_EQ(DT->getNode(BB0)->getDFSNumIn(), 0UL);
- EXPECT_EQ(DT->getNode(BB0)->getDFSNumOut(), 7UL);
+ EXPECT_EQ(DT->getNode(BB0)->getDFSNumOut(), 4UL);
EXPECT_EQ(DT->getNode(BB1)->getDFSNumIn(), 1UL);
EXPECT_EQ(DT->getNode(BB1)->getDFSNumOut(), 2UL);
- EXPECT_EQ(DT->getNode(BB2)->getDFSNumIn(), 5UL);
- EXPECT_EQ(DT->getNode(BB2)->getDFSNumOut(), 6UL);
- EXPECT_EQ(DT->getNode(BB4)->getDFSNumIn(), 3UL);
- EXPECT_EQ(DT->getNode(BB4)->getDFSNumOut(), 4UL);
+ EXPECT_EQ(DT->getNode(BB2)->getDFSNumIn(), 3UL);
+ EXPECT_EQ(DT->getNode(BB2)->getDFSNumOut(), 4UL);
+ EXPECT_EQ(DT->getNode(BB4)->getDFSNumIn(), 2UL);
+ EXPECT_EQ(DT->getNode(BB4)->getDFSNumOut(), 3UL);
// Check levels before
EXPECT_EQ(DT->getNode(BB0)->getLevel(), 0U);
@@ -275,15 +275,15 @@ TEST(DominatorTree, Unreachable) {
// Check DFS Numbers after
DT->updateDFSNumbers();
EXPECT_EQ(DT->getNode(BB0)->getDFSNumIn(), 0UL);
- EXPECT_EQ(DT->getNode(BB0)->getDFSNumOut(), 9UL);
+ EXPECT_EQ(DT->getNode(BB0)->getDFSNumOut(), 5UL);
EXPECT_EQ(DT->getNode(BB1)->getDFSNumIn(), 1UL);
- EXPECT_EQ(DT->getNode(BB1)->getDFSNumOut(), 4UL);
- EXPECT_EQ(DT->getNode(BB2)->getDFSNumIn(), 7UL);
- EXPECT_EQ(DT->getNode(BB2)->getDFSNumOut(), 8UL);
+ EXPECT_EQ(DT->getNode(BB1)->getDFSNumOut(), 3UL);
+ EXPECT_EQ(DT->getNode(BB2)->getDFSNumIn(), 4UL);
+ EXPECT_EQ(DT->getNode(BB2)->getDFSNumOut(), 5UL);
EXPECT_EQ(DT->getNode(BB3)->getDFSNumIn(), 2UL);
EXPECT_EQ(DT->getNode(BB3)->getDFSNumOut(), 3UL);
- EXPECT_EQ(DT->getNode(BB4)->getDFSNumIn(), 5UL);
- EXPECT_EQ(DT->getNode(BB4)->getDFSNumOut(), 6UL);
+ EXPECT_EQ(DT->getNode(BB4)->getDFSNumIn(), 3UL);
+ EXPECT_EQ(DT->getNode(BB4)->getDFSNumOut(), 4UL);
// Check levels after
EXPECT_EQ(DT->getNode(BB0)->getLevel(), 0U);
>From b30e3c23a537ca99937181d03bb53a8fe1d54c36 Mon Sep 17 00:00:00 2001
From: "Ron Green [NVIDIA]" <rogreen at nvidia.com>
Date: Mon, 6 Jul 2026 08:40:23 -0700
Subject: [PATCH 08/46] [flang][cuda] Make the second LAUNCH_BOUNDS() operand
optional (#207273)
The minimum-blocks-per-multiprocessor (second) and maximum-blocks-per-cluster
(third) operands of the CUDA Fortran `LAUNCH_BOUNDS()` prefix are optional, as in
CUDA C++ `__launch_bounds__`. A single-operand `LAUNCH_BOUNDS(N)` is now accepted;
1, 2, or 3 integer constants are valid.
Previously `LAUNCH_BOUNDS(N)` with a single operand was rejected, and the
diagnostic path dereferenced an empty `std::optional` (`currStmtSource()`). That
handler runs during the direct statement walk in `ResolveSpecificationParts`,
where the statement source is not set, so the dereference aborted the compiler.
The diagnostic source is now derived from the prefix, and the same fix is applied
to every diagnostic in the `LAUNCH_BOUNDS()` and `CLUSTER_DIMS()` handlers.
When only the maximum-threads-per-block operand is present, the
`cuf.launch_bounds` attribute omits `minBPM` (now an optional parameter) and no
`nvvm.minctasm` attribute is generated.
Assisted-by: AI
---
.../Dialect/CUF/Attributes/CUFAttr.td | 2 +-
flang/lib/Lower/CallInterface.cpp | 12 ++++--
.../CUDA/CUFDeviceFuncTransform.cpp | 6 ++-
flang/lib/Semantics/resolve-names.cpp | 26 +++++++++----
.../Fir/CUDA/cuda-device-func-transform.mlir | 11 ++++++
flang/test/Lower/CUDA/cuda-proc-attribute.cuf | 3 ++
flang/test/Semantics/CUDA/cuf08.cuf | 5 +--
flang/test/Semantics/CUDA/cuf31.cuf | 38 +++++++++++++++++++
8 files changed, 85 insertions(+), 18 deletions(-)
create mode 100644 flang/test/Semantics/CUDA/cuf31.cuf
diff --git a/flang/include/flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.td b/flang/include/flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.td
index 8e2b546725271..af21fe3d05fa4 100644
--- a/flang/include/flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.td
+++ b/flang/include/flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.td
@@ -62,7 +62,7 @@ def cuf_LaunchBoundsAttr : cuf_Attr<"LaunchBounds"> {
let parameters = (ins
"mlir::IntegerAttr":$maxTPB,
- "mlir::IntegerAttr":$minBPM,
+ OptionalParameter<"mlir::IntegerAttr">:$minBPM,
OptionalParameter<"mlir::IntegerAttr">:$upperBoundClusterSize
);
diff --git a/flang/lib/Lower/CallInterface.cpp b/flang/lib/Lower/CallInterface.cpp
index 3fcf314faefb6..c570e23a3078c 100644
--- a/flang/lib/Lower/CallInterface.cpp
+++ b/flang/lib/Lower/CallInterface.cpp
@@ -643,12 +643,16 @@ setCUDAAttributes(mlir::func::FuncOp func,
.detailsIf<Fortran::semantics::SubprogramDetails>()) {
mlir::Type i64Ty = mlir::IntegerType::get(func.getContext(), 64);
if (!details->cudaLaunchBounds().empty()) {
- assert(details->cudaLaunchBounds().size() >= 2 &&
- "expect at least 2 values");
+ assert(details->cudaLaunchBounds().size() >= 1 &&
+ details->cudaLaunchBounds().size() <= 3 &&
+ "expect 1, 2, or 3 values");
auto maxTPBAttr =
mlir::IntegerAttr::get(i64Ty, details->cudaLaunchBounds()[0]);
- auto minBPMAttr =
- mlir::IntegerAttr::get(i64Ty, details->cudaLaunchBounds()[1]);
+ // The minimum-blocks-per-multiprocessor operand is optional.
+ mlir::IntegerAttr minBPMAttr;
+ if (details->cudaLaunchBounds().size() > 1)
+ minBPMAttr =
+ mlir::IntegerAttr::get(i64Ty, details->cudaLaunchBounds()[1]);
mlir::IntegerAttr ubAttr;
if (details->cudaLaunchBounds().size() > 2)
ubAttr =
diff --git a/flang/lib/Optimizer/Transforms/CUDA/CUFDeviceFuncTransform.cpp b/flang/lib/Optimizer/Transforms/CUDA/CUFDeviceFuncTransform.cpp
index 97540820ef415..5091ea82f8b45 100644
--- a/flang/lib/Optimizer/Transforms/CUDA/CUFDeviceFuncTransform.cpp
+++ b/flang/lib/Optimizer/Transforms/CUDA/CUFDeviceFuncTransform.cpp
@@ -151,8 +151,10 @@ class CUFDeviceFuncTransform
auto maxntid =
builder.getDenseI32ArrayAttr({static_cast<int32_t>(maxTPB), 1, 1});
deviceFuncOp->setAttr(NVVM::NVVMDialect::getMaxntidAttrName(), maxntid);
- deviceFuncOp->setAttr(NVVM::NVVMDialect::getMinctasmAttrName(),
- launchBoundsAttr.getMinBPM());
+ // The minimum-blocks-per-multiprocessor operand is optional.
+ if (launchBoundsAttr.getMinBPM())
+ deviceFuncOp->setAttr(NVVM::NVVMDialect::getMinctasmAttrName(),
+ launchBoundsAttr.getMinBPM());
if (computeCap >= 90 && launchBoundsAttr.getUpperBoundClusterSize())
deviceFuncOp->setAttr(NVVM::NVVMDialect::getClusterMaxBlocksAttrName(),
launchBoundsAttr.getUpperBoundClusterSize());
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 3a006c6c3431f..9ddda9a92471b 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -5034,17 +5034,24 @@ void SubprogramVisitor::Post(const parser::PrefixSpec::Launch_Bounds &x) {
ok = false;
}
}
- if (!ok || bounds.size() < 2 || bounds.size() > 3) {
- Say(currStmtSource().value(),
- "Operands of LAUNCH_BOUNDS() must be 2 or 3 integer constants"_err_en_US);
+ // The second (minimum blocks per multiprocessor) and third (maximum blocks
+ // per cluster) operands of LAUNCH_BOUNDS() are optional, so 1, 2, or 3
+ // integer constants are accepted. This handler runs during the direct walk
+ // of the subprogram statement performed by ResolveSpecificationParts(), where
+ // currStmtSource() may not be set, so derive the diagnostic source from the
+ // prefix instead of dereferencing it.
+ parser::CharBlock source{parser::GetSource(x).value_or(
+ currStmtSource().value_or(parser::CharBlock{}))};
+ if (!ok || bounds.empty() || bounds.size() > 3) {
+ Say(source,
+ "Operands of LAUNCH_BOUNDS() must be 1, 2, or 3 integer constants"_err_en_US);
} else if (auto *subp{currScope().symbol()
? currScope().symbol()->detailsIf<SubprogramDetails>()
: nullptr}) {
if (subp->cudaLaunchBounds().empty()) {
subp->set_cudaLaunchBounds(std::move(bounds));
} else {
- Say(currStmtSource().value(),
- "LAUNCH_BOUNDS() may only appear once"_err_en_US);
+ Say(source, "LAUNCH_BOUNDS() may only appear once"_err_en_US);
}
}
}
@@ -5059,8 +5066,12 @@ void SubprogramVisitor::Post(const parser::PrefixSpec::Cluster_Dims &x) {
ok = false;
}
}
+ // As in Post(Launch_Bounds), currStmtSource() may be unset on this path, so
+ // take the diagnostic source from the prefix.
+ parser::CharBlock source{parser::GetSource(x).value_or(
+ currStmtSource().value_or(parser::CharBlock{}))};
if (!ok || dims.size() != 3) {
- Say(currStmtSource().value(),
+ Say(source,
"Operands of CLUSTER_DIMS() must be three integer constants"_err_en_US);
} else if (auto *subp{currScope().symbol()
? currScope().symbol()->detailsIf<SubprogramDetails>()
@@ -5068,8 +5079,7 @@ void SubprogramVisitor::Post(const parser::PrefixSpec::Cluster_Dims &x) {
if (subp->cudaClusterDims().empty()) {
subp->set_cudaClusterDims(std::move(dims));
} else {
- Say(currStmtSource().value(),
- "CLUSTER_DIMS() may only appear once"_err_en_US);
+ Say(source, "CLUSTER_DIMS() may only appear once"_err_en_US);
}
}
}
diff --git a/flang/test/Fir/CUDA/cuda-device-func-transform.mlir b/flang/test/Fir/CUDA/cuda-device-func-transform.mlir
index 6af5e021c33df..fae01500edd72 100644
--- a/flang/test/Fir/CUDA/cuda-device-func-transform.mlir
+++ b/flang/test/Fir/CUDA/cuda-device-func-transform.mlir
@@ -190,3 +190,14 @@ func.func @_QPsub_maxtnid() attributes {cuf.launch_bounds = #cuf.launch_bounds<m
}
// CHECK: gpu.func @_QPsub_maxtnid() kernel attributes {nvvm.maxntid = array<i32: 256, 1, 1>, nvvm.minctasm = 2 : i64}
+
+// -----
+
+// The minimum-blocks-per-multiprocessor operand is optional: only maxntid is set.
+func.func @_QPsub_maxtnid_only() attributes {cuf.launch_bounds = #cuf.launch_bounds<maxTPB = 256 : i64>, cuf.proc_attr = #cuf.cuda_proc<global>} {
+ %cst = arith.constant 2.000000e+00 : f32
+ return
+}
+
+// CHECK: gpu.func @_QPsub_maxtnid_only() kernel attributes {nvvm.maxntid = array<i32: 256, 1, 1>}
+// CHECK-NOT: nvvm.minctasm
diff --git a/flang/test/Lower/CUDA/cuda-proc-attribute.cuf b/flang/test/Lower/CUDA/cuda-proc-attribute.cuf
index f8b8dd8e296b4..dc3c925f888b6 100644
--- a/flang/test/Lower/CUDA/cuda-proc-attribute.cuf
+++ b/flang/test/Lower/CUDA/cuda-proc-attribute.cuf
@@ -33,6 +33,9 @@ attributes(host) attributes(device) integer function fct_host_device; end
attributes(device) attributes(host) integer function fct_device_host; end
! CHECK: func.func @_QPfct_device_host() -> i32 attributes {cuf.proc_attr = #cuf.cuda_proc<host_device>}
+attributes(global) launch_bounds(1) subroutine sub_lbounds0(); end
+! CHECK: func.func @_QPsub_lbounds0() attributes {cuf.launch_bounds = #cuf.launch_bounds<maxTPB = 1 : i64>, cuf.proc_attr = #cuf.cuda_proc<global>}
+
attributes(global) launch_bounds(1, 2) subroutine sub_lbounds1(); end
! CHECK: func.func @_QPsub_lbounds1() attributes {cuf.launch_bounds = #cuf.launch_bounds<maxTPB = 1 : i64, minBPM = 2 : i64>, cuf.proc_attr = #cuf.cuda_proc<global>}
diff --git a/flang/test/Semantics/CUDA/cuf08.cuf b/flang/test/Semantics/CUDA/cuf08.cuf
index caad766f2549a..e3569eaeb6c61 100644
--- a/flang/test/Semantics/CUDA/cuf08.cuf
+++ b/flang/test/Semantics/CUDA/cuf08.cuf
@@ -5,13 +5,12 @@ module m
launch_bounds(1,2) subroutine bad1; end
!ERROR: A subroutine may not have LAUNCH_BOUNDS() or CLUSTER_DIMS() unless it has ATTRIBUTES(GLOBAL) or ATTRIBUTES(GRID_GLOBAL)
cluster_dims(1,2,3) subroutine bad2; end
+ attributes(global) launch_bounds(1) subroutine good1a; end
attributes(global) launch_bounds(1,2) subroutine good1; end
attributes(global) launch_bounds(1,2,3) subroutine good2; end
!ERROR: LAUNCH_BOUNDS() may only appear once
attributes(global) launch_bounds(1,2) launch_bounds(3,4) subroutine bad3; end
- !ERROR: Operands of LAUNCH_BOUNDS() must be 2 or 3 integer constants
- attributes(global) launch_bounds(1) subroutine bad4; end
- !ERROR: Operands of LAUNCH_BOUNDS() must be 2 or 3 integer constants
+ !ERROR: Operands of LAUNCH_BOUNDS() must be 1, 2, or 3 integer constants
attributes(global) launch_bounds(1,2,3,4) subroutine bad5; end
attributes(global) cluster_dims(1,2,3) subroutine good3; end
!ERROR: CLUSTER_DIMS() may only appear once
diff --git a/flang/test/Semantics/CUDA/cuf31.cuf b/flang/test/Semantics/CUDA/cuf31.cuf
new file mode 100644
index 0000000000000..d38c5c91a76bc
--- /dev/null
+++ b/flang/test/Semantics/CUDA/cuf31.cuf
@@ -0,0 +1,38 @@
+! RUN: %python %S/../test_errors.py %s %flang_fc1
+! Regression test for a compiler crash on CUDA Fortran LAUNCH_BOUNDS() /
+! CLUSTER_DIMS() when the operand list is rejected: the diagnostic dereferenced
+! an empty currStmtSource() during ResolveSpecificationParts and aborted. The
+! crash only reproduces when the offending construct is the first program unit
+! processed (a later unit leaves currStmtSource() set, masking it), so an
+! invalid module leads here. m_lb_ok is the valid case -- the second and third
+! LAUNCH_BOUNDS operands are optional; the rest are invalid and must diagnose
+! cleanly rather than crash.
+
+module m_lb_dup
+ contains
+ !ERROR: LAUNCH_BOUNDS() may only appear once
+ attributes(global) launch_bounds(1,2) launch_bounds(3,4) subroutine s; end
+end module
+
+module m_lb_ok
+ contains
+ attributes(global) launch_bounds(32) subroutine s; end
+end module
+
+module m_lb_toomany
+ contains
+ !ERROR: Operands of LAUNCH_BOUNDS() must be 1, 2, or 3 integer constants
+ attributes(global) launch_bounds(1,2,3,4) subroutine s; end
+end module
+
+module m_cd_wrongcount
+ contains
+ !ERROR: Operands of CLUSTER_DIMS() must be three integer constants
+ attributes(global) cluster_dims(1) subroutine s; end
+end module
+
+module m_cd_dup
+ contains
+ !ERROR: CLUSTER_DIMS() may only appear once
+ attributes(global) cluster_dims(1,2,3) cluster_dims(4,5,6) subroutine s; end
+end module
>From 6645153025367af4f3bd4020dc81700a4cbe9132 Mon Sep 17 00:00:00 2001
From: Cyndy Ishida <cyndy_ishida at apple.com>
Date: Mon, 6 Jul 2026 08:40:53 -0700
Subject: [PATCH 09/46] [clang-scan-deps] Migrate more tests through
scan-deps-filter (#207283)
Migrate tests that would otherwise fail when the `-emit-visible-modules`
flag is removed.
* The tests now include a `scan-deps-filter` invocation that maintains
the same fields that are already checked, so there is no test coverage
lost.
* Also remove `command-line` fields that were not actually checked in
tests.
Assisted-by: Claude Opus 4.8
---
clang/test/ClangScanDeps/diagnostics.c | 4 +++-
.../header-search-pruning-transitive.c | 20 +++++--------------
...modules-context-hash-from-named-module.cpp | 4 +++-
.../modules-context-hash-ignore-macros.c | 4 +++-
.../modules-context-hash-outputs.c | 4 +++-
.../modules-context-hash-warnings.c | 4 +++-
.../test/ClangScanDeps/modules-context-hash.c | 8 ++++++--
clang/test/ClangScanDeps/modules-dep-args.c | 16 +++++++++++----
.../ClangScanDeps/modules-extern-submodule.c | 4 +++-
.../ClangScanDeps/modules-extern-unrelated.m | 4 +++-
.../modules-fmodule-name-no-module-built.m | 8 +++-----
.../modules-full-named-modules.cppm | 7 +++++++
.../modules-full-output-tu-order.c | 4 +++-
clang/test/ClangScanDeps/modules-full.cpp | 8 ++++++--
.../modules-has-include-umbrella-header.c | 6 +++---
.../ClangScanDeps/modules-header-sharing.m | 4 +++-
.../modules-implementation-module-map.c | 4 +++-
.../modules-implementation-private.m | 8 +++-----
.../modules-implementation-vfs.m | 4 +++-
.../modules-implicit-dot-private.m | 4 +++-
.../modules-incomplete-umbrella.c | 8 ++++++--
clang/test/ClangScanDeps/modules-inferred.m | 4 +++-
.../modules-no-undeclared-includes.c | 4 +++-
.../ClangScanDeps/modules-pch-common-stale.c | 8 ++++++--
.../modules-pch-common-submodule.c | 16 ++++++---------
.../modules-pch-common-via-submodule.c | 16 ++++++---------
clang/test/ClangScanDeps/modules-pch.c | 12 ++++++++---
.../ClangScanDeps/modules-priv-fw-from-pub.m | 4 +++-
clang/test/ClangScanDeps/multiple-commands.c | 4 +++-
clang/test/ClangScanDeps/tu-buffer.c | 12 +++--------
30 files changed, 129 insertions(+), 88 deletions(-)
diff --git a/clang/test/ClangScanDeps/diagnostics.c b/clang/test/ClangScanDeps/diagnostics.c
index 8e3cf4c9f9fa6..ca087aeeb2c2c 100644
--- a/clang/test/ClangScanDeps/diagnostics.c
+++ b/clang/test/ClangScanDeps/diagnostics.c
@@ -17,7 +17,9 @@ module mod { header "mod.h" }
// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json
// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full 2>&1 > %t/result.json
-// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t
+// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=name,context-hash,clang-context-hash,command-line,file-deps,clang-module-deps,clang-modulemap-file,link-libraries,input-file \
+// RUN: | FileCheck %s -DPREFIX=%/t
// Check that the '-Wno-error=invalid-ios-deployment-target' option is being
// respected and invalid arguments like '-target i386-apple-ios14.0-simulator'
diff --git a/clang/test/ClangScanDeps/header-search-pruning-transitive.c b/clang/test/ClangScanDeps/header-search-pruning-transitive.c
index 1e829bb02ddc4..830b8cdbc7528 100644
--- a/clang/test/ClangScanDeps/header-search-pruning-transitive.c
+++ b/clang/test/ClangScanDeps/header-search-pruning-transitive.c
@@ -54,9 +54,11 @@ module X { header "X.h" }
// RUN: sed -e "s|DIR|%/t|g" %t/cdb_with_a.json.template > %t/cdb_with_a.json
// RUN: sed -e "s|DIR|%/t|g" %t/cdb_without_a.json.template > %t/cdb_without_a.json
-// RUN: clang-scan-deps -compilation-database %t/cdb_with_a.json -format experimental-full -optimize-args=header-search > %t/results.json
-// RUN: clang-scan-deps -compilation-database %t/cdb_without_a.json -format experimental-full -optimize-args=header-search >> %t/results.json
-// RUN: cat %t/results.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t
+// RUN: clang-scan-deps -compilation-database %t/cdb_with_a.json -format experimental-full -optimize-args=header-search \
+// RUN: | sed 's:\\\\\?:/:g' | %scan-deps-filter --fields=clang-context-hash,clang-module-deps,clang-modulemap-file,context-hash,file-deps,input-file,link-libraries,name > %t/results.json
+// RUN: clang-scan-deps -compilation-database %t/cdb_without_a.json -format experimental-full -optimize-args=header-search \
+// RUN: | sed 's:\\\\\?:/:g' | %scan-deps-filter --fields=clang-context-hash,clang-module-deps,clang-modulemap-file,context-hash,file-deps,input-file,link-libraries,name >> %t/results.json
+// RUN: cat %t/results.json | FileCheck %s -DPREFIX=%/t
// CHECK: {
// CHECK-NEXT: "modules": [
@@ -68,8 +70,6 @@ module X { header "X.h" }
// CHECK-NEXT: }
// CHECK-NEXT: ],
// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap",
-// CHECK-NEXT: "command-line": [
-// CHECK: ],
// CHECK-NEXT: "context-hash": "[[HASH_X:.*]]",
// CHECK-NEXT: "file-deps": [
// CHECK-NEXT: "[[PREFIX]]/module.modulemap",
@@ -81,8 +81,6 @@ module X { header "X.h" }
// CHECK-NEXT: {
// CHECK-NEXT: "clang-module-deps": [],
// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap",
-// CHECK-NEXT: "command-line": [
-// CHECK: ],
// CHECK-NEXT: "context-hash": "[[HASH_Y_WITH_A]]",
// CHECK-NEXT: "file-deps": [
// CHECK-NEXT: "[[PREFIX]]/module.modulemap",
@@ -104,8 +102,6 @@ module X { header "X.h" }
// CHECK-NEXT: "module-name": "X"
// CHECK-NEXT: }
// CHECK-NEXT: ],
-// CHECK-NEXT: "command-line": [
-// CHECK: ],
// CHECK: "file-deps": [
// CHECK-NEXT: "[[PREFIX]]/test.c"
// CHECK-NEXT: ],
@@ -122,8 +118,6 @@ module X { header "X.h" }
// CHECK-NEXT: }
// CHECK-NEXT: ],
// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap",
-// CHECK-NEXT: "command-line": [
-// CHECK: ],
// Here is the actual check that this module X (which imports different version of Y)
// also has a different context hash from the first version of module X.
// CHECK-NOT: "context-hash": "[[HASH_X]]",
@@ -137,8 +131,6 @@ module X { header "X.h" }
// CHECK-NEXT: {
// CHECK-NEXT: "clang-module-deps": [],
// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap",
-// CHECK-NEXT: "command-line": [
-// CHECK: ],
// CHECK-NEXT: "context-hash": "[[HASH_Y_WITHOUT_A]]",
// CHECK-NEXT: "file-deps": [
// CHECK-NEXT: "[[PREFIX]]/module.modulemap",
@@ -159,8 +151,6 @@ module X { header "X.h" }
// CHECK-NEXT: "module-name": "X"
// CHECK-NEXT: }
// CHECK-NEXT: ],
-// CHECK-NEXT: "command-line": [
-// CHECK: ],
// CHECK: "file-deps": [
// CHECK-NEXT: "[[PREFIX]]/test.c"
// CHECK-NEXT: ],
diff --git a/clang/test/ClangScanDeps/modules-context-hash-from-named-module.cpp b/clang/test/ClangScanDeps/modules-context-hash-from-named-module.cpp
index c27202234f4c5..f3a8d3c4c0762 100644
--- a/clang/test/ClangScanDeps/modules-context-hash-from-named-module.cpp
+++ b/clang/test/ClangScanDeps/modules-context-hash-from-named-module.cpp
@@ -39,7 +39,9 @@ module root { header "root.h" }
// RUN: clang-scan-deps \
// RUN: -compilation-database=%t/compile_commands.json \
// RUN: -format experimental-full \
-// RUN: | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t
+// RUN: | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=clang-context-hash,clang-module-deps,clang-modulemap-file,context-hash,file-deps,input-file,link-libraries,name,named-module,named-module-deps \
+// RUN: | FileCheck %s -DPREFIX=%/t
// CHECK: {
// CHECK-NEXT: "modules": [
diff --git a/clang/test/ClangScanDeps/modules-context-hash-ignore-macros.c b/clang/test/ClangScanDeps/modules-context-hash-ignore-macros.c
index 9f7a62fb9eb74..2976d571868cc 100644
--- a/clang/test/ClangScanDeps/modules-context-hash-ignore-macros.c
+++ b/clang/test/ClangScanDeps/modules-context-hash-ignore-macros.c
@@ -7,7 +7,9 @@
// RUN: clang-scan-deps -compilation-database %t/cdb.json -j 1 \
// RUN: -format experimental-full > %t/deps.json
-// RUN: cat %t/deps.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s
+// RUN: cat %t/deps.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=clang-module-deps,command-line,context-hash,input-file,name \
+// RUN: | FileCheck -DPREFIX=%/t %s
// CHECK: {
// CHECK-NEXT: "modules": [
diff --git a/clang/test/ClangScanDeps/modules-context-hash-outputs.c b/clang/test/ClangScanDeps/modules-context-hash-outputs.c
index 504abd5f08720..d5dfffdcef6df 100644
--- a/clang/test/ClangScanDeps/modules-context-hash-outputs.c
+++ b/clang/test/ClangScanDeps/modules-context-hash-outputs.c
@@ -7,7 +7,9 @@
// RUN: clang-scan-deps -compilation-database %t/cdb.json -j 1 \
// RUN: -format experimental-full > %t/deps.json
-// RUN: cat %t/deps.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s
+// RUN: cat %t/deps.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=clang-module-deps,command-line,context-hash,input-file,name \
+// RUN: | FileCheck -DPREFIX=%/t %s
// CHECK: {
// CHECK-NEXT: "modules": [
diff --git a/clang/test/ClangScanDeps/modules-context-hash-warnings.c b/clang/test/ClangScanDeps/modules-context-hash-warnings.c
index 09d2f20b329e3..7a647de213a5d 100644
--- a/clang/test/ClangScanDeps/modules-context-hash-warnings.c
+++ b/clang/test/ClangScanDeps/modules-context-hash-warnings.c
@@ -7,7 +7,9 @@
// RUN: clang-scan-deps -compilation-database %t/cdb.json -j 1 \
// RUN: -format experimental-full > %t/deps.json
-// RUN: cat %t/deps.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s
+// RUN: cat %t/deps.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=clang-module-deps,command-line,context-hash,input-file,name \
+// RUN: | FileCheck -DPREFIX=%/t %s
// CHECK: {
// CHECK-NEXT: "modules": [
diff --git a/clang/test/ClangScanDeps/modules-context-hash.c b/clang/test/ClangScanDeps/modules-context-hash.c
index 881d416868ecb..c175070a33b9f 100644
--- a/clang/test/ClangScanDeps/modules-context-hash.c
+++ b/clang/test/ClangScanDeps/modules-context-hash.c
@@ -17,8 +17,12 @@
// RUN: clang-scan-deps -compilation-database %t/cdb_a.json -format experimental-full -j 1 > %t/result_a.json
// RUN: clang-scan-deps -compilation-database %t/cdb_b.json -format experimental-full -j 1 > %t/result_b.json
// RUN: clang-scan-deps -compilation-database %t/cdb_b2.json -format experimental-full -j 1 > %t/result_b2.json
-// RUN: cat %t/result_a.json %t/result_b.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t -check-prefix=CHECK
-// RUN: cat %t/result_b.json %t/result_b2.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t -check-prefix=FLAG_ONLY
+// Filter each unique JSON output before concatenating for FileCheck.
+// RUN: %scan-deps-filter --fields=clang-context-hash,clang-modulemap-file,command-line,context-hash,file-deps,input-file,link-libraries,name,clang-module-deps %t/result_a.json > %t/result_a.filt.json
+// RUN: %scan-deps-filter --fields=clang-context-hash,clang-modulemap-file,command-line,context-hash,file-deps,input-file,link-libraries,name,clang-module-deps %t/result_b.json > %t/result_b.filt.json
+// RUN: %scan-deps-filter --fields=clang-context-hash,clang-modulemap-file,command-line,context-hash,file-deps,input-file,link-libraries,name,clang-module-deps %t/result_b2.json > %t/result_b2.filt.json
+// RUN: cat %t/result_a.filt.json %t/result_b.filt.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t -check-prefix=CHECK
+// RUN: cat %t/result_b.filt.json %t/result_b2.filt.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t -check-prefix=FLAG_ONLY
// CHECK: {
// CHECK-NEXT: "modules": [
diff --git a/clang/test/ClangScanDeps/modules-dep-args.c b/clang/test/ClangScanDeps/modules-dep-args.c
index 19f915923b84c..40a8a3dedde2a 100644
--- a/clang/test/ClangScanDeps/modules-dep-args.c
+++ b/clang/test/ClangScanDeps/modules-dep-args.c
@@ -24,19 +24,27 @@ module Direct { header "direct.h" }
// Check that the PCM path defaults to the modules cache from implicit build.
// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full > %t/result_cache.json
-// RUN: cat %t/result_cache.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t --check-prefixes=CHECK,CHECK_CACHE
+// RUN: cat %t/result_cache.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=clang-modulemap-file,command-line,context-hash,file-deps,link-libraries,name,clang-context-hash,clang-module-deps,input-file \
+// RUN: | FileCheck %s -DPREFIX=%/t --check-prefixes=CHECK,CHECK_CACHE
// Check that the PCM path can be customized.
// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -module-files-dir %t/build > %t/result_build.json
-// RUN: cat %t/result_build.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t --check-prefixes=CHECK,CHECK_BUILD
+// RUN: cat %t/result_build.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=clang-modulemap-file,command-line,context-hash,file-deps,link-libraries,name,clang-context-hash,clang-module-deps,input-file \
+// RUN: | FileCheck %s -DPREFIX=%/t --check-prefixes=CHECK,CHECK_BUILD
// Check that the PCM file is loaded lazily by default.
// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full > %t/result_lazy.json
-// RUN: cat %t/result_lazy.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t --check-prefixes=CHECK,CHECK_LAZY
+// RUN: cat %t/result_lazy.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=clang-modulemap-file,command-line,context-hash,file-deps,link-libraries,name,clang-context-hash,clang-module-deps,input-file \
+// RUN: | FileCheck %s -DPREFIX=%/t --check-prefixes=CHECK,CHECK_LAZY
// Check that the PCM file can be loaded eagerly.
// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -eager-load-pcm > %t/result_eager.json
-// RUN: cat %t/result_eager.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t --check-prefixes=CHECK,CHECK_EAGER
+// RUN: cat %t/result_eager.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=clang-modulemap-file,command-line,context-hash,file-deps,link-libraries,name,clang-context-hash,clang-module-deps,input-file \
+// RUN: | FileCheck %s -DPREFIX=%/t --check-prefixes=CHECK,CHECK_EAGER
// CHECK: {
// CHECK-NEXT: "modules": [
diff --git a/clang/test/ClangScanDeps/modules-extern-submodule.c b/clang/test/ClangScanDeps/modules-extern-submodule.c
index 01d3d6ba5e0d3..6532b636976e8 100644
--- a/clang/test/ClangScanDeps/modules-extern-submodule.c
+++ b/clang/test/ClangScanDeps/modules-extern-submodule.c
@@ -28,7 +28,9 @@ module third {}
// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json
// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full > %t/result.json
-// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t
+// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=clang-modulemap-file,command-line,context-hash,file-deps,link-libraries,name,clang-context-hash,clang-module-deps,input-file \
+// RUN: | FileCheck %s -DPREFIX=%/t
// CHECK: {
// CHECK-NEXT: "modules": [
diff --git a/clang/test/ClangScanDeps/modules-extern-unrelated.m b/clang/test/ClangScanDeps/modules-extern-unrelated.m
index c003f0d9a2ee8..061d0912d086a 100644
--- a/clang/test/ClangScanDeps/modules-extern-unrelated.m
+++ b/clang/test/ClangScanDeps/modules-extern-unrelated.m
@@ -27,7 +27,9 @@
// RUN: clang-scan-deps -format experimental-full -o %t/result.json \
// RUN: -- %clang -fmodules -fmodules-cache-path=%t/cache -I %t/zeroth -I %t/first -I %t/second -c %t/tu.m -o %t/tu.o
-// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t
+// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=clang-modulemap-file,command-line,context-hash,file-deps,link-libraries,name,clang-context-hash,clang-module-deps,executable,input-file \
+// RUN: | FileCheck %s -DPREFIX=%/t
// CHECK: {
// CHECK-NEXT: "modules": [
diff --git a/clang/test/ClangScanDeps/modules-fmodule-name-no-module-built.m b/clang/test/ClangScanDeps/modules-fmodule-name-no-module-built.m
index f7edb011ad32a..5fd18d51a623e 100644
--- a/clang/test/ClangScanDeps/modules-fmodule-name-no-module-built.m
+++ b/clang/test/ClangScanDeps/modules-fmodule-name-no-module-built.m
@@ -11,7 +11,9 @@
// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -format experimental-full \
// RUN: -mode preprocess-dependency-directives > %t.result
-// RUN: cat %t.result | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t.dir --check-prefixes=CHECK %s
+// RUN: cat %t.result | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=clang-context-hash,clang-module-deps,clang-modulemap-file,context-hash,file-deps,input-file,link-libraries,name \
+// RUN: | FileCheck -DPREFIX=%/t.dir --check-prefixes=CHECK %s
#import "header3.h"
#import "header.h"
@@ -21,8 +23,6 @@
// CHECK-NEXT: {
// CHECK-NEXT: "clang-module-deps": []
// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/Inputs/module.modulemap",
-// CHECK-NEXT: "command-line": [
-// CHECK: ],
// CHECK-NEXT: "context-hash": "[[HASH_H2:[A-Z0-9]+]]",
// CHECK-NEXT: "file-deps": [
// CHECK-NEXT: "[[PREFIX]]/Inputs/module.modulemap",
@@ -41,8 +41,6 @@
// CHECK-NEXT: "module-name": "header2"
// CHECK-NEXT: }
// CHECK-NEXT: ],
-// CHECK-NEXT: "command-line": [
-// CHECK: ],
// CHECK: "file-deps": [
// CHECK-NEXT: "[[PREFIX]]/modules-fmodule-name-no-module-built.m",
// CHECK-NEXT: "[[PREFIX]]/Inputs/header3.h",
diff --git a/clang/test/ClangScanDeps/modules-full-named-modules.cppm b/clang/test/ClangScanDeps/modules-full-named-modules.cppm
index c69a215a62dc1..b739385fbc975 100644
--- a/clang/test/ClangScanDeps/modules-full-named-modules.cppm
+++ b/clang/test/ClangScanDeps/modules-full-named-modules.cppm
@@ -13,30 +13,37 @@
// RUN: clang-scan-deps -format=experimental-full \
// RUN: -- %clang++ -std=c++20 -c -fprebuilt-module-path=%t %t/M.cppm -o %t/M.o \
// RUN: | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=modules,named-module,named-module-deps,command-line,input-file \
// RUN: | FileCheck %t/M.cppm -DPREFIX=%/t
// RUN: clang-scan-deps -format=experimental-full \
// RUN: -- %clang++ -std=c++20 -c -fprebuilt-module-path=%t %t/Impl.cpp -o %t/Impl.o \
// RUN: | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=modules,named-module,named-module-deps,command-line,input-file \
// RUN: | FileCheck %t/Impl.cpp -DPREFIX=%/t
// RUN: clang-scan-deps -format=experimental-full \
// RUN: -- %clang++ -std=c++20 -c -fprebuilt-module-path=%t %t/impl_part.cppm -o %t/impl_part.o \
// RUN: | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=modules,named-module,named-module-deps,command-line,input-file \
// RUN: | FileCheck %t/impl_part.cppm -DPREFIX=%/t
// RUN: clang-scan-deps -format=experimental-full \
// RUN: -- %clang++ -std=c++20 -c -fprebuilt-module-path=%t %t/interface_part.cppm -o %t/interface_part.o \
// RUN: | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=modules,named-module,named-module-deps,command-line,input-file \
// RUN: | FileCheck %t/interface_part.cppm -DPREFIX=%/t
// RUN: clang-scan-deps -format=experimental-full \
// RUN: -- %clang++ -std=c++20 -c -fprebuilt-module-path=%t %t/User.cpp -o %t/User.o \
// RUN: | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=modules,named-module,named-module-deps,command-line,input-file \
// RUN: | FileCheck %t/User.cpp -DPREFIX=%/t
//
// Check the combined dependency format.
// RUN: clang-scan-deps -compilation-database %t/compile_commands.json -format=experimental-full \
// RUN: | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=modules,named-module,named-module-deps,command-line,input-file \
// RUN: | FileCheck %t/Checks.cpp -DPREFIX=%/t
// RUN: clang-scan-deps --mode=preprocess-dependency-directives -compilation-database %t/compile_commands.json -format=experimental-full \
// RUN: | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=modules,named-module,named-module-deps,command-line,input-file \
// RUN: | FileCheck %t/Checks.cpp -DPREFIX=%/t
//--- compile_commands.json.in
diff --git a/clang/test/ClangScanDeps/modules-full-output-tu-order.c b/clang/test/ClangScanDeps/modules-full-output-tu-order.c
index 065b8ac8ae07f..6f495987f8ed7 100644
--- a/clang/test/ClangScanDeps/modules-full-output-tu-order.c
+++ b/clang/test/ClangScanDeps/modules-full-output-tu-order.c
@@ -21,7 +21,9 @@
// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json
// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full > %t/result.json
-// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s
+// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=modules,clang-context-hash,clang-module-deps,command-line,file-deps,input-file \
+// RUN: | FileCheck -DPREFIX=%/t %s
// CHECK: {
// CHECK-NEXT: "modules": [],
diff --git a/clang/test/ClangScanDeps/modules-full.cpp b/clang/test/ClangScanDeps/modules-full.cpp
index 38db3af4403bb..170ab82441b9b 100644
--- a/clang/test/ClangScanDeps/modules-full.cpp
+++ b/clang/test/ClangScanDeps/modules-full.cpp
@@ -12,11 +12,15 @@
//
// RUN: clang-scan-deps -compilation-database %t.cdb -j 4 -format experimental-full \
// RUN: -mode preprocess-dependency-directives > %t.result
-// RUN: cat %t.result | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t.dir %s
+// RUN: cat %t.result | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=clang-module-deps,clang-modulemap-file,command-line,context-hash,file-deps,link-libraries,name,clang-context-hash,executable,input-file \
+// RUN: | FileCheck -DPREFIX=%/t.dir %s
//
// RUN: clang-scan-deps -compilation-database %t_clangcl.cdb -j 4 -format experimental-full \
// RUN: -mode preprocess-dependency-directives > %t_clangcl.result
-// RUN: cat %t_clangcl.result | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t.dir %s
+// RUN: cat %t_clangcl.result | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=clang-module-deps,clang-modulemap-file,command-line,context-hash,file-deps,link-libraries,name,clang-context-hash,executable,input-file \
+// RUN: | FileCheck -DPREFIX=%/t.dir %s
#include "header.h"
diff --git a/clang/test/ClangScanDeps/modules-has-include-umbrella-header.c b/clang/test/ClangScanDeps/modules-has-include-umbrella-header.c
index f7f804794ab41..047e154a112ca 100644
--- a/clang/test/ClangScanDeps/modules-has-include-umbrella-header.c
+++ b/clang/test/ClangScanDeps/modules-has-include-umbrella-header.c
@@ -47,7 +47,9 @@ module Dependency { header "dependency.h" }
// RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json
// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full > %t/deps.json
-// RUN: cat %t/deps.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t
+// RUN: cat %t/deps.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=translation-units.commands.clang-context-hash,translation-units.commands.clang-module-deps,translation-units.commands.file-deps,translation-units.commands.input-file \
+// RUN: | FileCheck %s -DPREFIX=%/t
// Let's check that the TU actually depends on `FW_Private` (and does not treat FW/B.h as textual).
// CHECK: {
@@ -62,8 +64,6 @@ module Dependency { header "dependency.h" }
// CHECK-NEXT: "module-name": "FW_Private"
// CHECK-NEXT: }
// CHECK: ],
-// CHECK-NEXT: "command-line": [
-// CHECK: ],
// CHECK: "file-deps": [
// CHECK-NEXT: "[[PREFIX]]/tu.c",
// CHECK-NEXT: "[[PREFIX]]/frameworks/FW.framework/PrivateHeaders/B.h"
diff --git a/clang/test/ClangScanDeps/modules-header-sharing.m b/clang/test/ClangScanDeps/modules-header-sharing.m
index c40c2891786cb..ddba69ab55a23 100644
--- a/clang/test/ClangScanDeps/modules-header-sharing.m
+++ b/clang/test/ClangScanDeps/modules-header-sharing.m
@@ -67,7 +67,9 @@
// RUN: sed -e "s|DIR|%/t|g" %t/overlay.json.template > %t/overlay.json
// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full > %t/result.json
-// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t
+// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=translation-units.commands.command-line,translation-units.commands.file-deps,translation-units.commands.input-file \
+// RUN: | FileCheck %s -DPREFIX=%/t
// CHECK: {
// CHECK: "translation-units": [
// CHECK-NEXT: {
diff --git a/clang/test/ClangScanDeps/modules-implementation-module-map.c b/clang/test/ClangScanDeps/modules-implementation-module-map.c
index a7170aab2448c..d03cd719e9bfd 100644
--- a/clang/test/ClangScanDeps/modules-implementation-module-map.c
+++ b/clang/test/ClangScanDeps/modules-implementation-module-map.c
@@ -18,7 +18,9 @@ framework module FWPrivate { header "private.h" }
// RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json
// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full > %t/result.json
-// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t
+// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=command-line,file-deps,input-file \
+// RUN: | FileCheck %s -DPREFIX=%/t
// CHECK: "translation-units": [
// CHECK-NEXT: {
// CHECK-NEXT: "commands": [
diff --git a/clang/test/ClangScanDeps/modules-implementation-private.m b/clang/test/ClangScanDeps/modules-implementation-private.m
index 210fbfb424aca..f4b1a48a92538 100644
--- a/clang/test/ClangScanDeps/modules-implementation-private.m
+++ b/clang/test/ClangScanDeps/modules-implementation-private.m
@@ -26,15 +26,15 @@
// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json
// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full > %t/result.json
-// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t
+// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=clang-context-hash,clang-module-deps,clang-modulemap-file,context-hash,file-deps,input-file,link-libraries,name \
+// RUN: | FileCheck %s -DPREFIX=%/t
// CHECK: {
// CHECK-NEXT: "modules": [
// CHECK-NEXT: {
// CHECK-NEXT: "clang-module-deps": [],
// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/frameworks/FW.framework/Modules/module.private.modulemap",
-// CHECK-NEXT: "command-line": [
-// CHECK: ],
// CHECK-NEXT: "context-hash": "{{.*}}",
// CHECK-NEXT: "file-deps": [
// CHECK-NEXT: "[[PREFIX]]/frameworks/FW.framework/Modules/module.private.modulemap",
@@ -60,8 +60,6 @@
// CHECK-NEXT: "module-name": "FW_Private"
// CHECK-NEXT: }
// CHECK-NEXT: ],
-// CHECK-NEXT: "command-line": [
-// CHECK: ],
// CHECK: "file-deps": [
// CHECK-NEXT: "[[PREFIX]]/tu.m",
// CHECK-NEXT: "[[PREFIX]]/frameworks/FW.framework/PrivateHeaders/Missed.h",
diff --git a/clang/test/ClangScanDeps/modules-implementation-vfs.m b/clang/test/ClangScanDeps/modules-implementation-vfs.m
index 6bd63d049b238..75c63090496a7 100644
--- a/clang/test/ClangScanDeps/modules-implementation-vfs.m
+++ b/clang/test/ClangScanDeps/modules-implementation-vfs.m
@@ -74,7 +74,9 @@
// RUN: sed -e "s|DIR|%/t|g" %t/overlay.json.template > %t/overlay.json
// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -j 1 > %t/result.json
-// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t
+// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=name,clang-module-deps,command-line,input-file \
+// RUN: | FileCheck %s -DPREFIX=%/t
// CHECK: {
// CHECK: "modules": [
// CHECK: {
diff --git a/clang/test/ClangScanDeps/modules-implicit-dot-private.m b/clang/test/ClangScanDeps/modules-implicit-dot-private.m
index aa8caf3451dc4..ce465975fdc2e 100644
--- a/clang/test/ClangScanDeps/modules-implicit-dot-private.m
+++ b/clang/test/ClangScanDeps/modules-implicit-dot-private.m
@@ -22,7 +22,9 @@
// RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json
// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full > %t/result.json
-// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t
+// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=clang-module-deps,clang-modulemap-file,context-hash,file-deps,link-libraries,name,clang-context-hash,command-line,input-file \
+// RUN: | FileCheck %s -DPREFIX=%/t
// CHECK: {
// CHECK-NEXT: "modules": [
// CHECK-NEXT: {
diff --git a/clang/test/ClangScanDeps/modules-incomplete-umbrella.c b/clang/test/ClangScanDeps/modules-incomplete-umbrella.c
index 696e621960b6f..800fdb6976c9b 100644
--- a/clang/test/ClangScanDeps/modules-incomplete-umbrella.c
+++ b/clang/test/ClangScanDeps/modules-incomplete-umbrella.c
@@ -34,7 +34,9 @@ framework module FW_Private {
// RUN: sed -e "s|DIR|%/t|g" %t/from_tu.cdb.json.template > %t/from_tu.cdb.json
// RUN: clang-scan-deps -compilation-database %t/from_tu.cdb.json -format experimental-full > %t/from_tu_result.json
-// RUN: cat %t/from_tu_result.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t --check-prefixes=CHECK_TU
+// RUN: cat %t/from_tu_result.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=clang-module-deps,clang-modulemap-file,command-line,context-hash,file-deps,link-libraries,name,clang-context-hash,input-file \
+// RUN: | FileCheck %s -DPREFIX=%/t --check-prefixes=CHECK_TU
// CHECK_TU: {
// CHECK_TU-NEXT: "modules": [
// CHECK_TU-NEXT: {
@@ -124,7 +126,9 @@ module Mod { header "Mod.h" }
// RUN: sed -e "s|DIR|%/t|g" %t/from_module.cdb.json.template > %t/from_module.cdb.json
// RUN: clang-scan-deps -compilation-database %t/from_module.cdb.json -format experimental-full > %t/from_module_result.json
-// RUN: cat %t/from_module_result.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t --check-prefixes=CHECK_MODULE
+// RUN: cat %t/from_module_result.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=clang-module-deps,clang-modulemap-file,command-line,context-hash,file-deps,link-libraries,name,clang-context-hash,input-file \
+// RUN: | FileCheck %s -DPREFIX=%/t --check-prefixes=CHECK_MODULE
// CHECK_MODULE: {
// CHECK_MODULE-NEXT: "modules": [
// CHECK_MODULE-NEXT: {
diff --git a/clang/test/ClangScanDeps/modules-inferred.m b/clang/test/ClangScanDeps/modules-inferred.m
index 4c7a4095c9bef..02013e08f7f5d 100644
--- a/clang/test/ClangScanDeps/modules-inferred.m
+++ b/clang/test/ClangScanDeps/modules-inferred.m
@@ -26,7 +26,9 @@
// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json
// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full > %t/result.json
-// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t
+// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=clang-module-deps,clang-modulemap-file,command-line,context-hash,file-deps,link-libraries,name,clang-context-hash,input-file \
+// RUN: | FileCheck %s -DPREFIX=%/t
// CHECK: {
// CHECK-NEXT: "modules": [
diff --git a/clang/test/ClangScanDeps/modules-no-undeclared-includes.c b/clang/test/ClangScanDeps/modules-no-undeclared-includes.c
index 6110c419949f9..8039e98e71626 100644
--- a/clang/test/ClangScanDeps/modules-no-undeclared-includes.c
+++ b/clang/test/ClangScanDeps/modules-no-undeclared-includes.c
@@ -31,7 +31,9 @@ module User [no_undeclared_includes] { header "user.h" }
// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json
// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full \
// RUN: -module-files-dir %t/build > %t/result.json
-// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t
+// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=name,clang-module-deps,clang-modulemap-file,command-line,context-hash,file-deps,link-libraries,clang-context-hash,input-file \
+// RUN: | FileCheck %s -DPREFIX=%/t
// CHECK: {
// CHECK-NEXT: "modules": [
diff --git a/clang/test/ClangScanDeps/modules-pch-common-stale.c b/clang/test/ClangScanDeps/modules-pch-common-stale.c
index b325536a8a668..b5f0b36feb807 100644
--- a/clang/test/ClangScanDeps/modules-pch-common-stale.c
+++ b/clang/test/ClangScanDeps/modules-pch-common-stale.c
@@ -40,7 +40,9 @@ module mod_tu_extra { header "mod_tu_extra.h" }
// RUN: clang-scan-deps -format experimental-full -o %t/deps_tu_clean.json -- \
// RUN: %clang -nostdinc -c %t/tu.c -o %t/tu.o -include %t/prefix.h \
// RUN: -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/cache
-// RUN: cat %t/deps_tu_clean.json | sed 's:\\\\\?:/:g' | FileCheck %s --check-prefix=CHECK-TU-CLEAN -DPREFIX=%/t
+// RUN: cat %t/deps_tu_clean.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=name,clang-module-deps,clang-modulemap-file,command-line,context-hash,file-deps,link-libraries,clang-context-hash,executable,input-file \
+// RUN: | FileCheck %s --check-prefix=CHECK-TU-CLEAN -DPREFIX=%/t
// CHECK-TU-CLEAN: {
// CHECK-TU-CLEAN-NEXT: "modules": [
// CHECK-TU-CLEAN-NEXT: {
@@ -109,7 +111,9 @@ module mod_tu_extra { header "mod_tu_extra.h" }
// RUN: clang-scan-deps -format experimental-full -o %t/deps_tu_incremental.json -- \
// RUN: %clang -nostdinc -c %t/tu.c -o %t/tu.o -include %t/prefix.h \
// RUN: -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/cache
-// RUN: cat %t/deps_tu_incremental.json | sed 's:\\\\\?:/:g' | FileCheck %s --check-prefix=CHECK-TU-INCREMENTAL -DPREFIX=%/t
+// RUN: cat %t/deps_tu_incremental.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=name,clang-module-deps,clang-modulemap-file,command-line,context-hash,file-deps,link-libraries,clang-context-hash,executable,input-file \
+// RUN: | FileCheck %s --check-prefix=CHECK-TU-INCREMENTAL -DPREFIX=%/t
// CHECK-TU-INCREMENTAL: {
// CHECK-TU-INCREMENTAL-NEXT: "modules": [
// CHECK-TU-INCREMENTAL-NEXT: {
diff --git a/clang/test/ClangScanDeps/modules-pch-common-submodule.c b/clang/test/ClangScanDeps/modules-pch-common-submodule.c
index 59dd3f172d8ee..78c60c8f3629f 100644
--- a/clang/test/ClangScanDeps/modules-pch-common-submodule.c
+++ b/clang/test/ClangScanDeps/modules-pch-common-submodule.c
@@ -16,15 +16,15 @@
// RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch-common-submodule/cdb_pch.json > %t/cdb.json
// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full \
// RUN: -module-files-dir %t/build > %t/result_pch.json
-// RUN: cat %t/result_pch.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t -check-prefix=CHECK-PCH
+// RUN: cat %t/result_pch.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=name,clang-module-deps,clang-modulemap-file,context-hash,file-deps,link-libraries,clang-context-hash,input-file \
+// RUN: | FileCheck %s -DPREFIX=%/t -check-prefix=CHECK-PCH
//
// CHECK-PCH: {
// CHECK-PCH-NEXT: "modules": [
// CHECK-PCH-NEXT: {
// CHECK-PCH-NEXT: "clang-module-deps": [],
// CHECK-PCH-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap",
-// CHECK-PCH-NEXT: "command-line": [
-// CHECK-PCH: ],
// CHECK-PCH-NEXT: "context-hash": "[[HASH_MOD_COMMON:.*]]",
// CHECK-PCH-NEXT: "file-deps": [
// CHECK-PCH-NEXT: "[[PREFIX]]/module.modulemap",
@@ -44,8 +44,6 @@
// CHECK-PCH-NEXT: "module-name": "ModCommon"
// CHECK-PCH-NEXT: }
// CHECK-PCH-NEXT: ],
-// CHECK-PCH-NEXT: "command-line": [
-// CHECK-PCH: ],
// CHECK-PCH: "file-deps": [
// CHECK-PCH-NEXT: "[[PREFIX]]/pch.h"
// CHECK-PCH-NEXT: ],
@@ -65,15 +63,15 @@
// RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch-common-submodule/cdb_tu.json > %t/cdb.json
// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full \
// RUN: -module-files-dir %t/build > %t/result_tu.json
-// RUN: cat %t/result_tu.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t -check-prefix=CHECK-TU
+// RUN: cat %t/result_tu.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=name,clang-module-deps,clang-modulemap-file,context-hash,file-deps,link-libraries,clang-context-hash,input-file \
+// RUN: | FileCheck %s -DPREFIX=%/t -check-prefix=CHECK-TU
//
// CHECK-TU: {
// CHECK-TU-NEXT: "modules": [
// CHECK-TU-NEXT: {
// CHECK-TU-NEXT: "clang-module-deps": [],
// CHECK-TU-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap",
-// CHECK-TU-NEXT: "command-line": [
-// CHECK-TU: ],
// CHECK-TU-NEXT: "context-hash": "[[HASH_MOD_TU:.*]]",
// CHECK-TU-NEXT: "file-deps": [
// CHECK-TU-NEXT: "[[PREFIX]]/module.modulemap",
@@ -92,8 +90,6 @@
// CHECK-TU-NEXT: "module-name": "ModTU"
// CHECK-TU-NEXT: }
// CHECK-TU-NEXT: ],
-// CHECK-TU-NEXT: "command-line": [
-// CHECK-TU: ],
// CHECK-TU: "file-deps": [
// CHECK-TU-NEXT: "[[PREFIX]]/tu.c",
// CHECK-TU-NEXT: "[[PREFIX]]/pch.h.pch"
diff --git a/clang/test/ClangScanDeps/modules-pch-common-via-submodule.c b/clang/test/ClangScanDeps/modules-pch-common-via-submodule.c
index a12492c0bec70..6d266f15c3ec9 100644
--- a/clang/test/ClangScanDeps/modules-pch-common-via-submodule.c
+++ b/clang/test/ClangScanDeps/modules-pch-common-via-submodule.c
@@ -13,15 +13,15 @@
// RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch-common-via-submodule/cdb_pch.json > %t/cdb.json
// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full \
// RUN: -module-files-dir %t/build > %t/result_pch.json
-// RUN: cat %t/result_pch.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t -check-prefix=CHECK-PCH
+// RUN: cat %t/result_pch.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=name,clang-module-deps,clang-modulemap-file,context-hash,file-deps,link-libraries,clang-context-hash,input-file \
+// RUN: | FileCheck %s -DPREFIX=%/t -check-prefix=CHECK-PCH
//
// CHECK-PCH: {
// CHECK-PCH-NEXT: "modules": [
// CHECK-PCH-NEXT: {
// CHECK-PCH-NEXT: "clang-module-deps": [],
// CHECK-PCH-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap",
-// CHECK-PCH-NEXT: "command-line": [
-// CHECK-PCH: ],
// CHECK-PCH-NEXT: "context-hash": "[[HASH_MOD_COMMON:.*]]",
// CHECK-PCH-NEXT: "file-deps": [
// CHECK-PCH-NEXT: "[[PREFIX]]/module.modulemap",
@@ -40,8 +40,6 @@
// CHECK-PCH-NEXT: "module-name": "ModCommon"
// CHECK-PCH-NEXT: }
// CHECK-PCH-NEXT: ],
-// CHECK-PCH-NEXT: "command-line": [
-// CHECK-PCH: ],
// CHECK-PCH: "file-deps": [
// CHECK-PCH-NEXT: "[[PREFIX]]/pch.h"
// CHECK-PCH-NEXT: ],
@@ -61,15 +59,15 @@
// RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch-common-via-submodule/cdb_tu.json > %t/cdb.json
// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full \
// RUN: -module-files-dir %t/build > %t/result_tu.json
-// RUN: cat %t/result_tu.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t -check-prefix=CHECK-TU
+// RUN: cat %t/result_tu.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=name,clang-module-deps,clang-modulemap-file,context-hash,file-deps,link-libraries,clang-context-hash,input-file \
+// RUN: | FileCheck %s -DPREFIX=%/t -check-prefix=CHECK-TU
//
// CHECK-TU: {
// CHECK-TU-NEXT: "modules": [
// CHECK-TU-NEXT: {
// CHECK-TU-NEXT: "clang-module-deps": [],
// CHECK-TU-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap",
-// CHECK-TU-NEXT: "command-line": [
-// CHECK-TU: ],
// CHECK-TU-NEXT: "context-hash": "[[HASH_MOD_TU:.*]]",
// CHECK-TU-NEXT: "file-deps": [
// CHECK-TU-NEXT: "[[PREFIX]]/module.modulemap",
@@ -89,8 +87,6 @@
// CHECK-TU-NEXT: "module-name": "ModTU"
// CHECK-TU-NEXT: }
// CHECK-TU-NEXT: ],
-// CHECK-TU-NEXT: "command-line": [
-// CHECK-TU: ],
// CHECK-TU: "file-deps": [
// CHECK-TU-NEXT: "[[PREFIX]]/tu.c",
// CHECK-TU-NEXT: "[[PREFIX]]/pch.h.pch"
diff --git a/clang/test/ClangScanDeps/modules-pch.c b/clang/test/ClangScanDeps/modules-pch.c
index 92b7f41ac4b9f..0235c2104bb07 100644
--- a/clang/test/ClangScanDeps/modules-pch.c
+++ b/clang/test/ClangScanDeps/modules-pch.c
@@ -11,7 +11,9 @@
// RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_pch.json > %t/cdb_pch.json
// RUN: clang-scan-deps -compilation-database %t/cdb_pch.json -format experimental-full \
// RUN: -module-files-dir %t/build > %t/result_pch.json
-// RUN: cat %t/result_pch.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t -check-prefix=CHECK-PCH
+// RUN: cat %t/result_pch.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=name,clang-module-deps,clang-modulemap-file,command-line,context-hash,file-deps,link-libraries,clang-context-hash,input-file \
+// RUN: | FileCheck %s -DPREFIX=%/t -check-prefix=CHECK-PCH
//
// Check we didn't build the PCH during dependency scanning.
// RUN: not cat %/t/pch.h.pch
@@ -102,7 +104,9 @@
// RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_tu.json > %t/cdb_tu.json
// RUN: clang-scan-deps -compilation-database %t/cdb_tu.json -format experimental-full \
// RUN: -module-files-dir %t/build > %t/result_tu.json
-// RUN: cat %t/result_tu.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t -check-prefix=CHECK-TU
+// RUN: cat %t/result_tu.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=name,clang-module-deps,clang-modulemap-file,command-line,context-hash,file-deps,link-libraries,clang-context-hash,input-file \
+// RUN: | FileCheck %s -DPREFIX=%/t -check-prefix=CHECK-TU
//
// CHECK-TU: {
// CHECK-TU-NEXT: "modules": [
@@ -152,7 +156,9 @@
// RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_tu_with_common.json > %t/cdb_tu_with_common.json
// RUN: clang-scan-deps -compilation-database %t/cdb_tu_with_common.json -format experimental-full \
// RUN: -module-files-dir %t/build > %t/result_tu_with_common.json
-// RUN: cat %t/result_tu_with_common.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t -check-prefix=CHECK-TU-WITH-COMMON
+// RUN: cat %t/result_tu_with_common.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=name,clang-module-deps,clang-modulemap-file,command-line,context-hash,file-deps,link-libraries,clang-context-hash,input-file \
+// RUN: | FileCheck %s -DPREFIX=%/t -check-prefix=CHECK-TU-WITH-COMMON
//
// CHECK-TU-WITH-COMMON: {
// CHECK-TU-WITH-COMMON-NEXT: "modules": [
diff --git a/clang/test/ClangScanDeps/modules-priv-fw-from-pub.m b/clang/test/ClangScanDeps/modules-priv-fw-from-pub.m
index dd314e1a3006a..6a52bc60f4c10 100644
--- a/clang/test/ClangScanDeps/modules-priv-fw-from-pub.m
+++ b/clang/test/ClangScanDeps/modules-priv-fw-from-pub.m
@@ -32,7 +32,9 @@
// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full > %t/deps.json
// Check that FW is reported to depend on FW_Private.
-// RUN: cat %t/deps.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t
+// RUN: cat %t/deps.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=clang-module-deps,clang-modulemap-file,command-line,context-hash,file-deps,link-libraries,name,clang-context-hash,input-file \
+// RUN: | FileCheck %s -DPREFIX=%/t
// CHECK: {
// CHECK-NEXT: "modules": [
// CHECK-NEXT: {
diff --git a/clang/test/ClangScanDeps/multiple-commands.c b/clang/test/ClangScanDeps/multiple-commands.c
index bb169ea10995a..b53e871afe636 100644
--- a/clang/test/ClangScanDeps/multiple-commands.c
+++ b/clang/test/ClangScanDeps/multiple-commands.c
@@ -14,7 +14,9 @@
// RUN: -j 1 -format experimental-full -mode preprocess-dependency-directives \
// RUN: > %t/deps.json
-// RUN: cat %t/deps.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t
+// RUN: cat %t/deps.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=clang-modulemap-file,name,clang-context-hash,clang-module-deps,command-line,executable,input-file \
+// RUN: | FileCheck %s -DPREFIX=%/t
// Build the -save-temps + -fmodules case
// RUN: %deps-to-rsp %t/deps.json --module-name=Mod > %t/Mod.rsp
diff --git a/clang/test/ClangScanDeps/tu-buffer.c b/clang/test/ClangScanDeps/tu-buffer.c
index b450b13ff434b..469af921f83c4 100644
--- a/clang/test/ClangScanDeps/tu-buffer.c
+++ b/clang/test/ClangScanDeps/tu-buffer.c
@@ -35,7 +35,9 @@ module addition { header "addition.h" }
// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json
// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -tu-buffer-path %t/tu.c > %t/result.json
-// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s --check-prefix=CHECK
+// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' \
+// RUN: | %scan-deps-filter --fields=clang-module-deps,clang-modulemap-file,context-hash,file-deps,name,clang-context-hash,input-file \
+// RUN: | FileCheck -DPREFIX=%/t %s --check-prefix=CHECK
// CHECK: {
// CHECK-NEXT: "modules": [
@@ -47,8 +49,6 @@ module addition { header "addition.h" }
// CHECK-NEXT: }
// CHECK-NEXT: ],
// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap",
-// CHECK-NEXT: "command-line": [
-// CHECK: ],
// CHECK-NEXT: "context-hash": "{{.*}}",
// CHECK-NEXT: "file-deps": [
// CHECK-NEXT: "[[PREFIX]]/module.modulemap"
@@ -64,8 +64,6 @@ module addition { header "addition.h" }
// CHECK-NEXT: }
// CHECK-NEXT: ],
// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap",
-// CHECK-NEXT: "command-line": [
-// CHECK: ],
// CHECK-NEXT: "context-hash": "{{.*}}",
// CHECK-NEXT: "file-deps": [
// CHECK-NEXT: "[[PREFIX]]/module.modulemap"
@@ -77,8 +75,6 @@ module addition { header "addition.h" }
// CHECK-NEXT: {
// CHECK-NEXT: "clang-module-deps": [],
// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap",
-// CHECK-NEXT: "command-line": [
-// CHECK: ],
// CHECK-NEXT: "context-hash": "{{.*}}",
// CHECK-NEXT: "file-deps": [
// CHECK-NEXT: "[[PREFIX]]/module.modulemap"
@@ -98,8 +94,6 @@ module addition { header "addition.h" }
// CHECK-NEXT: "module-name": "root"
// CHECK-NEXT: }
// CHECK-NEXT: ],
-// CHECK-NEXT: "command-line": [
-// CHECK: ],
// CHECK: "file-deps": [
// CHECK-NEXT: [[PREFIX]]/tu.c
// CHECK-NEXT: ],
>From ea165572a5e649b1ab8181322ff6d759ca8adceb Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Mon, 6 Jul 2026 17:41:36 +0200
Subject: [PATCH 10/46] IR: Use switch in getFloatingPointType (#207750)
---
llvm/lib/IR/Type.cpp | 34 +++++++++++++++++-----------------
1 file changed, 17 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp
index d5f4671487acb..8c108a4d1f275 100644
--- a/llvm/lib/IR/Type.cpp
+++ b/llvm/lib/IR/Type.cpp
@@ -123,24 +123,24 @@ bool Type::isScalableTargetExtTy() const {
}
Type *Type::getFloatingPointTy(LLVMContext &C, const fltSemantics &S) {
- Type *Ty;
- if (&S == &APFloat::IEEEhalf())
- Ty = Type::getHalfTy(C);
- else if (&S == &APFloat::BFloat())
- Ty = Type::getBFloatTy(C);
- else if (&S == &APFloat::IEEEsingle())
- Ty = Type::getFloatTy(C);
- else if (&S == &APFloat::IEEEdouble())
- Ty = Type::getDoubleTy(C);
- else if (&S == &APFloat::x87DoubleExtended())
- Ty = Type::getX86_FP80Ty(C);
- else if (&S == &APFloat::IEEEquad())
- Ty = Type::getFP128Ty(C);
- else {
- assert(&S == &APFloat::PPCDoubleDouble() && "Unknown FP format");
- Ty = Type::getPPC_FP128Ty(C);
+ switch (llvm::APFloat::SemanticsToEnum(S)) {
+ case llvm::APFloat::S_IEEEhalf:
+ return llvm::Type::getHalfTy(C);
+ case llvm::APFloat::S_BFloat:
+ return llvm::Type::getBFloatTy(C);
+ case llvm::APFloat::S_IEEEsingle:
+ return llvm::Type::getFloatTy(C);
+ case llvm::APFloat::S_IEEEdouble:
+ return llvm::Type::getDoubleTy(C);
+ case llvm::APFloat::S_IEEEquad:
+ return llvm::Type::getFP128Ty(C);
+ case llvm::APFloat::S_PPCDoubleDouble:
+ return llvm::Type::getPPC_FP128Ty(C);
+ case llvm::APFloat::S_x87DoubleExtended:
+ return llvm::Type::getX86_FP80Ty(C);
+ default:
+ llvm_unreachable("unhandled float format");
}
- return Ty;
}
bool Type::isRISCVVectorTupleTy() const {
>From 4ad89d11db9e4048328de408af5ae0503861fe4c Mon Sep 17 00:00:00 2001
From: Kai Nacke <kai.peter.nacke at ibm.com>
Date: Mon, 6 Jul 2026 11:47:07 -0400
Subject: [PATCH 11/46] [SystemZ][z/OS] Remove temporary gnu as output on z/OS
(#181906)
On z/OS, there is only HLASM as system assembler available. Since all
LIT tests are migrated, the temporary option can now be removed.
---
.../MCTargetDesc/SystemZHLASMAsmStreamer.cpp | 99 +++++++++++++++++++
.../MCTargetDesc/SystemZHLASMAsmStreamer.h | 4 +
.../MCTargetDesc/SystemZMCTargetDesc.cpp | 10 +-
3 files changed, 105 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.cpp
index 3999534fac1b8..6d6b25efbf525 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.cpp
@@ -115,6 +115,12 @@ void SystemZHLASMAsmStreamer::emitAlignmentDS(uint64_t ByteAlignment,
EmitEOL();
}
+raw_ostream &SystemZHLASMAsmStreamer::getCommentOS() {
+ if (!IsVerboseAsm)
+ return nulls(); // Discard comments unless in verbose asm mode.
+ return CommentStream;
+}
+
void SystemZHLASMAsmStreamer::AddComment(const Twine &T, bool EOL) {
if (!IsVerboseAsm)
return;
@@ -186,6 +192,99 @@ void SystemZHLASMAsmStreamer::emitBytes(StringRef Data) {
EmitEOL();
}
+void SystemZHLASMAsmStreamer::addEncodingComment(const MCInst &Inst,
+ const MCSubtargetInfo &STI) {
+ raw_ostream &OS = getCommentOS();
+ SmallString<256> Code;
+ SmallVector<MCFixup, 4> Fixups;
+
+ // If we have no code emitter, don't emit code.
+ if (!getAssembler().getEmitterPtr())
+ return;
+
+ getAssembler().getEmitter().encodeInstruction(Inst, Code, Fixups, STI);
+
+ // If we are showing fixups, create symbolic markers in the encoded
+ // representation. We do this by making a per-bit map to the fixup item index,
+ // then trying to display it as nicely as possible.
+ SmallVector<uint8_t, 64> FixupMap;
+ FixupMap.resize(Code.size() * 8);
+ for (unsigned I = 0, E = Code.size() * 8; I != E; ++I)
+ FixupMap[I] = 0;
+
+ for (unsigned I = 0, E = Fixups.size(); I != E; ++I) {
+ MCFixup &F = Fixups[I];
+ MCFixupKindInfo Info =
+ getAssembler().getBackend().getFixupKindInfo(F.getKind());
+ for (unsigned J = 0; J != Info.TargetSize; ++J) {
+ unsigned Index = F.getOffset() * 8 + Info.TargetOffset + J;
+ assert(Index < Code.size() * 8 && "Invalid offset in fixup!");
+ FixupMap[Index] = 1 + I;
+ }
+ }
+
+ OS << "encoding: [";
+ for (unsigned I = 0, E = Code.size(); I != E; ++I) {
+ if (I)
+ OS << ',';
+
+ // See if all bits are the same map entry.
+ uint8_t MapEntry = FixupMap[I * 8 + 0];
+ for (unsigned J = 1; J != 8; ++J) {
+ if (FixupMap[I * 8 + J] == MapEntry)
+ continue;
+
+ MapEntry = uint8_t(~0U);
+ break;
+ }
+
+ if (MapEntry != uint8_t(~0U)) {
+ if (MapEntry == 0) {
+ OS << format("0x%02x", uint8_t(Code[I]));
+ } else {
+ if (Code[I]) {
+ // FIXME: Some of the 8 bits require fix up.
+ OS << format("0x%02x", uint8_t(Code[I])) << '\''
+ << char('A' + MapEntry - 1) << '\'';
+ } else
+ OS << char('A' + MapEntry - 1);
+ }
+ } else {
+ // Otherwise, write out in binary.
+ OS << "0b";
+ for (unsigned J = 8; J--;) {
+ unsigned Bit = (Code[I] >> J) & 1;
+ unsigned FixupBit = I * 8 + (7 - J);
+ if (uint8_t MapEntry = FixupMap[FixupBit]) {
+ assert(Bit == 0 && "Encoder wrote into fixed up bit!");
+ OS << char('A' + MapEntry - 1);
+ } else
+ OS << Bit;
+ }
+ }
+ }
+ OS << "]\n";
+
+ for (unsigned I = 0, E = Fixups.size(); I != E; ++I) {
+ MCFixup &F = Fixups[I];
+ OS << " fixup " << char('A' + I) << " - "
+ << "offset: " << F.getOffset() << ", value: ";
+ MAI->printExpr(OS, *F.getValue());
+ auto Kind = F.getKind();
+ if (mc::isRelocation(Kind))
+ OS << ", relocation type: " << Kind;
+ else {
+ OS << ", kind: ";
+ auto Info = getAssembler().getBackend().getFixupKindInfo(Kind);
+ if (F.isPCRel() && StringRef(Info.Name).starts_with("FK_Data_"))
+ OS << "FK_PCRel_" << (Info.TargetSize / 8);
+ else
+ OS << Info.Name;
+ }
+ OS << "\n";
+ }
+}
+
void SystemZHLASMAsmStreamer::emitInstruction(const MCInst &Inst,
const MCSubtargetInfo &STI) {
// Show the encoding in a comment if we have a code emitter.
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.h b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.h
index 7a0eab9b940c3..41f388ca8dcce 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.h
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.h
@@ -71,6 +71,8 @@ class SystemZHLASMAsmStreamer final : public MCAsmBaseStreamer {
/// and only when verbose assembly output is enabled.
void AddComment(const Twine &T, bool EOL = true) override;
+ raw_ostream &getCommentOS() override;
+
void emitBytes(StringRef Data) override;
void emitAlignmentDS(uint64_t ByteAlignment, std::optional<int64_t> Value,
@@ -87,6 +89,8 @@ class SystemZHLASMAsmStreamer final : public MCAsmBaseStreamer {
/// Do we support EmitRawText?
bool hasRawTextSupport() const override { return true; }
+ void addEncodingComment(const MCInst &Inst, const MCSubtargetInfo &STI);
+
/// @name MCStreamer Interface
/// @{
void visitUsedSymbol(const MCSymbol &Sym) override;
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp
index 5d605964cf2c0..a0de6949f343e 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp
@@ -37,12 +37,6 @@ using namespace llvm;
#define GET_REGINFO_MC_DESC
#include "SystemZGenRegisterInfo.inc"
-// Temporary option to assist with the migration to a new HLASMAsmStreamer on
-// z/OS
-static cl::opt<bool> GNUAsOnzOSCL("emit-gnuas-syntax-on-zos",
- cl::desc("Emit GNU Assembly Syntax on z/OS."),
- cl::init(false));
-
const unsigned SystemZMC::GR32Regs[16] = {
SystemZ::R0L, SystemZ::R1L, SystemZ::R2L, SystemZ::R3L,
SystemZ::R4L, SystemZ::R5L, SystemZ::R6L, SystemZ::R7L,
@@ -203,7 +197,7 @@ static MCInstPrinter *createSystemZMCInstPrinter(const Triple &T,
static MCTargetStreamer *createAsmTargetStreamer(MCStreamer &S,
formatted_raw_ostream &OS,
MCInstPrinter *InstPrint) {
- if (S.getContext().getTargetTriple().isOSzOS() && !GNUAsOnzOSCL)
+ if (S.getContext().getTargetTriple().isOSzOS())
return new SystemZTargetHLASMStreamer(S, OS);
else
return new SystemZTargetGNUStreamer(S, OS);
@@ -215,7 +209,7 @@ static MCStreamer *createSystemZAsmStreamer(
std::unique_ptr<MCAsmBackend> TAB) {
auto TT = Ctx.getTargetTriple();
- if (TT.isOSzOS() && !GNUAsOnzOSCL)
+ if (TT.isOSzOS())
return new SystemZHLASMAsmStreamer(Ctx, std::move(OS), std::move(IP),
std::move(CE), std::move(TAB));
>From 60cccae52fda78a9cdc9323900e14952d71b2624 Mon Sep 17 00:00:00 2001
From: Chris B <chris.bieneman at me.com>
Date: Mon, 6 Jul 2026 10:48:40 -0500
Subject: [PATCH 12/46] [NFC] Remove dead code to fix build (#207753)
LLVM changed the interfaces around inline assembly, and while we could
have updated this code, in practice DXIL doesn't allow inline assembly
so I removed it.
---
llvm/lib/Target/DirectX/DXILWriter/DXILBitcodeWriter.cpp | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/DirectX/DXILWriter/DXILBitcodeWriter.cpp b/llvm/lib/Target/DirectX/DXILWriter/DXILBitcodeWriter.cpp
index 4680ba1630734..6f14f70014b01 100644
--- a/llvm/lib/Target/DirectX/DXILWriter/DXILBitcodeWriter.cpp
+++ b/llvm/lib/Target/DirectX/DXILWriter/DXILBitcodeWriter.cpp
@@ -1186,9 +1186,8 @@ void DXILBitcodeWriter::writeModuleInfo() {
writeStringRecord(Stream, bitc::MODULE_CODE_TRIPLE, Triple, 0 /*TODO*/);
writeStringRecord(Stream, bitc::MODULE_CODE_DATALAYOUT, DL, 0 /*TODO*/);
- if (!M.getModuleInlineAsm().empty())
- writeStringRecord(Stream, bitc::MODULE_CODE_ASM, M.getModuleInlineAsm(),
- 0 /*TODO*/);
+ // The original bitcode writer wrote inline assembly here. Inline assembly
+ // isn't valid in DXIL, so this is removed.
// Emit information about sections and GC, computing how many there are. Also
// compute the maximum alignment value.
>From d3d3fdd940d714af2ac060b9f3da3ed32bdbaa05 Mon Sep 17 00:00:00 2001
From: Corentin Jabot <corentinjabot at gmail.com>
Date: Mon, 6 Jul 2026 17:54:50 +0200
Subject: [PATCH 13/46] Disable test not supported on MSVC (#207752)
Fixes an Incorrectly applied merge conflict resolution after landing
#206990
---
clang/unittests/Basic/DiagnosticTest.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/clang/unittests/Basic/DiagnosticTest.cpp b/clang/unittests/Basic/DiagnosticTest.cpp
index 986eb058ece07..13553491397f2 100644
--- a/clang/unittests/Basic/DiagnosticTest.cpp
+++ b/clang/unittests/Basic/DiagnosticTest.cpp
@@ -452,7 +452,7 @@ TEST_F(SuppressionMappingTest, CanonicalizesSlashesOnWindows) {
TEST(EscapeSingleCodepointForDiagnosticTest, printableDisplaysQuoted) {
EXPECT_EQ(EscapeSingleCodepointForDiagnostic(U'A'), "'A'");
// This test fails when msvc is not using /utf-8.
- EXPECT_EQ(EscapeSingleCodepointForDiagnostic(U'🤡'), "'🤡' U+1F921");
+ // EXPECT_EQ(EscapeSingleCodepointForDiagnostic(U'🤡'), "'🤡' U+1F921");
EXPECT_EQ(EscapeSingleCodepointForDiagnostic(U' '), "' '");
}
>From f0bbde8630ed94d749b95624799659d6ccbb9418 Mon Sep 17 00:00:00 2001
From: Cheng Lingfei <53817093+clingfei at users.noreply.github.com>
Date: Tue, 7 Jul 2026 00:20:43 +0800
Subject: [PATCH 14/46] [AArch64] optimize lowering for icmp on i128 when RHS
is an immediate (#181822)
Optimize cases including eq, ne, ult, ule, ugt, and uge.
Closes #161273.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 128 ++++++++++-
.../CodeGen/AArch64/i128-imm-compare-ccmp.ll | 203 ++++++++++++++++++
llvm/test/CodeGen/AArch64/isinf.ll | 10 +-
.../umulo-128-legalisation-lowering.ll | 6 +-
4 files changed, 328 insertions(+), 19 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/i128-imm-compare-ccmp.ll
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 7df8aa045dbf9..8de465afa1ece 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -12186,32 +12186,98 @@ SDValue AArch64TargetLowering::LowerBitreverse(SDValue Op,
DAG.getNode(ISD::BITREVERSE, DL, VST, REVB));
}
+// A CCMP folds in only a 5-bit unsigned immediate (or its negation, via CCMN);
+// any other constant must be materialized into a register first.
+static bool isLegalCondCmpImmediate(const APInt &Imm) {
+ return Imm.sgt(-32) && Imm.slt(32);
+}
+
+// True unless one of the operands is a constant that cannot be encoded as a
+// CMP/CMN immediate. A non-constant operand always lives in a register, so it
+// is fine.
+static bool hasLegalCmpImmediate(SDValue LHS, SDValue RHS) {
+ ConstantSDNode *C = dyn_cast<ConstantSDNode>(LHS);
+ if (C && !llvm::AArch64_AM::isLegalCmpImmed(C->getAPIntValue()))
+ return false;
+ C = dyn_cast<ConstantSDNode>(RHS);
+ if (C && !llvm::AArch64_AM::isLegalCmpImmed(C->getAPIntValue()))
+ return false;
+ return true;
+}
+
+// As hasLegalCmpImmediate, but for the tighter CCMP immediate form.
+static bool hasLegalCondCmpImmediate(SDValue LHS, SDValue RHS) {
+ ConstantSDNode *C = dyn_cast<ConstantSDNode>(LHS);
+ if (C && !isLegalCondCmpImmediate(C->getAPIntValue()))
+ return false;
+ C = dyn_cast<ConstantSDNode>(RHS);
+ if (C && !isLegalCondCmpImmediate(C->getAPIntValue()))
+ return false;
+ return true;
+}
+
+// True if either operand is a constant that does not fit a CCMP immediate but
+// is a legal logical immediate: folding it into the xor is a single cheap
+// instruction, yet the result still needs a scratch register for the CCMP.
+// Chaining more than two such compares costs more than the CCMP form saves.
+static bool hasCheapXorImmediateThatNeedsCondCmpReg(
+ const std::pair<SDValue, SDValue> &Pair) {
+ for (SDValue V : {Pair.first, Pair.second}) {
+ auto *C = dyn_cast<ConstantSDNode>(V);
+ if (!C || isLegalCondCmpImmediate(C->getAPIntValue()))
+ continue;
+ const APInt &Imm = C->getAPIntValue();
+ unsigned BitWidth = Imm.getBitWidth() <= 32 ? 32 : 64;
+ if (Imm.getBitWidth() <= BitWidth &&
+ AArch64_AM::isLogicalImmediate(Imm.getZExtValue(), BitWidth))
+ return true;
+ }
+ return false;
+}
+
// Check whether the continuous comparison sequence.
static bool
-isOrXorChain(SDValue N, unsigned &Num,
- SmallVector<std::pair<SDValue, SDValue>, 16> &WorkList) {
- if (Num == MaxXors)
+isOrXorChain(SDValue N, SelectionDAG &DAG, unsigned &NumLeaves,
+ unsigned &NumXors, bool &SawXor, bool RequireLegalCmpImmediates,
+ SmallVectorImpl<std::pair<SDValue, SDValue>> &WorkList) {
+ if (NumLeaves == MaxXors)
return false;
// Skip the one-use zext
if (N->getOpcode() == ISD::ZERO_EXTEND && N->hasOneUse())
N = N->getOperand(0);
- // The leaf node must be XOR
if (N->getOpcode() == ISD::XOR) {
+ if (RequireLegalCmpImmediates &&
+ !hasLegalCmpImmediate(N->getOperand(0), N->getOperand(1)))
+ return false;
WorkList.push_back(std::make_pair(N->getOperand(0), N->getOperand(1)));
- Num++;
+ NumLeaves++;
+ NumXors++;
+ SawXor = true;
return true;
}
// All the non-leaf nodes must be OR.
- if (N->getOpcode() != ISD::OR || !N->hasOneUse())
+ if (N->getOpcode() == ISD::OR && N->hasOneUse())
+ return isOrXorChain(N->getOperand(0), DAG, NumLeaves, NumXors, SawXor,
+ RequireLegalCmpImmediates, WorkList) &&
+ isOrXorChain(N->getOperand(1), DAG, NumLeaves, NumXors, SawXor,
+ RequireLegalCmpImmediates, WorkList);
+ if (N->getOpcode() == ISD::OR)
return false;
- if (isOrXorChain(N->getOperand(0), Num, WorkList) &&
- isOrXorChain(N->getOperand(1), Num, WorkList))
- return true;
- return false;
+ EVT VT = N.getValueType();
+ if (!VT.isScalarInteger())
+ return false;
+
+ // A xor with zero may have been folded away before this combine sees it.
+ // Treat such leaves as comparisons with zero so the original OR/XOR form and
+ // type-legalized wide integer equality compares converge to the same SETCC
+ // tree.
+ WorkList.push_back(std::make_pair(N, DAG.getConstant(0, SDLoc(N), VT)));
+ NumLeaves++;
+ return true;
}
// Transform chains of ORs and XORs, which usually outlined by memcmp/bmp.
@@ -12229,10 +12295,50 @@ static SDValue performOrXorChainCombine(SDNode *N, SelectionDAG &DAG) {
ISD::CondCode Cond = cast<CondCodeSDNode>(N->getOperand(2))->get();
// Try to express conjunction "cmp 0 (or (xor A0 A1) (xor B0 B1))" as:
// sub A0, A1; ccmp B0, B1, 0, eq; cmp inv(Cond) flag
+ unsigned NumLeaves = 0;
unsigned NumXors = 0;
+ bool SawXor = false;
+ bool RequireLegalCmpImmediates = any_of(N->users(), [](SDNode *User) {
+ return User->getOpcode() == ISD::BRCOND ||
+ User->getOpcode() == AArch64ISD::BRCOND;
+ });
if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && isNullConstant(RHS) &&
LHS->getOpcode() == ISD::OR && LHS->hasOneUse() &&
- isOrXorChain(LHS, NumXors, WorkList)) {
+ isOrXorChain(LHS, DAG, NumLeaves, NumXors, SawXor,
+ RequireLegalCmpImmediates, WorkList) &&
+ SawXor) {
+ // A CCMP sequence serializes the comparisons through NZCV. Keep the
+ // default transform to short chains, but account for real XOR leaves: each
+ // one removed is a code-size and front-end win. Under size optimization,
+ // prefer the smaller CCMP form unless another guard rejects it.
+ const Function &F = DAG.getMachineFunction().getFunction();
+ unsigned Limit = 5;
+ if (NumXors >= 6) {
+ Limit = 6;
+ if (NumXors == NumLeaves)
+ Limit = std::min<unsigned>(8, NumXors);
+ }
+ if (F.hasMinSize())
+ Limit = MaxXors;
+ if (WorkList.size() > Limit)
+ return SDValue();
+
+ if (WorkList.size() > 2 &&
+ any_of(WorkList, hasCheapXorImmediateThatNeedsCondCmpReg))
+ return SDValue();
+
+ // Only the leading compare of the chain uses the wider CMP immediate; the
+ // rest become CCMPs. So a compare whose immediate is a legal CMP operand
+ // but not a legal CCMP operand is cheapest at the front.
+ auto PreferAsFirstCmp = [](const std::pair<SDValue, SDValue> &Pair) {
+ return hasLegalCmpImmediate(Pair.first, Pair.second) &&
+ !hasLegalCondCmpImmediate(Pair.first, Pair.second);
+ };
+ SmallVector<std::pair<SDValue, SDValue>, 16>::iterator First =
+ find_if(WorkList, PreferAsFirstCmp);
+ if (First != WorkList.end())
+ std::iter_swap(WorkList.begin(), First);
+
SDValue XOR0, XOR1;
std::tie(XOR0, XOR1) = WorkList[0];
unsigned LogicOp = (Cond == ISD::SETEQ) ? ISD::AND : ISD::OR;
diff --git a/llvm/test/CodeGen/AArch64/i128-imm-compare-ccmp.ll b/llvm/test/CodeGen/AArch64/i128-imm-compare-ccmp.ll
new file mode 100644
index 0000000000000..77e697781c006
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/i128-imm-compare-ccmp.ll
@@ -0,0 +1,203 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -O3 -o - < %s | FileCheck %s
+
+declare void @foo()
+
+define i1 @eq_imm(i128 %x) {
+; CHECK-LABEL: eq_imm:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmp x0, #5
+; CHECK-NEXT: ccmp x1, #0, #0, eq
+; CHECK-NEXT: cset w0, eq
+; CHECK-NEXT: ret
+ %cmp = icmp eq i128 %x, 5
+ ret i1 %cmp
+}
+
+define i1 @ne_imm(i128 %x) {
+; CHECK-LABEL: ne_imm:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmp x0, #5
+; CHECK-NEXT: ccmp x1, #0, #0, eq
+; CHECK-NEXT: cset w0, ne
+; CHECK-NEXT: ret
+ %cmp = icmp ne i128 %x, 5
+ ret i1 %cmp
+}
+
+define i1 @eq_or_xor_zero_hi(i64 %lo, i64 %hi) {
+; CHECK-LABEL: eq_or_xor_zero_hi:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmp x0, #5
+; CHECK-NEXT: ccmp x1, #0, #0, eq
+; CHECK-NEXT: cset w0, eq
+; CHECK-NEXT: ret
+ %xorlo = xor i64 %lo, 5
+ %or = or i64 %xorlo, %hi
+ %cmp = icmp eq i64 %or, 0
+ ret i1 %cmp
+}
+
+define i1 @ne_or_xor_zero_hi(i64 %lo, i64 %hi) {
+; CHECK-LABEL: ne_or_xor_zero_hi:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmp x0, #5
+; CHECK-NEXT: ccmp x1, #0, #0, eq
+; CHECK-NEXT: cset w0, ne
+; CHECK-NEXT: ret
+ %xorlo = xor i64 %lo, 5
+ %or = or i64 %xorlo, %hi
+ %cmp = icmp ne i64 %or, 0
+ ret i1 %cmp
+}
+
+define void @eq_or_xor_large_cmp_imm_branch(i64 %lo, i64 %hi) {
+; CHECK-LABEL: eq_or_xor_large_cmp_imm_branch:
+; CHECK: // %bb.0:
+; CHECK-NEXT: eor x8, x1, #0x8000000000000000
+; CHECK-NEXT: orr x8, x0, x8
+; CHECK-NEXT: cbz x8, .LBB4_2
+; CHECK-NEXT: // %bb.1: // %common.ret
+; CHECK-NEXT: ret
+; CHECK-NEXT: .LBB4_2: // %true
+; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: .cfi_offset w30, -16
+; CHECK-NEXT: bl foo
+; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %xor = xor i64 %hi, -9223372036854775808
+ %or = or i64 %lo, %xor
+ %cmp = icmp eq i64 %or, 0
+ br i1 %cmp, label %true, label %false
+
+true:
+ call void @foo()
+ ret void
+
+false:
+ ret void
+}
+
+define void @eq_or_xor_large_condcmp_imm_branch(i64 %a, i64 %b, i64 %c, i64 %d) {
+; CHECK-LABEL: eq_or_xor_large_condcmp_imm_branch:
+; CHECK: // %bb.0:
+; CHECK-NEXT: eor x8, x3, #0xff
+; CHECK-NEXT: orr x9, x0, x1
+; CHECK-NEXT: orr x8, x2, x8
+; CHECK-NEXT: orr x8, x9, x8
+; CHECK-NEXT: cbz x8, .LBB5_2
+; CHECK-NEXT: // %bb.1: // %common.ret
+; CHECK-NEXT: ret
+; CHECK-NEXT: .LBB5_2: // %true
+; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: .cfi_offset w30, -16
+; CHECK-NEXT: bl foo
+; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %xor = xor i64 %d, 255
+ %or0 = or i64 %a, %b
+ %or1 = or i64 %c, %xor
+ %or = or i64 %or0, %or1
+ %cmp = icmp eq i64 %or, 0
+ br i1 %cmp, label %true, label %false
+
+true:
+ call void @foo()
+ ret void
+
+false:
+ ret void
+}
+
+define i1 @eq_or_xor_two_condcmp_imm_reordered(i64 %zero, i64 %value) {
+; CHECK-LABEL: eq_or_xor_two_condcmp_imm_reordered:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmp x1, #255
+; CHECK-NEXT: ccmp x0, #0, #0, eq
+; CHECK-NEXT: cset w0, eq
+; CHECK-NEXT: ret
+ %xor = xor i64 %value, 255
+ %or = or i64 %zero, %xor
+ %cmp = icmp eq i64 %or, 0
+ ret i1 %cmp
+}
+
+define void @eq_or_xor_multiple_large_condcmp_imm_branch(i64 %a, i64 %b, i64 %c, i64 %d) {
+; CHECK-LABEL: eq_or_xor_multiple_large_condcmp_imm_branch:
+; CHECK: // %bb.0:
+; CHECK-NEXT: eor x8, x1, #0xff
+; CHECK-NEXT: eor x9, x3, #0xffff
+; CHECK-NEXT: orr x8, x0, x8
+; CHECK-NEXT: orr x9, x2, x9
+; CHECK-NEXT: orr x8, x8, x9
+; CHECK-NEXT: cbz x8, .LBB7_2
+; CHECK-NEXT: // %bb.1: // %common.ret
+; CHECK-NEXT: ret
+; CHECK-NEXT: .LBB7_2: // %true
+; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: .cfi_offset w30, -16
+; CHECK-NEXT: bl foo
+; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %xorb = xor i64 %b, 255
+ %xord = xor i64 %d, 65535
+ %or0 = or i64 %a, %xorb
+ %or1 = or i64 %c, %xord
+ %or = or i64 %or0, %or1
+ %cmp = icmp eq i64 %or, 0
+ br i1 %cmp, label %true, label %false
+
+true:
+ call void @foo()
+ ret void
+
+false:
+ ret void
+}
+
+define i1 @ult_imm(i128 %x) {
+; CHECK-LABEL: ult_imm:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmp x1, #0
+; CHECK-NEXT: ccmp x0, #5, #2, eq
+; CHECK-NEXT: cset w0, lo
+; CHECK-NEXT: ret
+ %cmp = icmp ult i128 %x, 5
+ ret i1 %cmp
+}
+
+define i1 @ule_imm(i128 %x) {
+; CHECK-LABEL: ule_imm:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmp x1, #0
+; CHECK-NEXT: ccmp x0, #6, #2, eq
+; CHECK-NEXT: cset w0, lo
+; CHECK-NEXT: ret
+ %cmp = icmp ule i128 %x, 5
+ ret i1 %cmp
+}
+
+define i1 @ugt_imm(i128 %x) {
+; CHECK-LABEL: ugt_imm:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmp x1, #0
+; CHECK-NEXT: ccmp x0, #5, #2, eq
+; CHECK-NEXT: cset w0, hi
+; CHECK-NEXT: ret
+ %cmp = icmp ugt i128 %x, 5
+ ret i1 %cmp
+}
+
+define i1 @uge_imm(i128 %x) {
+; CHECK-LABEL: uge_imm:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmp x1, #0
+; CHECK-NEXT: ccmp x0, #4, #2, eq
+; CHECK-NEXT: cset w0, hi
+; CHECK-NEXT: ret
+ %cmp = icmp uge i128 %x, 5
+ ret i1 %cmp
+}
diff --git a/llvm/test/CodeGen/AArch64/isinf.ll b/llvm/test/CodeGen/AArch64/isinf.ll
index f3283d2cf7ec2..249a339f2de0a 100644
--- a/llvm/test/CodeGen/AArch64/isinf.ll
+++ b/llvm/test/CodeGen/AArch64/isinf.ll
@@ -82,11 +82,11 @@ define i32 @replace_isinf_call_f128(fp128 %x) {
; CHECK-SD: // %bb.0:
; CHECK-SD-NEXT: str q0, [sp, #-16]!
; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT: ldp x9, x8, [sp], #16
-; CHECK-SD-NEXT: and x8, x8, #0x7fffffffffffffff
-; CHECK-SD-NEXT: eor x8, x8, #0x7fff000000000000
-; CHECK-SD-NEXT: orr x8, x9, x8
-; CHECK-SD-NEXT: cmp x8, #0
+; CHECK-SD-NEXT: mov x8, #-562949953421312 // =0xfffe000000000000
+; CHECK-SD-NEXT: ldp x10, x9, [sp], #16
+; CHECK-SD-NEXT: lsl x9, x9, #1
+; CHECK-SD-NEXT: cmp x10, #0
+; CHECK-SD-NEXT: ccmp x8, x9, #0, eq
; CHECK-SD-NEXT: cset w0, eq
; CHECK-SD-NEXT: ret
;
diff --git a/llvm/test/CodeGen/AArch64/umulo-128-legalisation-lowering.ll b/llvm/test/CodeGen/AArch64/umulo-128-legalisation-lowering.ll
index d7348ea154b24..f9993e53c318a 100644
--- a/llvm/test/CodeGen/AArch64/umulo-128-legalisation-lowering.ll
+++ b/llvm/test/CodeGen/AArch64/umulo-128-legalisation-lowering.ll
@@ -81,9 +81,9 @@ define i128 @__muloti4(i128 %0, i128 %1, ptr nocapture nonnull writeonly align 4
; AARCH-NEXT: mov w9, wzr
; AARCH-NEXT: mul x0, x0, x2
; AARCH-NEXT: .LBB1_3: // %overflow.res
-; AARCH-NEXT: eor x10, x3, #0x8000000000000000
-; AARCH-NEXT: orr x10, x2, x10
-; AARCH-NEXT: cmp x10, #0
+; AARCH-NEXT: mov x10, #-9223372036854775808 // =0x8000000000000000
+; AARCH-NEXT: cmp x3, x10
+; AARCH-NEXT: ccmp x2, #0, #0, eq
; AARCH-NEXT: ccmp x1, #0, #0, eq
; AARCH-NEXT: b.mi .LBB1_5
; AARCH-NEXT: // %bb.4: // %Else2
>From 1aa74f6d98b357189d735c800510424035edb9d5 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Mon, 6 Jul 2026 09:24:07 -0700
Subject: [PATCH 15/46] [Analysis] Modernize CFGBlock GraphTraits; drop generic
dominator tree special case (#207552)
Make clang analyzer's CFG behave like other graphs in the generic
dominator tree builder by teaching the CFGBlock GraphTraits two things.
* Node numbers: getNumber/getMaxNumber/getNumberEpoch backed by the
dense, never-renumbered BlockIDs. GraphHasNodeNumbers<CFGBlock *>
becomes true (#101706) so Semi-NCA indexes a SmallVector instead of a
DenseMap and DominatorTreeBase drops its CFGBlock-to-index map.
* Plain `CFGBlock *` children: map through the reachable target and skip
`AB_Unreachable` (null) edges. This drops the
is_same_v/make_filter_range
fallback in getChildren() and the CFGBlock ChildrenGetterTy
specialization.
(Post)dominator and IDF results are unchanged.
Aided by Claude Opus 4.8
---
.../clang/Analysis/Analyses/Dominators.h | 27 ---
clang/include/clang/Analysis/CFG.h | 183 ++++++++++++++----
llvm/include/llvm/Support/CFGDiff.h | 3 -
.../llvm/Support/GenericDomTreeConstruction.h | 12 +-
4 files changed, 150 insertions(+), 75 deletions(-)
diff --git a/clang/include/clang/Analysis/Analyses/Dominators.h b/clang/include/clang/Analysis/Analyses/Dominators.h
index 7dd54c5ce262c..3dd5319e36468 100644
--- a/clang/include/clang/Analysis/Analyses/Dominators.h
+++ b/clang/include/clang/Analysis/Analyses/Dominators.h
@@ -185,33 +185,6 @@ using CFGPostDomTree = CFGDominatorTreeImpl</*IsPostDom*/ true>;
template<> void CFGDominatorTreeImpl<true>::anchor();
template<> void CFGDominatorTreeImpl<false>::anchor();
-} // end of namespace clang
-
-namespace llvm {
-namespace IDFCalculatorDetail {
-
-/// Specialize ChildrenGetterTy to skip nullpointer successors.
-template <bool IsPostDom>
-struct ChildrenGetterTy<clang::CFGBlock, IsPostDom> {
- using NodeRef = typename GraphTraits<clang::CFGBlock *>::NodeRef;
- using ChildrenTy = SmallVector<NodeRef, 8>;
-
- ChildrenTy get(const NodeRef &N) {
- using OrderedNodeTy =
- typename IDFCalculatorBase<clang::CFGBlock, IsPostDom>::OrderedNodeTy;
-
- auto Children = children<OrderedNodeTy>(N);
- ChildrenTy Ret{Children.begin(), Children.end()};
- llvm::erase(Ret, nullptr);
- return Ret;
- }
-};
-
-} // end of namespace IDFCalculatorDetail
-} // end of namespace llvm
-
-namespace clang {
-
class ControlDependencyCalculator : public ManagedAnalysis {
using IDFCalculator = llvm::IDFCalculatorBase<CFGBlock, /*IsPostDom=*/true>;
using CFGBlockVector = llvm::SmallVector<CFGBlock *, 4>;
diff --git a/clang/include/clang/Analysis/CFG.h b/clang/include/clang/Analysis/CFG.h
index 2079f99046021..749ed3fc76452 100644
--- a/clang/include/clang/Analysis/CFG.h
+++ b/clang/include/clang/Analysis/CFG.h
@@ -1520,12 +1520,43 @@ class CFG {
Expr *extractElementInitializerFromNestedAILE(const ArrayInitLoopExpr *AILE);
-} // namespace clang
-
//===----------------------------------------------------------------------===//
// GraphTraits specializations for CFG basic block graphs (source-level CFGs)
//===----------------------------------------------------------------------===//
+namespace detail {
+// The GraphTraits below present a CFGBlock's successors and predecessors as
+// plain CFGBlock *, so generic graph algorithms need not know about
+// AdjacentBlock. AB_Unreachable edges, whose reachable target is null, are kept
+// in the CFG to preserve terminator successor positions but are not real
+// control-flow edges and are skipped here.
+struct CFGBlockReachableEdge {
+ bool operator()(const CFGBlock::AdjacentBlock &AB) const {
+ return AB.getReachableBlock() != nullptr;
+ }
+};
+template <typename BlockT> struct CFGBlockEdgeTarget {
+ BlockT *operator()(const CFGBlock::AdjacentBlock &AB) const {
+ return AB.getReachableBlock();
+ }
+};
+
+// A GraphTraits child iterator over BlockT: each AdjacentBlock edge projected
+// to its reachable target, with unreachable (null) edges skipped.
+template <typename IterT, typename BlockT>
+using CFGReachableChildIterator =
+ llvm::mapped_iterator<llvm::filter_iterator<IterT, CFGBlockReachableEdge>,
+ CFGBlockEdgeTarget<BlockT>>;
+
+template <typename BlockT, typename RangeT>
+auto cfgReachableChildren(RangeT Edges) {
+ return llvm::map_range(
+ llvm::make_filter_range(Edges, CFGBlockReachableEdge()),
+ CFGBlockEdgeTarget<BlockT>());
+}
+} // namespace detail
+} // namespace clang
+
namespace llvm {
/// Implement simplify_type for CFGTerminator, so that we can dyn_cast from
@@ -1540,62 +1571,129 @@ template <> struct simplify_type< ::clang::CFGTerminator> {
// Traits for: CFGBlock
-template <> struct GraphTraits< ::clang::CFGBlock *> {
- using NodeRef = ::clang::CFGBlock *;
- using ChildIteratorType = ::clang::CFGBlock::succ_iterator;
+// CFGBlocks are numbered densely from 0 by their BlockID, so dominator tree
+// construction can index by number instead of using a DenseMap.
+template <> struct GraphTraits<clang::CFGBlock *> {
+ using NodeRef = clang::CFGBlock *;
+ using ChildIteratorType =
+ clang::detail::CFGReachableChildIterator<clang::CFGBlock::succ_iterator,
+ clang::CFGBlock>;
- static NodeRef getEntryNode(::clang::CFGBlock *BB) { return BB; }
- static ChildIteratorType child_begin(NodeRef N) { return N->succ_begin(); }
- static ChildIteratorType child_end(NodeRef N) { return N->succ_end(); }
+ static NodeRef getEntryNode(clang::CFGBlock *BB) { return BB; }
+ static ChildIteratorType child_begin(NodeRef N) {
+ return clang::detail::cfgReachableChildren<clang::CFGBlock>(N->succs())
+ .begin();
+ }
+ static ChildIteratorType child_end(NodeRef N) {
+ return clang::detail::cfgReachableChildren<clang::CFGBlock>(N->succs())
+ .end();
+ }
+
+ static unsigned getNumber(const clang::CFGBlock *BB) {
+ return BB->getBlockID();
+ }
};
-template <> struct GraphTraits< const ::clang::CFGBlock *> {
- using NodeRef = const ::clang::CFGBlock *;
- using ChildIteratorType = ::clang::CFGBlock::const_succ_iterator;
+static_assert(GraphHasNodeNumbers<clang::CFGBlock *>,
+ "GraphTraits getNumber() not detected");
+
+template <> struct GraphTraits<const clang::CFGBlock *> {
+ using NodeRef = const clang::CFGBlock *;
+ using ChildIteratorType = clang::detail::CFGReachableChildIterator<
+ clang::CFGBlock::const_succ_iterator, const clang::CFGBlock>;
static NodeRef getEntryNode(const clang::CFGBlock *BB) { return BB; }
- static ChildIteratorType child_begin(NodeRef N) { return N->succ_begin(); }
- static ChildIteratorType child_end(NodeRef N) { return N->succ_end(); }
+ static ChildIteratorType child_begin(NodeRef N) {
+ return clang::detail::cfgReachableChildren<const clang::CFGBlock>(
+ N->succs())
+ .begin();
+ }
+ static ChildIteratorType child_end(NodeRef N) {
+ return clang::detail::cfgReachableChildren<const clang::CFGBlock>(
+ N->succs())
+ .end();
+ }
+
+ static unsigned getNumber(const clang::CFGBlock *BB) {
+ return BB->getBlockID();
+ }
};
-template <> struct GraphTraits<Inverse< ::clang::CFGBlock *>> {
- using NodeRef = ::clang::CFGBlock *;
- using ChildIteratorType = ::clang::CFGBlock::const_pred_iterator;
+static_assert(GraphHasNodeNumbers<const clang::CFGBlock *>,
+ "GraphTraits getNumber() not detected");
- static NodeRef getEntryNode(Inverse<::clang::CFGBlock *> G) {
- return G.Graph;
+template <> struct GraphTraits<Inverse<clang::CFGBlock *>> {
+ using NodeRef = clang::CFGBlock *;
+ using ChildIteratorType =
+ clang::detail::CFGReachableChildIterator<clang::CFGBlock::pred_iterator,
+ clang::CFGBlock>;
+
+ static NodeRef getEntryNode(Inverse<clang::CFGBlock *> G) { return G.Graph; }
+ static ChildIteratorType child_begin(NodeRef N) {
+ return clang::detail::cfgReachableChildren<clang::CFGBlock>(N->preds())
+ .begin();
+ }
+ static ChildIteratorType child_end(NodeRef N) {
+ return clang::detail::cfgReachableChildren<clang::CFGBlock>(N->preds())
+ .end();
}
- static ChildIteratorType child_begin(NodeRef N) { return N->pred_begin(); }
- static ChildIteratorType child_end(NodeRef N) { return N->pred_end(); }
+ static unsigned getNumber(const clang::CFGBlock *BB) {
+ return BB->getBlockID();
+ }
};
-template <> struct GraphTraits<Inverse<const ::clang::CFGBlock *>> {
- using NodeRef = const ::clang::CFGBlock *;
- using ChildIteratorType = ::clang::CFGBlock::const_pred_iterator;
+static_assert(GraphHasNodeNumbers<Inverse<clang::CFGBlock *>>,
+ "GraphTraits getNumber() not detected");
+
+template <> struct GraphTraits<Inverse<const clang::CFGBlock *>> {
+ using NodeRef = const clang::CFGBlock *;
+ using ChildIteratorType = clang::detail::CFGReachableChildIterator<
+ clang::CFGBlock::const_pred_iterator, const clang::CFGBlock>;
- static NodeRef getEntryNode(Inverse<const ::clang::CFGBlock *> G) {
+ static NodeRef getEntryNode(Inverse<const clang::CFGBlock *> G) {
return G.Graph;
}
+ static ChildIteratorType child_begin(NodeRef N) {
+ return clang::detail::cfgReachableChildren<const clang::CFGBlock>(
+ N->preds())
+ .begin();
+ }
+ static ChildIteratorType child_end(NodeRef N) {
+ return clang::detail::cfgReachableChildren<const clang::CFGBlock>(
+ N->preds())
+ .end();
+ }
- static ChildIteratorType child_begin(NodeRef N) { return N->pred_begin(); }
- static ChildIteratorType child_end(NodeRef N) { return N->pred_end(); }
+ static unsigned getNumber(const clang::CFGBlock *BB) {
+ return BB->getBlockID();
+ }
};
+static_assert(GraphHasNodeNumbers<Inverse<const clang::CFGBlock *>>,
+ "GraphTraits getNumber() not detected");
+
// Traits for: CFG
-template <> struct GraphTraits< ::clang::CFG* >
- : public GraphTraits< ::clang::CFGBlock *> {
+template <>
+struct GraphTraits<::clang::CFG *> : public GraphTraits<clang::CFGBlock *> {
using nodes_iterator = ::clang::CFG::iterator;
static NodeRef getEntryNode(::clang::CFG *F) { return &F->getEntry(); }
static nodes_iterator nodes_begin(::clang::CFG* F) { return F->nodes_begin();}
static nodes_iterator nodes_end(::clang::CFG* F) { return F->nodes_end(); }
static unsigned size(::clang::CFG* F) { return F->size(); }
+
+ static unsigned getMaxNumber(const ::clang::CFG *F) {
+ return F->getNumBlockIDs();
+ }
+ // Block numbers are fixed when the CFG is built and never change afterwards.
+ static unsigned getNumberEpoch(const ::clang::CFG *F) { return 0; }
};
-template <> struct GraphTraits<const ::clang::CFG* >
- : public GraphTraits<const ::clang::CFGBlock *> {
+template <>
+struct GraphTraits<const ::clang::CFG *>
+ : public GraphTraits<const clang::CFGBlock *> {
using nodes_iterator = ::clang::CFG::const_iterator;
static NodeRef getEntryNode(const ::clang::CFG *F) { return &F->getEntry(); }
@@ -1611,19 +1709,31 @@ template <> struct GraphTraits<const ::clang::CFG* >
static unsigned size(const ::clang::CFG* F) {
return F->size();
}
+
+ static unsigned getMaxNumber(const ::clang::CFG *F) {
+ return F->getNumBlockIDs();
+ }
+ static unsigned getNumberEpoch(const ::clang::CFG *F) { return 0; }
};
-template <> struct GraphTraits<Inverse< ::clang::CFG *>>
- : public GraphTraits<Inverse< ::clang::CFGBlock *>> {
+template <>
+struct GraphTraits<Inverse<::clang::CFG *>>
+ : public GraphTraits<Inverse<clang::CFGBlock *>> {
using nodes_iterator = ::clang::CFG::iterator;
static NodeRef getEntryNode(::clang::CFG *F) { return &F->getExit(); }
static nodes_iterator nodes_begin( ::clang::CFG* F) {return F->nodes_begin();}
static nodes_iterator nodes_end( ::clang::CFG* F) { return F->nodes_end(); }
+
+ static unsigned getMaxNumber(const ::clang::CFG *F) {
+ return F->getNumBlockIDs();
+ }
+ static unsigned getNumberEpoch(const ::clang::CFG *F) { return 0; }
};
-template <> struct GraphTraits<Inverse<const ::clang::CFG *>>
- : public GraphTraits<Inverse<const ::clang::CFGBlock *>> {
+template <>
+struct GraphTraits<Inverse<const ::clang::CFG *>>
+ : public GraphTraits<Inverse<const clang::CFGBlock *>> {
using nodes_iterator = ::clang::CFG::const_iterator;
static NodeRef getEntryNode(const ::clang::CFG *F) { return &F->getExit(); }
@@ -1635,6 +1745,11 @@ template <> struct GraphTraits<Inverse<const ::clang::CFG *>>
static nodes_iterator nodes_end(const ::clang::CFG* F) {
return F->nodes_end();
}
+
+ static unsigned getMaxNumber(const ::clang::CFG *F) {
+ return F->getNumBlockIDs();
+ }
+ static unsigned getNumberEpoch(const ::clang::CFG *F) { return 0; }
};
} // namespace llvm
diff --git a/llvm/include/llvm/Support/CFGDiff.h b/llvm/include/llvm/Support/CFGDiff.h
index 88f4fe52d2019..f81215ed71bd4 100644
--- a/llvm/include/llvm/Support/CFGDiff.h
+++ b/llvm/include/llvm/Support/CFGDiff.h
@@ -136,9 +136,6 @@ template <typename NodePtr, bool InverseGraph = false> class GraphDiff {
auto R = children<DirectedNodeT>(N);
VectRet Res = VectRet(detail::reverse_if<!InverseEdge>(R));
- // Remove nullptr children for clang.
- llvm::erase(Res, nullptr);
-
auto &Children = (InverseEdge != InverseGraph) ? Pred : Succ;
auto It = Children.find(N);
if (It == Children.end())
diff --git a/llvm/include/llvm/Support/GenericDomTreeConstruction.h b/llvm/include/llvm/Support/GenericDomTreeConstruction.h
index 80365e72e8d07..eb2e1e4ec0d74 100644
--- a/llvm/include/llvm/Support/GenericDomTreeConstruction.h
+++ b/llvm/include/llvm/Support/GenericDomTreeConstruction.h
@@ -121,9 +121,6 @@ template <typename DomTreeT> struct SemiNCAInfo {
static SmallVector<NodePtr, 8> getChildren(NodePtr N, BatchUpdatePtr BUI) {
if (BUI)
return BUI->PreViewCFG.template getChildren<Inversed>(N);
- // Force the element type to NodePtr. some graphs (clang's
- // CFGBlock::AdjacentBlock) yield a proxy convertible to NodePtr rather than
- // NodePtr itself.
auto Children = getChildren<Inversed>(N);
return SmallVector<NodePtr, 8>(Children.begin(), Children.end());
}
@@ -133,14 +130,7 @@ template <typename DomTreeT> struct SemiNCAInfo {
template <bool Inversed> static auto getChildren(NodePtr N) {
using DirectedNodeT =
std::conditional_t<Inversed, Inverse<NodePtr>, NodePtr>;
- auto R = detail::reverse_if<!Inversed>(children<DirectedNodeT>(N));
- // Most graphs' iterators yield NodePtr directly; return the range as is.
- // clang's CFGBlock instead yields a CFGBlock::AdjacentBlock proxy that is
- // convertible to NodePtr but can be null for AB_Unreachable.
- if constexpr (std::is_same_v<std::decay_t<decltype(*R.begin())>, NodePtr>)
- return R;
- else
- return llvm::make_filter_range(R, [](NodePtr C) { return C != nullptr; });
+ return detail::reverse_if<!Inversed>(children<DirectedNodeT>(N));
}
InfoRec &getNodeInfo(NodePtr BB) {
>From 8e8e4e50f501bac04b9a00dd7305cf35549b5afc Mon Sep 17 00:00:00 2001
From: "Chi-Chun, Chen" <chichun.chen at hpe.com>
Date: Mon, 6 Jul 2026 11:33:34 -0500
Subject: [PATCH 16/46] [flang][OpenMP][NFC] Move variant-matching context to
semantics (#207087)
Move FlangOMPContext from lowering into semantics as the
MLIR-independent OmpVariantMatchContext. This lets an upcoming semantic
check reuse the matcher to skip the loop-nest checks for a metadirective
variant that cannot be selected on the current target, such as a
`device={kind(nohost)}` variant in a host compilation. No functional
change to metadirective lowering.
Assisted with copilot.
---
flang/include/flang/Semantics/openmp-utils.h | 16 +++++++++
flang/include/flang/Semantics/semantics.h | 12 +++++++
flang/lib/Frontend/CompilerInvocation.cpp | 4 ++-
flang/lib/Lower/OpenMP/OpenMP.cpp | 6 ++--
flang/lib/Lower/OpenMP/Utils.cpp | 34 +++++++-------------
flang/lib/Lower/OpenMP/Utils.h | 21 ++++++------
flang/lib/Semantics/openmp-utils.cpp | 34 ++++++++++++++++++++
7 files changed, 89 insertions(+), 38 deletions(-)
diff --git a/flang/include/flang/Semantics/openmp-utils.h b/flang/include/flang/Semantics/openmp-utils.h
index 75bab5f6cb8ec..81bf960a35dcb 100644
--- a/flang/include/flang/Semantics/openmp-utils.h
+++ b/flang/include/flang/Semantics/openmp-utils.h
@@ -209,6 +209,22 @@ std::optional<DynamicUserCondition> MakeVariantMatchInfo(
const parser::traits::OmpContextSelectorSpecification &ctxSel,
SemanticsContext &semaCtx);
+/// An `llvm::omp::OMPContext` describing the current compilation, used for
+/// OpenMP variant matching. It overrides ISA-trait matching to test against
+/// the target features.
+class OmpVariantMatchContext : public llvm::omp::OMPContext {
+public:
+ OmpVariantMatchContext(bool isDeviceCompilation, llvm::Triple targetTriple,
+ llvm::Triple targetOffloadTriple, std::string targetFeatures,
+ llvm::ArrayRef<llvm::omp::TraitProperty> constructTraits = {});
+ OmpVariantMatchContext(const SemanticsContext &context,
+ llvm::ArrayRef<llvm::omp::TraitProperty> constructTraits = {});
+ bool matchesISATrait(llvm::StringRef rawString) const override;
+
+private:
+ std::string features_;
+};
+
std::vector<SomeExpr> GetTopLevelDesignators(const SomeExpr &expr);
const SomeExpr *HasStorageOverlap(
const SomeExpr &base, llvm::ArrayRef<SomeExpr> exprs);
diff --git a/flang/include/flang/Semantics/semantics.h b/flang/include/flang/Semantics/semantics.h
index d6ba4e81232f8..575851920aa9f 100644
--- a/flang/include/flang/Semantics/semantics.h
+++ b/flang/include/flang/Semantics/semantics.h
@@ -113,6 +113,8 @@ class SemanticsContext {
evaluate::TargetCharacteristics &targetCharacteristics() {
return targetCharacteristics_;
}
+ const std::string &targetTriple() const { return targetTriple_; }
+ const std::string &targetFeatures() const { return targetFeatures_; }
Scope &globalScope() { return globalScope_; }
Scope &intrinsicModulesScope() { return intrinsicModulesScope_; }
Scope *currentHermeticModuleFileScope() {
@@ -176,6 +178,14 @@ class SemanticsContext {
openAccDefaultNoneScalarsStrictDisableOption_ = std::move(x);
return *this;
}
+ SemanticsContext &set_targetTriple(const std::string &x) {
+ targetTriple_ = x;
+ return *this;
+ }
+ SemanticsContext &set_targetFeatures(const std::string &x) {
+ targetFeatures_ = x;
+ return *this;
+ }
const DeclTypeSpec &MakeNumericType(TypeCategory, int kind = 0);
const DeclTypeSpec &MakeLogicalType(int kind = 0);
@@ -383,6 +393,8 @@ class SemanticsContext {
std::vector<std::string> intrinsicModuleDirectories_;
std::string moduleDirectory_{"."s};
std::string moduleFileSuffix_{".mod"};
+ std::string targetTriple_;
+ std::string targetFeatures_;
bool underscoring_{true};
bool warnOnNonstandardUsage_{false};
bool warningsAreErrors_{false};
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 8e13fc9f05324..eb3659443ada2 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -1978,7 +1978,9 @@ CompilerInvocation::getSemanticsCtx(
clang::getDriverOptTable()
.getOptionPrefixedName(
clang::options::OPT_fno_openacc_default_none_scalars_strict)
- .str());
+ .str())
+ .set_targetTriple(targetMachine.getTargetTriple().str())
+ .set_targetFeatures(targetMachine.getTargetFeatureString().str());
std::string compilerVersion = Fortran::common::getFlangFullVersion();
Fortran::tools::setUpTargetCharacteristics(
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index d630d825062b7..bfb63c36991be 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -5098,7 +5098,8 @@ static void genMetadirective(lower::AbstractConverter &converter,
llvm::SmallVector<llvm::omp::TraitProperty, 8> constructTraits;
collectEnclosingConstructTraits(builder.getInsertionBlock()->getParentOp(),
constructTraits);
- FlangOMPContext ompCtx(builder.getModule(), constructTraits);
+ semantics::omp::OmpVariantMatchContext ompCtx =
+ makeVariantMatchContext(builder.getModule(), constructTraits);
llvm::SmallVector<MetadirectiveCandidate, 4> candidates;
// A null directive specification represents either the implicit `nothing`
@@ -5295,7 +5296,8 @@ static void genMetadirective(lower::AbstractConverter &converter,
auto selectBestCandidate =
[](llvm::ArrayRef<unsigned> candidateIndices,
llvm::ArrayRef<MetadirectiveCandidate> candidates,
- const FlangOMPContext &ompCtx) -> std::optional<unsigned> {
+ const semantics::omp::OmpVariantMatchContext &ompCtx)
+ -> std::optional<unsigned> {
if (candidateIndices.empty())
return std::nullopt;
if (candidateIndices.size() == 1)
diff --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp
index 660a7f53f0096..e0d843ef4eca5 100644
--- a/flang/lib/Lower/OpenMP/Utils.cpp
+++ b/flang/lib/Lower/OpenMP/Utils.cpp
@@ -1319,7 +1319,7 @@ mlir::Value genIteratorCoordinate(Fortran::lower::AbstractConverter &converter,
}
// ---------------------------------------------------------------------------
-// FlangOMPContext — shared OMPContext for metadirective variant-matching
+// OpenMP variant-matching context construction from an MLIR module
// ---------------------------------------------------------------------------
static llvm::Triple getOffloadTargetTriple(mlir::ModuleOp module) {
@@ -1333,29 +1333,17 @@ static llvm::Triple getOffloadTargetTriple(mlir::ModuleOp module) {
return llvm::Triple();
}
-bool FlangOMPContext::isDeviceCompilation(mlir::ModuleOp module) {
- return llvm::cast<mlir::omp::OffloadModuleInterface>(module.getOperation())
- .getIsTargetDevice();
-}
-
-FlangOMPContext::FlangOMPContext(
+semantics::omp::OmpVariantMatchContext makeVariantMatchContext(
mlir::ModuleOp module,
- llvm::ArrayRef<llvm::omp::TraitProperty> constructTraits)
- // No specific device is selected during variant matching; use an unknown
- // device number so OMPContext does not inadvertently describe the host
- // device (which would cause target-device selectors to match incorrectly).
- : OMPContext(isDeviceCompilation(module), fir::getTargetTriple(module),
- getOffloadTargetTriple(module),
- /*DeviceNum=*/-1),
- targetFeatures(fir::getTargetFeatures(module)) {
- for (llvm::omp::TraitProperty trait : constructTraits)
- addTrait(trait);
-}
-
-bool FlangOMPContext::matchesISATrait(llvm::StringRef rawString) const {
- if (!targetFeatures || targetFeatures.nullOrEmpty())
- return false;
- return targetFeatures.contains(("+" + rawString).str());
+ llvm::ArrayRef<llvm::omp::TraitProperty> constructTraits) {
+ bool isDeviceCompilation =
+ llvm::cast<mlir::omp::OffloadModuleInterface>(module.getOperation())
+ .getIsTargetDevice();
+ mlir::LLVM::TargetFeaturesAttr features = fir::getTargetFeatures(module);
+ return semantics::omp::OmpVariantMatchContext(
+ isDeviceCompilation, fir::getTargetTriple(module),
+ getOffloadTargetTriple(module),
+ features ? features.getFeaturesString() : std::string{}, constructTraits);
}
void collectEnclosingConstructTraits(
diff --git a/flang/lib/Lower/OpenMP/Utils.h b/flang/lib/Lower/OpenMP/Utils.h
index efe6c963a3778..94f85c43f7033 100644
--- a/flang/lib/Lower/OpenMP/Utils.h
+++ b/flang/lib/Lower/OpenMP/Utils.h
@@ -30,6 +30,9 @@ namespace Fortran {
namespace semantics {
class Symbol;
+namespace omp {
+class OmpVariantMatchContext;
+} // namespace omp
} // namespace semantics
namespace parser {
@@ -264,18 +267,12 @@ void collectEnclosingConstructTraits(
mlir::Operation *op,
llvm::SmallVectorImpl<llvm::omp::TraitProperty> &constructTraits);
-/// `OMPContext` flavour used by Flang's OpenMP variant matching. Adds an
-/// ISA-trait override based on the module's target-features attribute.
-class FlangOMPContext final : public llvm::omp::OMPContext {
-public:
- FlangOMPContext(mlir::ModuleOp module,
- llvm::ArrayRef<llvm::omp::TraitProperty> constructTraits);
- bool matchesISATrait(llvm::StringRef rawString) const override;
-
-private:
- static bool isDeviceCompilation(mlir::ModuleOp module);
- mlir::LLVM::TargetFeaturesAttr targetFeatures;
-};
+/// Build the OpenMP variant-matching context for \p module. The device flag,
+/// host triple, offload triple, and target features are read from the module;
+/// \p constructTraits seeds the enclosing-construct traits.
+semantics::omp::OmpVariantMatchContext makeVariantMatchContext(
+ mlir::ModuleOp module,
+ llvm::ArrayRef<llvm::omp::TraitProperty> constructTraits);
} // namespace omp
} // namespace lower
diff --git a/flang/lib/Semantics/openmp-utils.cpp b/flang/lib/Semantics/openmp-utils.cpp
index 8cf174a0139af..3503eb0e3a004 100644
--- a/flang/lib/Semantics/openmp-utils.cpp
+++ b/flang/lib/Semantics/openmp-utils.cpp
@@ -2419,4 +2419,38 @@ std::optional<DynamicUserCondition> MakeVariantMatchInfo(
}
return dynamicCond;
}
+
+OmpVariantMatchContext::OmpVariantMatchContext(bool isDeviceCompilation,
+ llvm::Triple targetTriple, llvm::Triple targetOffloadTriple,
+ std::string targetFeatures,
+ llvm::ArrayRef<llvm::omp::TraitProperty> constructTraits)
+ // No specific device is selected during variant matching; use an unknown
+ // device number so OMPContext does not inadvertently describe the host
+ // device (which would cause target-device selectors to match incorrectly).
+ : llvm::omp::OMPContext(isDeviceCompilation, std::move(targetTriple),
+ std::move(targetOffloadTriple), /*DeviceNum=*/-1),
+ features_(std::move(targetFeatures)) {
+ for (llvm::omp::TraitProperty trait : constructTraits) {
+ addTrait(trait);
+ }
+}
+
+OmpVariantMatchContext::OmpVariantMatchContext(const SemanticsContext &context,
+ llvm::ArrayRef<llvm::omp::TraitProperty> constructTraits)
+ : OmpVariantMatchContext(context.langOptions().OpenMPIsTargetDevice,
+ llvm::Triple(context.targetTriple()),
+ context.langOptions().OMPTargetTriples.empty()
+ ? llvm::Triple()
+ : context.langOptions().OMPTargetTriples.front(),
+ context.targetFeatures(), constructTraits) {}
+
+bool OmpVariantMatchContext::matchesISATrait(llvm::StringRef rawString) const {
+ // The target feature list is a comma-separated string such as
+ // "+sse,+avx2,-foo"; an ISA trait matches when its "+" form is present.
+ std::string want{("+" + rawString).str()};
+ llvm::SmallVector<llvm::StringRef> tokens;
+ llvm::StringRef(features_).split(
+ tokens, ',', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
+ return llvm::is_contained(tokens, want);
+}
} // namespace Fortran::semantics::omp
>From 902273247db1f8b62613f38fa1a6f93700de517b Mon Sep 17 00:00:00 2001
From: Alexey Karyakin <akaryaki at qti.qualcomm.com>
Date: Mon, 6 Jul 2026 11:33:58 -0500
Subject: [PATCH 17/46] [clang][tablegen] Allow qualifiers on _Vector builtin
types (#207091)
The builtin prototype parser accepts pointers to _Vector<> template
types but not const/volatile/restrict pointers. Currently, _Vector<>
types are parsed in a special way: first parsing the type itself,
consuming it, and saving the resulting type string. Then, the remaining
qualifiers are parsed while the type is removed from the input. For this
to work, parsing qualifiers must accept empty base type, which was the
case only for plain pointers and references.
This change extends this approach to const/volatile/restrict qualifiers.
Signed-off-by: Alexey Karyakin <akaryaki at qti.qualcomm.com>
---
.../target-builtins-prototype-parser.td | 24 +++++++++++++++++++
clang/utils/TableGen/ClangBuiltinsEmitter.cpp | 9 ++++---
2 files changed, 30 insertions(+), 3 deletions(-)
diff --git a/clang/test/TableGen/target-builtins-prototype-parser.td b/clang/test/TableGen/target-builtins-prototype-parser.td
index 0fa1246a01008..09189333f584d 100644
--- a/clang/test/TableGen/target-builtins-prototype-parser.td
+++ b/clang/test/TableGen/target-builtins-prototype-parser.td
@@ -87,6 +87,30 @@ def : Builtin {
let Spellings = ["__builtin_13"];
}
+def : Builtin {
+// CHECK: Builtin::Info{{.*}} __builtin_14 {{.*}} /* vV2iC* */
+ let Prototype = "void(_Vector<2, int> const *)";
+ let Spellings = ["__builtin_14"];
+}
+
+def : Builtin {
+// CHECK: Builtin::Info{{.*}} __builtin_15 {{.*}} /* vV2iD* */
+ let Prototype = "void(_Vector<2, int> volatile *)";
+ let Spellings = ["__builtin_15"];
+}
+
+def : Builtin {
+// CHECK: Builtin::Info{{.*}} __builtin_16 {{.*}} /* vV2iR* */
+ let Prototype = "void(_Vector<2, int> restrict *)";
+ let Spellings = ["__builtin_16"];
+}
+
+def : Builtin {
+// CHECK: Builtin::Info{{.*}} __builtin_17 {{.*}} /* V2iC*i */
+ let Prototype = "_Vector<2, int> const *(int)";
+ let Spellings = ["__builtin_17"];
+}
+
#ifdef ERROR_EXPECTED_LANES
def : Builtin {
// ERROR_EXPECTED_LANES: :[[# @LINE + 1]]:7: error: Expected number of lanes after '_ExtVector<'
diff --git a/clang/utils/TableGen/ClangBuiltinsEmitter.cpp b/clang/utils/TableGen/ClangBuiltinsEmitter.cpp
index 4f7f437bcf73c..22c81522f9e41 100644
--- a/clang/utils/TableGen/ClangBuiltinsEmitter.cpp
+++ b/clang/utils/TableGen/ClangBuiltinsEmitter.cpp
@@ -274,13 +274,16 @@ class PrototypeParser {
if (AS)
Type += std::to_string(*AS);
} else if (T.consume_back("const")) {
- ParseType(T);
+ if (!T.empty())
+ ParseType(T);
Type += "C";
} else if (T.consume_back("volatile")) {
- ParseType(T);
+ if (!T.empty())
+ ParseType(T);
Type += "D";
} else if (T.consume_back("restrict")) {
- ParseType(T);
+ if (!T.empty())
+ ParseType(T);
Type += "R";
} else if (T.consume_back("&")) {
// References may have an address space qualifier immediately before them.
>From a913e893c3f3e820f413ccd3877f19eb4ab028f6 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Mon, 6 Jul 2026 12:38:25 -0400
Subject: [PATCH 18/46] [SLP]Support Mul as the main copyable instruction
Added support for the missed Mul as a main copyable instruction opcode
Reviewers: RKSimon, hiraditya, bababuck
Pull Request: https://github.com/llvm/llvm-project/pull/207622
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 13 +-
.../AArch64/externally-used-copyables.ll | 188 +++++++++---------
.../buildvector-postpone-for-dependency.ll | 2 +-
.../X86/copyable-operands-reordering.ll | 21 +-
.../X86/gathered-node-with-in-order-parent.ll | 13 +-
.../X86/non-schedulable-before-main.ll | 6 +-
.../X86/reduction-with-removed-extracts.ll | 12 +-
.../X86/vec_list_bias-inseltpoison.ll | 12 +-
.../SLPVectorizer/X86/vec_list_bias.ll | 12 +-
.../vec_list_bias_external_insert_shuffled.ll | 12 +-
.../X86/vect_copyable_in_binops.ll | 15 +-
11 files changed, 143 insertions(+), 163 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 92befc3c2ff9b..ea97a18403dc8 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -11893,12 +11893,12 @@ class InstructionsCompatibilityAnalysis {
/// elements.
static bool isSupportedOpcode(const unsigned Opcode) {
return Opcode == Instruction::Add || Opcode == Instruction::Sub ||
- Opcode == Instruction::LShr || Opcode == Instruction::Shl ||
- Opcode == Instruction::SDiv || Opcode == Instruction::UDiv ||
- Opcode == Instruction::And || Opcode == Instruction::Or ||
- Opcode == Instruction::Xor || Opcode == Instruction::FAdd ||
- Opcode == Instruction::FSub || Opcode == Instruction::FMul ||
- Opcode == Instruction::FDiv;
+ Opcode == Instruction::Mul || Opcode == Instruction::LShr ||
+ Opcode == Instruction::Shl || Opcode == Instruction::SDiv ||
+ Opcode == Instruction::UDiv || Opcode == Instruction::And ||
+ Opcode == Instruction::Or || Opcode == Instruction::Xor ||
+ Opcode == Instruction::FAdd || Opcode == Instruction::FSub ||
+ Opcode == Instruction::FMul || Opcode == Instruction::FDiv;
}
/// Identifies the best candidate value, which represents main opcode
@@ -12520,6 +12520,7 @@ class InstructionsCompatibilityAnalysis {
switch (MainOpcode) {
case Instruction::Add:
case Instruction::Sub:
+ case Instruction::Mul:
case Instruction::LShr:
case Instruction::Shl:
case Instruction::SDiv:
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/externally-used-copyables.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/externally-used-copyables.ll
index b32d606b7d46a..2c9eea2571f07 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/externally-used-copyables.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/externally-used-copyables.ll
@@ -5,94 +5,84 @@ define void @test(i64 %0, i64 %1, i64 %2, i64 %3, i64 %.sroa.3341.0.copyload, i6
; CHECK-LABEL: define void @test(
; CHECK-SAME: i64 [[TMP0:%.*]], i64 [[TMP1:%.*]], i64 [[TMP2:%.*]], i64 [[TMP3:%.*]], i64 [[DOTSROA_3341_0_COPYLOAD:%.*]], i64 [[DOTSROA_3308_0_COPYLOAD:%.*]], i64 [[DOTNEG1:%.*]], i64 [[INDVAR3788:%.*]], i64 [[TMP4:%.*]], i64 [[TMP5:%.*]], i64 [[TMP6:%.*]], i64 [[TMP7:%.*]], i64 [[TMP8:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[_LR_PH_PREHEADER:.*:]]
-; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP0]], 40
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x i64> poison, i64 [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP13:%.*]] = mul <2 x i64> [[TMP10]], <i64 1, i64 24>
+; CHECK-NEXT: [[TMP17:%.*]] = mul <2 x i64> [[TMP10]], <i64 1, i64 40>
; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP0]], 48
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <2 x i64> <i64 poison, i64 1>, i64 [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <2 x i64> [[TMP14]], <2 x i64> <i64 -1, i64 poison>, <2 x i32> <i32 2, i32 0>
+; CHECK-NEXT: [[TMP16:%.*]] = sub <2 x i64> [[TMP14]], [[TMP15]]
+; CHECK-NEXT: [[TMP20:%.*]] = shl i64 [[TMP0]], 11
; CHECK-NEXT: [[TMP35:%.*]] = insertelement <2 x i64> poison, i64 [[TMP0]], i32 0
; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <2 x i64> [[TMP35]], <2 x i64> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP13:%.*]] = add <2 x i64> [[TMP12]], <i64 0, i64 1>
-; CHECK-NEXT: [[TMP14:%.*]] = sub i64 1, [[TMP0]]
-; CHECK-NEXT: [[TMP20:%.*]] = shl i64 [[TMP0]], 11
+; CHECK-NEXT: [[TMP25:%.*]] = shl <2 x i64> [[TMP12]], <i64 11, i64 0>
; CHECK-NEXT: [[TMP21:%.*]] = sub i64 1, [[TMP20]]
-; CHECK-NEXT: [[TMP17:%.*]] = add i64 [[TMP20]], 8
+; CHECK-NEXT: [[TMP22:%.*]] = add <2 x i64> [[TMP25]], <i64 8, i64 1>
+; CHECK-NEXT: [[TMP23:%.*]] = or <2 x i64> [[TMP25]], <i64 8, i64 1>
+; CHECK-NEXT: [[TMP33:%.*]] = shufflevector <2 x i64> [[TMP22]], <2 x i64> [[TMP23]], <2 x i32> <i32 0, i32 3>
; CHECK-NEXT: [[TMP18:%.*]] = shl i64 [[TMP0]], 1
; CHECK-NEXT: [[TMP19:%.*]] = or i64 [[TMP18]], [[TMP0]]
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <2 x i64> [[TMP35]], i64 [[TMP19]], i32 1
-; CHECK-NEXT: [[TMP16:%.*]] = or <2 x i64> [[TMP15]], splat (i64 1)
-; CHECK-NEXT: [[TMP23:%.*]] = shufflevector <2 x i64> [[TMP12]], <2 x i64> <i64 24, i64 poison>, <2 x i32> <i32 2, i32 0>
-; CHECK-NEXT: [[TMP41:%.*]] = mul <2 x i64> [[TMP12]], [[TMP23]]
-; CHECK-NEXT: [[TMP44:%.*]] = insertelement <32 x i64> poison, i64 [[TMP1]], i32 5
-; CHECK-NEXT: [[TMP25:%.*]] = insertelement <16 x i64> poison, i64 [[TMP1]], i32 2
-; CHECK-NEXT: [[TMP31:%.*]] = insertelement <16 x i64> [[TMP25]], i64 [[TMP10]], i32 3
-; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <2 x i64> [[TMP12]], <2 x i64> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP34:%.*]] = or i64 [[TMP19]], 1
+; CHECK-NEXT: [[TMP55:%.*]] = mul i64 [[TMP0]], [[TMP0]]
+; CHECK-NEXT: [[TMP58:%.*]] = insertelement <8 x i64> poison, i64 [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP30:%.*]] = shufflevector <8 x i64> [[TMP58]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP31:%.*]] = shufflevector <8 x i64> [[TMP30]], <8 x i64> <i64 poison, i64 1, i64 1, i64 1, i64 poison, i64 poison, i64 poison, i64 poison>, <6 x i32> <i32 0, i32 9, i32 10, i32 11, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP32:%.*]] = shufflevector <8 x i64> [[TMP30]], <8 x i64> <i64 0, i64 0, i64 0, i64 0, i64 poison, i64 1, i64 1, i64 1>, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 0, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <8 x i64> [[TMP30]], <8 x i64> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP28:%.*]] = insertelement <3 x i64> [[TMP27]], i64 [[TMP1]], i32 1
; CHECK-NEXT: [[TMP29:%.*]] = insertelement <3 x i64> [[TMP28]], i64 [[DOTNEG1]], i32 2
-; CHECK-NEXT: [[TMP30:%.*]] = shufflevector <3 x i64> [[TMP29]], <3 x i64> poison, <8 x i32> <i32 0, i32 0, i32 1, i32 1, i32 1, i32 2, i32 0, i32 0>
-; CHECK-NEXT: [[TMP85:%.*]] = shufflevector <2 x i64> [[TMP15]], <2 x i64> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP32:%.*]] = insertelement <4 x i64> poison, i64 [[TMP1]], i32 1
-; CHECK-NEXT: [[TMP33:%.*]] = insertelement <2 x i64> [[TMP15]], i64 [[TMP2]], i32 1
-; CHECK-NEXT: [[TMP34:%.*]] = shufflevector <2 x i64> [[TMP33]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 0>
-; CHECK-NEXT: [[TMP40:%.*]] = insertelement <2 x i64> poison, i64 [[TMP21]], i32 0
-; CHECK-NEXT: [[TMP43:%.*]] = insertelement <2 x i64> [[TMP40]], i64 [[TMP17]], i32 1
-; CHECK-NEXT: [[TMP37:%.*]] = insertelement <2 x i64> <i64 poison, i64 1>, i64 [[TMP0]], i32 0
-; CHECK-NEXT: [[TMP38:%.*]] = shufflevector <2 x i64> [[TMP41]], <2 x i64> poison, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP74:%.*]] = shufflevector <32 x i64> [[TMP44]], <32 x i64> [[TMP38]], <32 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 32, i32 33, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP36:%.*]] = shufflevector <3 x i64> [[TMP29]], <3 x i64> poison, <8 x i32> <i32 0, i32 1, i32 1, i32 1, i32 2, i32 0, i32 0, i32 0>
+; CHECK-NEXT: [[TMP37:%.*]] = insertelement <4 x i64> poison, i64 [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP38:%.*]] = insertelement <4 x i64> [[TMP37]], i64 [[TMP8]], i32 1
+; CHECK-NEXT: [[TMP39:%.*]] = insertelement <4 x i64> [[TMP38]], i64 [[DOTSROA_3308_0_COPYLOAD]], i32 2
+; CHECK-NEXT: [[TMP40:%.*]] = insertelement <4 x i64> [[TMP39]], i64 [[TMP0]], i32 3
+; CHECK-NEXT: [[TMP41:%.*]] = shufflevector <2 x i64> [[TMP17]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP74:%.*]] = insertelement <4 x i64> poison, i64 [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP43:%.*]] = mul i64 [[TMP0]], [[TMP0]]
+; CHECK-NEXT: [[TMP44:%.*]] = shufflevector <2 x i64> [[TMP13]], <2 x i64> poison, <32 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP45:%.*]] = shufflevector <32 x i64> [[TMP44]], <32 x i64> poison, <32 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP46:%.*]] = shufflevector <2 x i64> [[TMP33]], <2 x i64> poison, <6 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP47:%.*]] = shufflevector <6 x i64> [[TMP31]], <6 x i64> [[TMP46]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 7>
+; CHECK-NEXT: [[TMP48:%.*]] = shufflevector <2 x i64> [[TMP16]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; CHECK-NEXT: br label %[[DOTLR_PH1977_US:.*]]
; CHECK: [[_LR_PH1977_US:.*:]]
; CHECK-NEXT: [[INDVAR37888:%.*]] = phi i64 [ 0, [[DOTLR_PH_PREHEADER:%.*]] ], [ 1, %[[DOTLR_PH1977_US]] ]
-; CHECK-NEXT: [[TMP36:%.*]] = mul <2 x i64> [[TMP16]], [[TMP85]]
-; CHECK-NEXT: [[TMP73:%.*]] = mul <2 x i64> [[TMP43]], [[TMP12]]
+; CHECK-NEXT: [[TMP49:%.*]] = shufflevector <6 x i64> [[TMP47]], <6 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 5, i32 5>
+; CHECK-NEXT: [[TMP50:%.*]] = mul <8 x i64> [[TMP30]], [[TMP49]]
+; CHECK-NEXT: [[TMP51:%.*]] = or <8 x i64> [[TMP30]], [[TMP49]]
+; CHECK-NEXT: [[TMP52:%.*]] = shufflevector <8 x i64> [[TMP50]], <8 x i64> [[TMP51]], <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP53:%.*]] = mul i64 [[TMP34]], [[TMP0]]
+; CHECK-NEXT: [[TMP54:%.*]] = mul i64 [[TMP21]], [[TMP0]]
; CHECK-NEXT: [[TMP26:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[DIFF_CHECK3783:%.*]] = icmp ult i64 [[TMP11]], [[TMP26]]
-; CHECK-NEXT: [[TMP39:%.*]] = mul i64 [[TMP0]], [[TMP0]]
-; CHECK-NEXT: [[TMP90:%.*]] = or <2 x i64> [[TMP12]], splat (i64 1)
+; CHECK-NEXT: [[DIFF_CHECK3790:%.*]] = icmp ult i64 [[INDVAR37888]], [[TMP0]]
; CHECK-NEXT: [[DIFF_CHECK3805:%.*]] = icmp ugt i64 [[TMP26]], 1
-; CHECK-NEXT: [[TMP45:%.*]] = shufflevector <2 x i64> [[TMP36]], <2 x i64> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP46:%.*]] = add <2 x i64> [[TMP45]], splat (i64 1)
-; CHECK-NEXT: [[TMP47:%.*]] = insertelement <8 x i64> poison, i64 [[INDVAR37888]], i32 0
-; CHECK-NEXT: [[TMP91:%.*]] = insertelement <8 x i64> [[TMP47]], i64 [[TMP39]], i32 1
-; CHECK-NEXT: [[TMP92:%.*]] = shufflevector <2 x i64> [[TMP46]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP93:%.*]] = shufflevector <2 x i64> [[TMP90]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP24:%.*]] = extractelement <2 x i64> [[TMP36]], i32 0
+; CHECK-NEXT: [[TMP56:%.*]] = add <8 x i64> [[TMP52]], [[TMP32]]
+; CHECK-NEXT: [[TMP57:%.*]] = icmp ult <8 x i64> [[TMP56]], [[TMP36]]
+; CHECK-NEXT: [[TMP24:%.*]] = extractelement <8 x i64> [[TMP52]], i32 5
; CHECK-NEXT: [[TMP42:%.*]] = add i64 [[TMP24]], 1
; CHECK-NEXT: [[DIFF_CHECK3842:%.*]] = icmp ult i64 [[TMP42]], [[TMP0]]
-; CHECK-NEXT: [[TMP94:%.*]] = add <2 x i64> [[TMP36]], splat (i64 1)
-; CHECK-NEXT: [[TMP95:%.*]] = shufflevector <2 x i64> [[TMP36]], <2 x i64> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP55:%.*]] = shufflevector <2 x i64> [[TMP36]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP56:%.*]] = shufflevector <4 x i64> [[TMP32]], <4 x i64> [[TMP55]], <4 x i32> <i32 poison, i32 1, i32 poison, i32 4>
-; CHECK-NEXT: [[TMP57:%.*]] = shufflevector <2 x i64> [[TMP94]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP96:%.*]] = shufflevector <4 x i64> [[TMP57]], <4 x i64> [[TMP56]], <4 x i32> <i32 0, i32 5, i32 1, i32 7>
-; CHECK-NEXT: [[TMP97:%.*]] = icmp ult <4 x i64> [[TMP96]], [[TMP34]]
+; CHECK-NEXT: [[DIFF_CHECK3817:%.*]] = icmp ult i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP76:%.*]] = add i64 [[TMP53]], 1
+; CHECK-NEXT: [[DIFF_CHECK3820:%.*]] = icmp ult i64 [[TMP76]], [[TMP0]]
; CHECK-NEXT: [[TMP98:%.*]] = shl <2 x i64> [[TMP12]], splat (i64 1)
-; CHECK-NEXT: [[TMP48:%.*]] = mul <2 x i64> [[TMP12]], [[TMP13]]
-; CHECK-NEXT: [[TMP99:%.*]] = shufflevector <2 x i64> [[TMP73]], <2 x i64> poison, <2 x i32> <i32 poison, i32 0>
-; CHECK-NEXT: [[TMP100:%.*]] = insertelement <2 x i64> [[TMP99]], i64 [[TMP14]], i32 0
-; CHECK-NEXT: [[TMP101:%.*]] = mul <2 x i64> [[TMP100]], [[TMP37]]
-; CHECK-NEXT: [[TMP102:%.*]] = add <2 x i64> [[TMP100]], [[TMP37]]
-; CHECK-NEXT: [[TMP49:%.*]] = shufflevector <2 x i64> [[TMP101]], <2 x i64> [[TMP102]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP50:%.*]] = insertelement <32 x i64> [[TMP74]], i64 [[INDVAR37888]], i32 0
-; CHECK-NEXT: [[TMP51:%.*]] = shufflevector <2 x i64> [[TMP48]], <2 x i64> poison, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP52:%.*]] = shufflevector <32 x i64> [[TMP50]], <32 x i64> [[TMP51]], <32 x i32> <i32 0, i32 32, i32 33, i32 poison, i32 poison, i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 10, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP53:%.*]] = shufflevector <2 x i64> [[TMP49]], <2 x i64> poison, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP54:%.*]] = shufflevector <32 x i64> [[TMP52]], <32 x i64> [[TMP53]], <32 x i32> <i32 0, i32 1, i32 2, i32 32, i32 33, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT: [[TMP103:%.*]] = shufflevector <2 x i64> [[TMP36]], <2 x i64> poison, <32 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP58:%.*]] = shufflevector <2 x i64> [[TMP36]], <2 x i64> poison, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP77:%.*]] = insertelement <4 x i64> [[TMP74]], i64 [[INDVAR37888]], i32 0
+; CHECK-NEXT: [[TMP78:%.*]] = shufflevector <4 x i64> [[TMP77]], <4 x i64> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 1>
+; CHECK-NEXT: [[TMP80:%.*]] = shufflevector <4 x i64> [[TMP77]], <4 x i64> <i64 1, i64 poison, i64 poison, i64 poison>, <4 x i32> <i32 4, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP81:%.*]] = shufflevector <4 x i64> [[TMP80]], <4 x i64> [[TMP48]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT: [[TMP82:%.*]] = mul <4 x i64> [[TMP78]], [[TMP81]]
+; CHECK-NEXT: [[TMP84:%.*]] = add i64 [[TMP54]], 1
+; CHECK-NEXT: [[TMP85:%.*]] = insertelement <32 x i64> [[TMP45]], i64 [[TMP55]], i32 11
+; CHECK-NEXT: [[TMP86:%.*]] = shufflevector <4 x i64> [[TMP82]], <4 x i64> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP89:%.*]] = shufflevector <32 x i64> [[TMP85]], <32 x i64> [[TMP86]], <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 poison, i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 10, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP90:%.*]] = insertelement <32 x i64> [[TMP89]], i64 [[TMP84]], i32 4
+; CHECK-NEXT: [[TMP100:%.*]] = shufflevector <8 x i64> [[TMP52]], <8 x i64> poison, <32 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 4, i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP73:%.*]] = shufflevector <8 x i64> [[TMP52]], <8 x i64> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP87:%.*]] = shufflevector <32 x i64> [[TMP90]], <32 x i64> [[TMP73]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 36, i32 37, i32 poison, i32 poison, i32 10, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP125:%.*]] = shufflevector <2 x i64> [[TMP98]], <2 x i64> poison, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP131:%.*]] = shufflevector <2 x i64> [[TMP73]], <2 x i64> <i64 1, i64 poison>, <2 x i32> <i32 2, i32 1>
-; CHECK-NEXT: [[TMP76:%.*]] = or <2 x i64> [[TMP12]], [[TMP131]]
-; CHECK-NEXT: [[TMP77:%.*]] = add <2 x i64> [[TMP12]], [[TMP131]]
-; CHECK-NEXT: [[TMP78:%.*]] = shufflevector <2 x i64> [[TMP76]], <2 x i64> [[TMP77]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP132:%.*]] = shufflevector <2 x i64> [[TMP78]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP80:%.*]] = shufflevector <8 x i64> [[TMP91]], <8 x i64> [[TMP132]], <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 8, i32 9, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP81:%.*]] = shufflevector <8 x i64> [[TMP80]], <8 x i64> [[TMP92]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
-; CHECK-NEXT: [[TMP82:%.*]] = shufflevector <8 x i64> [[TMP81]], <8 x i64> [[TMP93]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP83:%.*]] = icmp ult <8 x i64> [[TMP82]], [[TMP30]]
-; CHECK-NEXT: [[TMP84:%.*]] = shufflevector <2 x i64> [[TMP73]], <2 x i64> poison, <32 x i32> <i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP133:%.*]] = shufflevector <2 x i64> [[TMP73]], <2 x i64> poison, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP86:%.*]] = shufflevector <32 x i64> [[TMP54]], <32 x i64> [[TMP133]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 33, i32 poison, i32 poison, i32 poison, i32 10, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP87:%.*]] = shufflevector <32 x i64> [[TMP86]], <32 x i64> [[TMP58]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 32, i32 poison, i32 poison, i32 10, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP88:%.*]] = shufflevector <32 x i64> [[TMP87]], <32 x i64> [[TMP125]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 33, i32 10, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP89:%.*]] = shufflevector <2 x i64> [[TMP73]], <2 x i64> poison, <32 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP59:%.*]] = shufflevector <32 x i64> [[TMP88]], <32 x i64> [[TMP133]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 32, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP59:%.*]] = insertelement <32 x i64> [[TMP88]], i64 [[TMP54]], i32 12
; CHECK-NEXT: [[TMP60:%.*]] = shufflevector <32 x i64> [[TMP59]], <32 x i64> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 5, i32 5, i32 7, i32 5, i32 8, i32 9, i32 10, i32 5, i32 0, i32 1, i32 11, i32 12, i32 5, i32 5, i32 6, i32 7, i32 5, i32 5, i32 7, i32 5, i32 9, i32 10, i32 0, i32 1>
; CHECK-NEXT: [[TMP61:%.*]] = shufflevector <32 x i64> [[TMP59]], <32 x i64> poison, <10 x i32> <i32 poison, i32 poison, i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP62:%.*]] = insertelement <10 x i64> [[TMP61]], i64 [[TMP0]], i32 0
@@ -106,41 +96,49 @@ define void @test(i64 %0, i64 %1, i64 %2, i64 %3, i64 %.sroa.3341.0.copyload, i6
; CHECK-NEXT: [[TMP70:%.*]] = insertelement <10 x i64> [[TMP69]], i64 [[DOTSROA_3341_0_COPYLOAD]], i32 9
; CHECK-NEXT: [[TMP71:%.*]] = shufflevector <10 x i64> [[TMP70]], <10 x i64> poison, <32 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 0, i32 2, i32 3, i32 4, i32 0, i32 0, i32 2, i32 2, i32 0, i32 5, i32 0, i32 0, i32 0, i32 0, i32 6, i32 7, i32 0, i32 2, i32 8, i32 9, i32 0, i32 0, i32 2, i32 0, i32 0, i32 0>
; CHECK-NEXT: [[TMP72:%.*]] = icmp ult <32 x i64> [[TMP60]], [[TMP71]]
-; CHECK-NEXT: [[TMP104:%.*]] = add <2 x i64> [[TMP73]], <i64 1, i64 0>
-; CHECK-NEXT: [[TMP105:%.*]] = shufflevector <2 x i64> [[TMP104]], <2 x i64> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP106:%.*]] = shufflevector <16 x i64> [[TMP31]], <16 x i64> [[TMP105]], <16 x i32> <i32 poison, i32 poison, i32 2, i32 3, i32 16, i32 17, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP107:%.*]] = shufflevector <2 x i64> [[TMP36]], <2 x i64> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP108:%.*]] = shufflevector <16 x i64> [[TMP106]], <16 x i64> [[TMP107]], <16 x i32> <i32 poison, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 17, i32 16, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP109:%.*]] = insertelement <16 x i64> [[TMP108]], i64 [[INDVAR37888]], i32 8
-; CHECK-NEXT: [[TMP110:%.*]] = shufflevector <2 x i64> [[TMP48]], <2 x i64> poison, <16 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP111:%.*]] = shufflevector <2 x i64> [[TMP48]], <2 x i64> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP112:%.*]] = shufflevector <16 x i64> [[TMP109]], <16 x i64> [[TMP111]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 16, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP113:%.*]] = shufflevector <16 x i64> [[TMP112]], <16 x i64> poison, <10 x i32> <i32 poison, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
-; CHECK-NEXT: [[TMP114:%.*]] = shufflevector <2 x i64> [[TMP98]], <2 x i64> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP115:%.*]] = shufflevector <2 x i64> [[TMP98]], <2 x i64> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP116:%.*]] = shufflevector <16 x i64> [[TMP115]], <16 x i64> [[TMP112]], <10 x i32> <i32 0, i32 1, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25>
-; CHECK-NEXT: [[TMP117:%.*]] = shufflevector <10 x i64> [[TMP116]], <10 x i64> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 5, i32 5, i32 2, i32 2, i32 6, i32 7, i32 0, i32 2, i32 8, i32 9>
-; CHECK-NEXT: [[TMP118:%.*]] = shufflevector <10 x i64> [[TMP116]], <10 x i64> poison, <5 x i32> <i32 2, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP119:%.*]] = insertelement <5 x i64> [[TMP118]], i64 [[TMP0]], i32 1
-; CHECK-NEXT: [[TMP120:%.*]] = insertelement <5 x i64> [[TMP119]], i64 [[TMP2]], i32 2
-; CHECK-NEXT: [[TMP121:%.*]] = insertelement <5 x i64> [[TMP120]], i64 [[TMP8]], i32 3
-; CHECK-NEXT: [[TMP122:%.*]] = insertelement <5 x i64> [[TMP121]], i64 [[DOTSROA_3308_0_COPYLOAD]], i32 4
-; CHECK-NEXT: [[TMP123:%.*]] = shufflevector <5 x i64> [[TMP122]], <5 x i64> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 0, i32 2, i32 3, i32 4, i32 1, i32 1, i32 0, i32 1, i32 1, i32 1>
-; CHECK-NEXT: [[TMP124:%.*]] = icmp ult <16 x i64> [[TMP117]], [[TMP123]]
+; CHECK-NEXT: [[TMP91:%.*]] = add i64 [[TMP54]], 1
+; CHECK-NEXT: [[TMP92:%.*]] = shufflevector <8 x i64> [[TMP52]], <8 x i64> poison, <8 x i32> <i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 4, i32 4>
+; CHECK-NEXT: [[TMP93:%.*]] = shufflevector <8 x i64> [[TMP92]], <8 x i64> [[TMP41]], <8 x i32> <i32 0, i32 1, i32 2, i32 8, i32 9, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP94:%.*]] = shufflevector <2 x i64> [[TMP98]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP95:%.*]] = shufflevector <8 x i64> [[TMP93]], <8 x i64> [[TMP94]], <8 x i32> <i32 0, i32 8, i32 9, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP96:%.*]] = insertelement <8 x i64> [[TMP95]], i64 [[TMP91]], i32 5
+; CHECK-NEXT: [[TMP105:%.*]] = shufflevector <8 x i64> [[TMP96]], <8 x i64> poison, <2 x i32> <i32 poison, i32 3>
+; CHECK-NEXT: [[TMP107:%.*]] = insertelement <2 x i64> [[TMP105]], i64 [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP99:%.*]] = shufflevector <2 x i64> [[TMP107]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 1, i32 0, i32 0, i32 0, i32 0, i32 1>
+; CHECK-NEXT: [[TMP83:%.*]] = icmp ult <8 x i64> [[TMP96]], [[TMP99]]
+; CHECK-NEXT: [[TMP101:%.*]] = shufflevector <8 x i64> [[TMP52]], <8 x i64> poison, <4 x i32> <i32 4, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP102:%.*]] = insertelement <4 x i64> [[TMP101]], i64 [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP103:%.*]] = insertelement <4 x i64> [[TMP102]], i64 [[TMP53]], i32 3
+; CHECK-NEXT: [[TMP104:%.*]] = shufflevector <4 x i64> [[TMP103]], <4 x i64> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 3>
+; CHECK-NEXT: [[TMP97:%.*]] = icmp ult <4 x i64> [[TMP104]], [[TMP40]]
+; CHECK-NEXT: [[DIFF_CHECK3930:%.*]] = icmp ult i64 [[TMP24]], [[TMP0]]
+; CHECK-NEXT: [[TMP106:%.*]] = extractelement <2 x i64> [[TMP98]], i32 0
+; CHECK-NEXT: [[DIFF_CHECK3932:%.*]] = icmp ult i64 [[TMP106]], [[TMP1]]
+; CHECK-NEXT: [[OP_RDX50:%.*]] = icmp ult i64 [[TMP1]], [[TMP0]]
+; CHECK-NEXT: [[TMP126:%.*]] = icmp ult i64 [[INDVAR37888]], [[TMP0]]
+; CHECK-NEXT: [[DIFF_CHECK3938:%.*]] = icmp ult i64 [[TMP43]], [[TMP0]]
; CHECK-NEXT: [[DIFF_CHECK3950:%.*]] = icmp ult i64 [[TMP1]], [[TMP2]]
; CHECK-NEXT: [[TMP75:%.*]] = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> [[TMP72]])
-; CHECK-NEXT: [[TMP126:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP124]])
+; CHECK-NEXT: [[TMP108:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP57]])
; CHECK-NEXT: [[TMP127:%.*]] = shufflevector <8 x i1> [[TMP83]], <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[RDX_OP:%.*]] = or <4 x i1> [[TMP127]], [[TMP97]]
; CHECK-NEXT: [[TMP128:%.*]] = shufflevector <4 x i1> [[RDX_OP]], <4 x i1> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP129:%.*]] = shufflevector <8 x i1> [[TMP83]], <8 x i1> [[TMP128]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: [[TMP130:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP129]])
; CHECK-NEXT: [[OP_RDX26:%.*]] = or i1 [[TMP130]], [[DIFF_CHECK3783]]
-; CHECK-NEXT: [[OP_RDX27:%.*]] = or i1 [[DIFF_CHECK3842]], [[DIFF_CHECK3950]]
-; CHECK-NEXT: [[OP_RDX50:%.*]] = or i1 [[DIFF_CHECK3805]], [[TMP75]]
-; CHECK-NEXT: [[OP_RDX29:%.*]] = or i1 [[OP_RDX26]], [[OP_RDX27]]
+; CHECK-NEXT: [[OP_RDX57:%.*]] = or i1 [[DIFF_CHECK3790]], [[DIFF_CHECK3842]]
+; CHECK-NEXT: [[OP_RDX58:%.*]] = or i1 [[DIFF_CHECK3817]], [[DIFF_CHECK3820]]
+; CHECK-NEXT: [[OP_RDX59:%.*]] = or i1 [[DIFF_CHECK3930]], [[DIFF_CHECK3932]]
; CHECK-NEXT: [[OP_RDX52:%.*]] = or i1 [[OP_RDX50]], [[TMP126]]
-; CHECK-NEXT: [[TMP79:%.*]] = or i1 [[OP_RDX29]], [[OP_RDX52]]
+; CHECK-NEXT: [[OP_RDX61:%.*]] = or i1 [[DIFF_CHECK3938]], [[DIFF_CHECK3950]]
+; CHECK-NEXT: [[OP_RDX62:%.*]] = or i1 [[DIFF_CHECK3805]], [[TMP75]]
+; CHECK-NEXT: [[OP_RDX63:%.*]] = or i1 [[OP_RDX26]], [[OP_RDX57]]
+; CHECK-NEXT: [[OP_RDX64:%.*]] = or i1 [[OP_RDX58]], [[OP_RDX59]]
+; CHECK-NEXT: [[OP_RDX65:%.*]] = or i1 [[OP_RDX52]], [[OP_RDX61]]
+; CHECK-NEXT: [[OP_RDX66:%.*]] = or i1 [[OP_RDX62]], [[TMP108]]
+; CHECK-NEXT: [[OP_RDX67:%.*]] = or i1 [[OP_RDX63]], [[OP_RDX64]]
+; CHECK-NEXT: [[OP_RDX68:%.*]] = or i1 [[OP_RDX65]], [[OP_RDX66]]
+; CHECK-NEXT: [[TMP79:%.*]] = or i1 [[OP_RDX67]], [[OP_RDX68]]
; CHECK-NEXT: br i1 [[TMP79]], label %[[SCALAR_PH4023:.*]], label %[[DOTLR_PH1977_US]]
; CHECK: [[SCALAR_PH4023]]:
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-postpone-for-dependency.ll b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-postpone-for-dependency.ll
index aa424b9031e77..3632c75a1afdd 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-postpone-for-dependency.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-postpone-for-dependency.ll
@@ -14,7 +14,7 @@ define void @test() {
; CHECK-NEXT: [[TMP1:%.*]] = phi <2 x i32> [ zeroinitializer, %[[BB]] ], [ [[TMP8:%.*]], %[[BB6]] ]
; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> <i32 1, i32 0, i32 poison, i32 poison>, <4 x i32> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 5, i32 4>
-; CHECK-NEXT: [[TMP4]] = mul <4 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT: [[TMP4]] = mul <4 x i32> zeroinitializer, [[TMP3]]
; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> <i32 0, i32 poison>, <2 x i32> <i32 2, i32 1>
; CHECK-NEXT: [[TMP8]] = mul <2 x i32> zeroinitializer, [[TMP7]]
; CHECK-NEXT: br i1 false, label %[[BB2]], label %[[BB6]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/copyable-operands-reordering.ll b/llvm/test/Transforms/SLPVectorizer/X86/copyable-operands-reordering.ll
index 4e1fa21eae704..cf50c19edf1c9 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/copyable-operands-reordering.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/copyable-operands-reordering.ll
@@ -6,24 +6,21 @@ define i64 @test(ptr %buf) {
; CHECK-SAME: ptr [[BUF:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[BUF]], align 1
-; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP0]] to i64
-; CHECK-NEXT: [[MUL:%.*]] = mul i64 [[CONV]], 24
; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[BUF]], i64 1
; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
-; CHECK-NEXT: [[CONV9:%.*]] = zext i8 [[TMP8]] to i64
-; CHECK-NEXT: [[MUL3:%.*]] = mul i64 [[CONV9]], 16
; CHECK-NEXT: [[ARRAYIDX22:%.*]] = getelementptr inbounds nuw i8, ptr [[BUF]], i64 2
; CHECK-NEXT: [[TMP10:%.*]] = load i8, ptr [[ARRAYIDX22]], align 1
-; CHECK-NEXT: [[CONV23:%.*]] = zext i8 [[TMP10]] to i64
-; CHECK-NEXT: [[MUL6:%.*]] = mul i64 [[CONV23]], 8
; CHECK-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds nuw i8, ptr [[BUF]], i64 3
-; CHECK-NEXT: [[TMP3:%.*]] = trunc i64 [[MUL3]] to i32
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i32> <i32 poison, i32 0, i32 0, i32 0>, i32 [[TMP3]], i32 0
-; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[MUL]] to i32
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> <i32 poison, i32 0, i32 0, i32 0>, i32 [[TMP5]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> <i32 poison, i32 1, i32 1, i32 1>, i32 [[TMP3]], i32 0
+; CHECK-NEXT: [[TMP6:%.*]] = mul <4 x i32> <i32 24, i32 0, i32 0, i32 0>, [[TMP5]]
+; CHECK-NEXT: [[TMP17:%.*]] = zext i8 [[TMP8]] to i32
+; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i32> <i32 poison, i32 1, i32 1, i32 1>, i32 [[TMP17]], i32 0
+; CHECK-NEXT: [[TMP4:%.*]] = mul <4 x i32> <i32 16, i32 0, i32 0, i32 0>, [[TMP20]]
; CHECK-NEXT: [[TMP7:%.*]] = or disjoint <4 x i32> [[TMP4]], [[TMP6]]
-; CHECK-NEXT: [[TMP17:%.*]] = trunc i64 [[MUL6]] to i32
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> <i32 poison, i32 0, i32 0, i32 0>, i32 [[TMP17]], i32 0
+; CHECK-NEXT: [[TMP21:%.*]] = zext i8 [[TMP10]] to i32
+; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i32> <i32 poison, i32 1, i32 1, i32 1>, i32 [[TMP21]], i32 0
+; CHECK-NEXT: [[TMP9:%.*]] = mul <4 x i32> <i32 8, i32 0, i32 0, i32 0>, [[TMP22]]
; CHECK-NEXT: [[TMP18:%.*]] = or disjoint <4 x i32> [[TMP7]], [[TMP9]]
; CHECK-NEXT: [[TMP19:%.*]] = load <4 x i8>, ptr [[ARRAYIDX8]], align 1
; CHECK-NEXT: [[TMP12:%.*]] = zext <4 x i8> [[TMP19]] to <4 x i32>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/gathered-node-with-in-order-parent.ll b/llvm/test/Transforms/SLPVectorizer/X86/gathered-node-with-in-order-parent.ll
index 66223d76cab81..222697ea03757 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/gathered-node-with-in-order-parent.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/gathered-node-with-in-order-parent.ll
@@ -9,17 +9,14 @@ define double @test() {
; CHECK-NEXT: [[TMP0:%.*]] = phi <4 x i32> [ zeroinitializer, %[[BB]] ], [ [[TMP3:%.*]], %[[BB4:.*]] ]
; CHECK-NEXT: br label %[[BB4]]
; CHECK: [[BB4]]:
-; CHECK-NEXT: [[MUL:%.*]] = mul i32 0, 1
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i32> <i32 poison, i32 0, i32 0, i32 1>, i32 [[MUL]], i32 0
-; CHECK-NEXT: [[TMP5:%.*]] = or <4 x i32> [[TMP0]], [[TMP4]]
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> <i32 poison, i32 0, i32 0, i32 0>, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> <i32 poison, i32 0, i32 0, i32 0>, i32 [[MUL]], i32 0
-; CHECK-NEXT: [[TMP3]] = or <4 x i32> [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[TMP5]], i32 2
+; CHECK-NEXT: [[TMP1:%.*]] = or <4 x i32> [[TMP0]], <i32 0, i32 0, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> <i32 0, i32 poison, i32 0, i32 0>, <4 x i32> <i32 4, i32 1, i32 6, i32 7>
+; CHECK-NEXT: [[TMP3]] = or <4 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
; CHECK-NEXT: [[AND:%.*]] = and i32 [[TMP6]], 0
; CHECK-NEXT: br i1 false, label %[[BB7:.*]], label %[[BB1]]
; CHECK: [[BB7]]:
-; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x i32> [ [[TMP5]], %[[BB4]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = phi <4 x i32> [ [[TMP1]], %[[BB4]] ]
; CHECK-NEXT: ret double 0.000000e+00
;
bb:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/non-schedulable-before-main.ll b/llvm/test/Transforms/SLPVectorizer/X86/non-schedulable-before-main.ll
index ccc71fadff2bd..ab563de8ff800 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/non-schedulable-before-main.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/non-schedulable-before-main.ll
@@ -8,11 +8,11 @@ define void @test() {
; CHECK: [[BB1]]:
; CHECK-NEXT: br label %[[BB5:.*]]
; CHECK: [[BB2]]:
-; CHECK-NEXT: [[MUL:%.*]] = mul i32 0, -51345
; CHECK-NEXT: [[ADD3:%.*]] = add i32 0, 1
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> <i32 0, i32 0, i32 poison, i32 1>, i32 [[MUL]], i32 2
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[TMP0]], <i32 1, i32 1, i32 0, i32 0>
; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[ADD3]] to i64
+; CHECK-NEXT: [[ADD4:%.*]] = add i32 0, 1
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> <i32 poison, i32 poison, i32 poison, i32 1>, i32 [[ADD4]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> <i32 1, i32 0, i32 undef, i32 undef>, <4 x i32> <i32 0, i32 4, i32 5, i32 3>
; CHECK-NEXT: br i1 false, label %[[BB5]], label %[[BB5]]
; CHECK: [[BB5]]:
; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x i32> [ [[TMP1]], %[[BB2]] ], [ zeroinitializer, %[[BB1]] ], [ [[TMP1]], %[[BB2]] ]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-with-removed-extracts.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-with-removed-extracts.ll
index fe5f4deecb8b3..a63c1a98605c5 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction-with-removed-extracts.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-with-removed-extracts.ll
@@ -8,19 +8,21 @@ define i32 @test(i32 %arg) {
; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> <i32 0, i32 poison, i32 0, i32 0>, i32 [[ARG]], i32 1
; CHECK-NEXT: br label %[[BB1:.*]]
; CHECK: [[BB1]]:
-; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[OP_RDX:%.*]], %[[BB1]] ]
+; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[OP_RDX5:%.*]], %[[BB1]] ]
; CHECK-NEXT: [[TMP5:%.*]] = or <4 x i32> [[TMP0]], zeroinitializer
; CHECK-NEXT: [[TMP6:%.*]] = or <4 x i32> [[TMP5]], zeroinitializer
; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> poison, <8 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3>
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> poison, <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP9:%.*]] = mul <2 x i32> zeroinitializer, [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> poison, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> poison, <4 x i32> <i32 poison, i32 0, i32 0, i32 3>
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP10:%.*]] = mul <4 x i32> zeroinitializer, [[TMP8]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i32> [[TMP10]], i32 3
; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <8 x i32> [[TMP7]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[RDX_OP:%.*]] = mul <4 x i32> [[TMP11]], [[TMP10]]
; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <4 x i32> [[RDX_OP]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <8 x i32> [[TMP7]], <8 x i32> [[TMP14]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.mul.v8i32(<8 x i32> [[TMP12]])
-; CHECK-NEXT: [[OP_RDX]] = mul i32 0, [[TMP13]]
+; CHECK-NEXT: [[OP_RDX4:%.*]] = mul i32 0, [[TMP9]]
+; CHECK-NEXT: [[OP_RDX5]] = mul i32 [[OP_RDX4]], [[TMP13]]
; CHECK-NEXT: br label %[[BB1]]
;
bb:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias-inseltpoison.ll
index 48853ef2af5da..ccc413486f6fb 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias-inseltpoison.ll
@@ -28,10 +28,8 @@ define void @test(ptr nocapture %t2) {
; CHECK-NEXT: [[T37:%.*]] = add nsw i32 [[T25]], [[T11]]
; CHECK-NEXT: [[T38:%.*]] = add nsw i32 [[T17]], [[T5]]
; CHECK-NEXT: [[T39:%.*]] = add nsw i32 [[T37]], [[T38]]
-; CHECK-NEXT: [[T40:%.*]] = mul nsw i32 [[T39]], 9633
; CHECK-NEXT: [[T41:%.*]] = mul nsw i32 [[T25]], 2446
; CHECK-NEXT: [[T42:%.*]] = mul nsw i32 [[T17]], 16819
-; CHECK-NEXT: [[T47:%.*]] = mul nsw i32 [[T37]], -16069
; CHECK-NEXT: [[T48:%.*]] = mul nsw i32 [[T38]], -3196
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[T8]], align 4
; CHECK-NEXT: [[T15:%.*]] = load i32, ptr [[T14]], align 4
@@ -39,14 +37,14 @@ define void @test(ptr nocapture %t2) {
; CHECK-NEXT: [[T29:%.*]] = sub nsw i32 [[T9]], [[T15]]
; CHECK-NEXT: [[T30:%.*]] = add nsw i32 [[T27]], [[T29]]
; CHECK-NEXT: [[T31:%.*]] = mul nsw i32 [[T30]], 4433
-; CHECK-NEXT: [[T32:%.*]] = mul nsw i32 [[T27]], 6270
; CHECK-NEXT: [[T34:%.*]] = mul nsw i32 [[T29]], -15137
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[T32]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[T47]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[T27]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[T37]], i32 1
; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 5, i32 poison>
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[T40]], i32 3
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>, i32 [[T40]], i32 1
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[T39]], i32 3
+; CHECK-NEXT: [[TMP6:%.*]] = mul nsw <4 x i32> [[TMP13]], <i32 6270, i32 -16069, i32 1, i32 9633>
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>, <4 x i32> [[TMP6]], <4 x i32> <i32 0, i32 7, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP7]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[T48]], i32 3
; CHECK-NEXT: [[TMP10:%.*]] = add nsw <4 x i32> [[TMP6]], [[TMP9]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias.ll b/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias.ll
index e82f68df2d877..9e87385c64df9 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias.ll
@@ -28,10 +28,8 @@ define void @test(ptr nocapture %t2) {
; CHECK-NEXT: [[T37:%.*]] = add nsw i32 [[T25]], [[T11]]
; CHECK-NEXT: [[T38:%.*]] = add nsw i32 [[T17]], [[T5]]
; CHECK-NEXT: [[T39:%.*]] = add nsw i32 [[T37]], [[T38]]
-; CHECK-NEXT: [[T40:%.*]] = mul nsw i32 [[T39]], 9633
; CHECK-NEXT: [[T41:%.*]] = mul nsw i32 [[T25]], 2446
; CHECK-NEXT: [[T42:%.*]] = mul nsw i32 [[T17]], 16819
-; CHECK-NEXT: [[T47:%.*]] = mul nsw i32 [[T37]], -16069
; CHECK-NEXT: [[T48:%.*]] = mul nsw i32 [[T38]], -3196
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[T8]], align 4
; CHECK-NEXT: [[T15:%.*]] = load i32, ptr [[T14]], align 4
@@ -39,14 +37,14 @@ define void @test(ptr nocapture %t2) {
; CHECK-NEXT: [[T29:%.*]] = sub nsw i32 [[T9]], [[T15]]
; CHECK-NEXT: [[T30:%.*]] = add nsw i32 [[T27]], [[T29]]
; CHECK-NEXT: [[T31:%.*]] = mul nsw i32 [[T30]], 4433
-; CHECK-NEXT: [[T32:%.*]] = mul nsw i32 [[T27]], 6270
; CHECK-NEXT: [[T34:%.*]] = mul nsw i32 [[T29]], -15137
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[T32]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[T47]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[T27]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[T37]], i32 1
; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 5, i32 poison>
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[T40]], i32 3
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>, i32 [[T40]], i32 1
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[T39]], i32 3
+; CHECK-NEXT: [[TMP6:%.*]] = mul nsw <4 x i32> [[TMP13]], <i32 6270, i32 -16069, i32 1, i32 9633>
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>, <4 x i32> [[TMP6]], <4 x i32> <i32 0, i32 7, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP7]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[T48]], i32 3
; CHECK-NEXT: [[TMP10:%.*]] = add nsw <4 x i32> [[TMP6]], [[TMP9]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias_external_insert_shuffled.ll b/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias_external_insert_shuffled.ll
index d1b223570ff98..6736e621ced66 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias_external_insert_shuffled.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias_external_insert_shuffled.ll
@@ -22,10 +22,8 @@ define void @test(ptr nocapture %t2) {
; CHECK-NEXT: [[T37:%.*]] = add nsw i32 [[T25]], [[T11]]
; CHECK-NEXT: [[T38:%.*]] = add nsw i32 [[T17]], [[T5]]
; CHECK-NEXT: [[T39:%.*]] = add nsw i32 [[T37]], [[T38]]
-; CHECK-NEXT: [[T40:%.*]] = mul nsw i32 [[T39]], 9633
; CHECK-NEXT: [[T41:%.*]] = mul nsw i32 [[T25]], 2446
; CHECK-NEXT: [[T42:%.*]] = mul nsw i32 [[T17]], 16819
-; CHECK-NEXT: [[T47:%.*]] = mul nsw i32 [[T37]], -16069
; CHECK-NEXT: [[T48:%.*]] = mul nsw i32 [[T38]], -3196
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[T8]], align 4
; CHECK-NEXT: [[T15:%.*]] = load i32, ptr [[T14]], align 4
@@ -33,14 +31,14 @@ define void @test(ptr nocapture %t2) {
; CHECK-NEXT: [[T29:%.*]] = sub nsw i32 [[T9]], [[T15]]
; CHECK-NEXT: [[T30:%.*]] = add nsw i32 [[T27]], [[T29]]
; CHECK-NEXT: [[T31:%.*]] = mul nsw i32 [[T30]], 4433
-; CHECK-NEXT: [[T32:%.*]] = mul nsw i32 [[T27]], 6270
; CHECK-NEXT: [[T34:%.*]] = mul nsw i32 [[T29]], -15137
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[T32]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[T47]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[T27]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[T37]], i32 1
; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 5, i32 poison>
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[T40]], i32 3
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>, i32 [[T40]], i32 1
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[T39]], i32 3
+; CHECK-NEXT: [[TMP6:%.*]] = mul nsw <4 x i32> [[TMP13]], <i32 6270, i32 -16069, i32 1, i32 9633>
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>, <4 x i32> [[TMP6]], <4 x i32> <i32 0, i32 7, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP7]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[T48]], i32 3
; CHECK-NEXT: [[TMP10:%.*]] = add nsw <4 x i32> [[TMP6]], [[TMP9]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll b/llvm/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll
index 7388bef723134..f76e5049cf141 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll
@@ -211,18 +211,9 @@ entry:
define void @mul(ptr noalias %dst, ptr noalias %src) {
; CHECK-LABEL: @mul(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[INCDEC_PTR2:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 2
-; CHECK-NEXT: [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 2
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[SRC]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = mul nsw <2 x i32> [[TMP0]], <i32 257, i32 -3>
-; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[DST]], align 4
-; CHECK-NEXT: [[INCDEC_PTR5:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 3
-; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[INCDEC_PTR2]], align 4
-; CHECK-NEXT: [[INCDEC_PTR7:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 3
-; CHECK-NEXT: store i32 [[TMP2]], ptr [[INCDEC_PTR4]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[INCDEC_PTR5]], align 4
-; CHECK-NEXT: [[MUL9:%.*]] = mul nsw i32 [[TMP3]], -9
-; CHECK-NEXT: store i32 [[MUL9]], ptr [[INCDEC_PTR7]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr [[SRC:%.*]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = mul nsw <4 x i32> [[TMP0]], <i32 257, i32 -3, i32 1, i32 -9>
+; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[DST:%.*]], align 4
; CHECK-NEXT: ret void
;
entry:
>From b0d35497b666b3e0c0c4244c1f8c19f2cfdc1fb2 Mon Sep 17 00:00:00 2001
From: Quentin Colombet <quentin.colombet at gmail.com>
Date: Mon, 6 Jul 2026 18:53:16 +0200
Subject: [PATCH 19/46] [MIR] Serialize/Deserialize MachineInstr::LRSplit flag
(#197362)
The LRSplit MachineInstr flag is set by SplitKit on copies inserted for
live-range splitting.
Until now the flag had no MIR-text representation.
This patch fixes that so that it gets easier to reproduce/capture issues
that involves SplitKit.
Round-trip coverage in
llvm/test/CodeGen/MIR/AMDGPU/lr-split-flag.mir.
---
llvm/lib/CodeGen/MIRParser/MILexer.cpp | 1 +
llvm/lib/CodeGen/MIRParser/MILexer.h | 1 +
llvm/lib/CodeGen/MIRParser/MIParser.cpp | 5 +-
llvm/lib/CodeGen/MIRPrinter.cpp | 4 +-
llvm/lib/CodeGen/MachineInstr.cpp | 2 +
.../CodeGen/AArch64/nested-iv-regalloc.mir | 4 +-
.../branch-folding-implicit-def-subreg.ll | 4 +-
.../CodeGen/AMDGPU/bug-undef-spilled-agpr.mir | 4 +-
llvm/test/CodeGen/AMDGPU/dead_bundle.mir | 4 +-
.../extend-phi-subrange-not-in-parent.mir | 6 +-
.../AMDGPU/greedy-global-heuristic.mir | 8 +-
...class-vgpr-mfma-to-agpr-negative-tests.mir | 14 +-
...class-vgpr-mfma-to-av-with-load-source.mir | 28 +-
.../inflate-reg-class-vgpr-mfma-to-av.mir | 6 +-
...-reg-class-snippet-copy-use-after-free.mir | 44 +--
...nfloop-subrange-spill-inspect-subrange.mir | 4 +-
.../CodeGen/AMDGPU/infloop-subrange-spill.mir | 4 +-
...al-regcopy-and-spill-missed-at-regalloc.ll | 14 +-
.../ra-inserted-scalar-instructions.mir | 72 ++--
.../ran-out-of-sgprs-allocation-failure.mir | 54 +--
...-unsatisfiable-overlapping-tuple-hints.mir | 2 +-
.../AMDGPU/regalloc-undef-copy-fold.mir | 2 +-
.../split-liverange-overlapping-copies.mir | 12 +-
.../CodeGen/AMDGPU/splitkit-copy-bundle.mir | 64 ++--
.../AMDGPU/splitkit-copy-live-lanes.mir | 336 +++++++++---------
...-do-not-undo-subclass-split-with-remat.mir | 4 +-
llvm/test/CodeGen/AMDGPU/splitkit.mir | 4 +-
...-last-chance-recoloring-alloc-succeeds.mir | 2 +-
.../test/CodeGen/MIR/AMDGPU/lr-split-flag.mir | 25 ++
llvm/test/CodeGen/RISCV/pr176001.ll | 4 +-
.../X86/AMX/amx-greedy-ra-spill-shape.ll | 26 +-
llvm/test/CodeGen/X86/apx/foldmemory.mir | 16 +-
.../X86/statepoint-invoke-ra-enter-at-end.mir | 12 +-
.../statepoint-invoke-ra-inline-spiller.mir | 44 +--
.../test/CodeGen/X86/statepoint-invoke-ra.mir | 2 +-
.../MIR/X86/dvl-livedebugvars-movements.mir | 2 +-
.../MIR/X86/dvl-livedebugvars-stackptr.mir | 2 +-
llvm/utils/UpdateTestChecks/mir.py | 2 +-
38 files changed, 439 insertions(+), 405 deletions(-)
create mode 100644 llvm/test/CodeGen/MIR/AMDGPU/lr-split-flag.mir
diff --git a/llvm/lib/CodeGen/MIRParser/MILexer.cpp b/llvm/lib/CodeGen/MIRParser/MILexer.cpp
index 4188ff8f85131..cabe910ef28ca 100644
--- a/llvm/lib/CodeGen/MIRParser/MILexer.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MILexer.cpp
@@ -300,6 +300,7 @@ static MIToken::TokenKind getIdentifierKind(StringRef Identifier) {
.Case("call-frame-size", MIToken::kw_call_frame_size)
.Case("noconvergent", MIToken::kw_noconvergent)
.Case("mmra", MIToken::kw_mmra)
+ .Case("lr-split", MIToken::kw_lr_split)
.Default(MIToken::Identifier);
}
diff --git a/llvm/lib/CodeGen/MIRParser/MILexer.h b/llvm/lib/CodeGen/MIRParser/MILexer.h
index 3915222896263..3922caf4220d3 100644
--- a/llvm/lib/CodeGen/MIRParser/MILexer.h
+++ b/llvm/lib/CodeGen/MIRParser/MILexer.h
@@ -152,6 +152,7 @@ struct MIToken {
kw_call_frame_size,
kw_noconvergent,
kw_mmra,
+ kw_lr_split,
// Metadata types.
kw_distinct,
diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
index 2924c0dda2390..f38c0d9e8baca 100644
--- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
@@ -1536,7 +1536,8 @@ bool MIParser::parseInstruction(unsigned &OpCode, unsigned &Flags) {
Token.is(MIToken::kw_disjoint) ||
Token.is(MIToken::kw_nusw) ||
Token.is(MIToken::kw_samesign) ||
- Token.is(MIToken::kw_inbounds)) {
+ Token.is(MIToken::kw_inbounds) ||
+ Token.is(MIToken::kw_lr_split)) {
// clang-format on
// Mine frame and fast math flags
if (Token.is(MIToken::kw_frame_setup))
@@ -1579,6 +1580,8 @@ bool MIParser::parseInstruction(unsigned &OpCode, unsigned &Flags) {
Flags |= MachineInstr::SameSign;
if (Token.is(MIToken::kw_inbounds))
Flags |= MachineInstr::InBounds;
+ if (Token.is(MIToken::kw_lr_split))
+ Flags |= MachineInstr::LRSplit;
lex();
}
diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp
index b65de01898fd5..8acd6f14ebc2e 100644
--- a/llvm/lib/CodeGen/MIRPrinter.cpp
+++ b/llvm/lib/CodeGen/MIRPrinter.cpp
@@ -892,9 +892,11 @@ static void printMI(raw_ostream &OS, MFPrintState &State,
OS << "samesign ";
if (MI.getFlag(MachineInstr::InBounds))
OS << "inbounds ";
+ if (MI.getFlag(MachineInstr::LRSplit))
+ OS << "lr-split ";
// NOTE: Please add new MIFlags also to the MI_FLAGS_STR in
- // llvm/utils/update_mir_test_checks.py.
+ // llvm/utils/UpdateTestChecks/mir.py.
OS << TII->getName(MI.getOpcode());
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 374c92241b9fb..0fbeea153d3c9 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -1897,6 +1897,8 @@ void MachineInstr::print(raw_ostream &OS, ModuleSlotTracker &MST,
OS << "samesign ";
if (getFlag(MachineInstr::InBounds))
OS << "inbounds ";
+ if (getFlag(MachineInstr::LRSplit))
+ OS << "lr-split ";
// Print the opcode name.
if (TII)
diff --git a/llvm/test/CodeGen/AArch64/nested-iv-regalloc.mir b/llvm/test/CodeGen/AArch64/nested-iv-regalloc.mir
index 78aa57f778807..ef48ff55ac973 100644
--- a/llvm/test/CodeGen/AArch64/nested-iv-regalloc.mir
+++ b/llvm/test/CodeGen/AArch64/nested-iv-regalloc.mir
@@ -209,7 +209,7 @@ body: |
; CHECK-NEXT: renamable $w9 = LDRWui killed renamable $x0, 1 :: (load (s32) from %ir.arrayidx2)
; CHECK-NEXT: dead $wzr = SUBSWri renamable $w8, 1, 0, implicit-def $nzcv
; CHECK-NEXT: renamable $w11 = CSINCWr killed renamable $w8, $wzr, 12, implicit $nzcv
- ; CHECK-NEXT: renamable $x8 = COPY killed renamable $x10
+ ; CHECK-NEXT: renamable $x8 = lr-split COPY killed renamable $x10
; CHECK-NEXT: dead $wzr = SUBSWri renamable $w9, 1, 0, implicit-def $nzcv
; CHECK-NEXT: renamable $w10 = CSINCWr killed renamable $w9, $wzr, 12, implicit $nzcv
; CHECK-NEXT: STRXui renamable $x2, %stack.0, 0 :: (store (s64) into %stack.0)
@@ -266,7 +266,7 @@ body: |
; CHECK-NEXT: successors: %bb.8(0x80000000)
; CHECK-NEXT: liveins: $w20
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $w1 = COPY killed renamable $w20
+ ; CHECK-NEXT: renamable $w1 = lr-split COPY killed renamable $w20
; CHECK-NEXT: INLINEASM &"nop;nop", sideeffect attdialect, clobber, implicit-def dead early-clobber $x0, clobber, implicit-def dead early-clobber $x2, clobber, implicit-def dead early-clobber $x3, clobber, implicit-def dead early-clobber $x4, clobber, implicit-def dead early-clobber $x5, clobber, implicit-def dead early-clobber $x6, clobber, implicit-def dead early-clobber $x7, clobber, implicit-def dead early-clobber $x8, clobber, implicit-def dead early-clobber $x9, clobber, implicit-def dead early-clobber $x10, clobber, implicit-def dead early-clobber $x11, clobber, implicit-def dead early-clobber $x12, clobber, implicit-def dead early-clobber $x13, clobber, implicit-def dead early-clobber $x14, clobber, implicit-def dead early-clobber $x15, clobber, implicit-def dead early-clobber $x16, clobber, implicit-def dead early-clobber $x17, clobber, implicit-def dead early-clobber $x18, clobber, implicit-def dead early-clobber $x19, clobber, implicit-def dead early-clobber $x20, clobber, implicit-def dead early-clobber $x21, clobber, implicit-def dead early-clobber $x22, clobber, implicit-def dead early-clobber $x23, clobber, implicit-def dead early-clobber $x24, clobber, implicit-def dead early-clobber $x25, clobber, implicit-def dead early-clobber $x26, clobber, implicit-def dead early-clobber $x27, clobber, implicit-def dead early-clobber $x28, clobber, implicit-def dead early-clobber $fp, clobber, implicit-def dead early-clobber $lr
; CHECK-NEXT: renamable $x2 = LDRXui %stack.0, 0 :: (load (s64) from %stack.0)
; CHECK-NEXT: renamable $x12 = LDRXui %stack.4, 0 :: (load (s64) from %stack.4)
diff --git a/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll b/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
index 5531b2f5ad5d5..86913fef502ad 100644
--- a/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
@@ -942,7 +942,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: successors: %bb.72(0x40000000), %bb.69(0x40000000)
; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000F, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr26_vgpr27:0x000000000000000F, $vgpr28_vgpr29:0x000000000000000F, $vgpr32_vgpr33:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: renamable $vgpr30 = COPY killed renamable $vgpr14, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr30 = lr-split COPY killed renamable $vgpr14, implicit $exec
; GFX90A-NEXT: renamable $vgpr34 = V_OR_B32_e32 1, $vgpr32, implicit $exec
; GFX90A-NEXT: renamable $vgpr54 = V_OR_B32_e32 $vgpr34, $vgpr28, implicit $exec
; GFX90A-NEXT: renamable $vgpr48 = V_OR_B32_e32 $vgpr54, $vgpr26, implicit $exec
@@ -998,7 +998,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr16_vgpr17:0x000000000000000C, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x000000000000000C, $vgpr30_vgpr31:0x0000000000000003, $vgpr32_vgpr33:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $sgpr24_sgpr25 = S_MOV_B64 0
- ; GFX90A-NEXT: renamable $vgpr14 = COPY killed renamable $vgpr30, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr14 = lr-split COPY killed renamable $vgpr30, implicit $exec
; GFX90A-NEXT: S_BRANCH %bb.63
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.72.bb196:
diff --git a/llvm/test/CodeGen/AMDGPU/bug-undef-spilled-agpr.mir b/llvm/test/CodeGen/AMDGPU/bug-undef-spilled-agpr.mir
index 3bf6f4387228d..7e6d841b7ec13 100644
--- a/llvm/test/CodeGen/AMDGPU/bug-undef-spilled-agpr.mir
+++ b/llvm/test/CodeGen/AMDGPU/bug-undef-spilled-agpr.mir
@@ -34,7 +34,7 @@ body: |
; GCN-NEXT: renamable $vgpr62 = IMPLICIT_DEF
; GCN-NEXT: $vgpr62 = SI_SPILL_S32_TO_VGPR $sgpr15, 0, killed $vgpr62
; GCN-NEXT: $noreg = S_OR_SAVEEXEC_B64 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec
- ; GCN-NEXT: renamable $agpr0 = COPY killed renamable $vgpr62
+ ; GCN-NEXT: renamable $agpr0 = lr-split COPY killed renamable $vgpr62
; GCN-NEXT: $exec = S_MOV_B64 killed $noreg
; GCN-NEXT: renamable $vgpr62 = IMPLICIT_DEF
; GCN-NEXT: dead renamable $vgpr62 = V_AND_B32_e32 1, killed $vgpr62, implicit $exec
@@ -58,7 +58,7 @@ body: |
; GCN-NEXT: liveins: $agpr0
; GCN-NEXT: {{ $}}
; GCN-NEXT: $noreg = S_OR_SAVEEXEC_B64 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec
- ; GCN-NEXT: renamable $vgpr62 = COPY renamable $agpr0
+ ; GCN-NEXT: renamable $vgpr62 = lr-split COPY renamable $agpr0
; GCN-NEXT: $exec = S_MOV_B64 killed $noreg
; GCN-NEXT: $sgpr14 = SI_RESTORE_S32_FROM_VGPR killed $vgpr62, 1
; GCN-NEXT: S_BRANCH %bb.2
diff --git a/llvm/test/CodeGen/AMDGPU/dead_bundle.mir b/llvm/test/CodeGen/AMDGPU/dead_bundle.mir
index af656ea1c7193..d68a8dc9d1560 100644
--- a/llvm/test/CodeGen/AMDGPU/dead_bundle.mir
+++ b/llvm/test/CodeGen/AMDGPU/dead_bundle.mir
@@ -22,8 +22,8 @@ body: |
; CHECK-NEXT: dead [[V_CVT_U32_F32_e64_:%[0-9]+]]:vgpr_32 = V_CVT_U32_F32_e64 0, $sgpr4, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: dead renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = IMPLICIT_DEF
; CHECK-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11 = S_BUFFER_LOAD_DWORDX8_IMM undef renamable $sgpr0_sgpr1_sgpr2_sgpr3, 416, 0 :: (dereferenceable invariant load (s256), align 4)
- ; CHECK-NEXT: renamable $sgpr3 = COPY killed renamable $sgpr7
- ; CHECK-NEXT: renamable $sgpr5 = COPY renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr3 = lr-split COPY killed renamable $sgpr7
+ ; CHECK-NEXT: renamable $sgpr5 = lr-split COPY renamable $sgpr9
; CHECK-NEXT: dead undef [[COPY:%[0-9]+]].sub0:vreg_64 = COPY renamable $sgpr3
; CHECK-NEXT: dead undef [[COPY1:%[0-9]+]].sub1:vreg_64 = COPY killed renamable $sgpr5
; CHECK-NEXT: dead [[IMAGE_SAMPLE_V1_V2_gfx11_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_V1_V2_gfx11 undef [[COPY]], undef renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
diff --git a/llvm/test/CodeGen/AMDGPU/extend-phi-subrange-not-in-parent.mir b/llvm/test/CodeGen/AMDGPU/extend-phi-subrange-not-in-parent.mir
index 7ce35b76ba5b8..b43c2ea469370 100644
--- a/llvm/test/CodeGen/AMDGPU/extend-phi-subrange-not-in-parent.mir
+++ b/llvm/test/CodeGen/AMDGPU/extend-phi-subrange-not-in-parent.mir
@@ -35,9 +35,9 @@ body: |
; CHECK-NEXT: bb.2:
; CHECK-NEXT: successors: %bb.3(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: undef [[COPY1:%[0-9]+]].sub1_sub2_sub3_sub4_sub5_sub6_sub7_sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15_sub16:av_1024_align2 = COPY [[COPY]].sub1_sub2_sub3_sub4_sub5_sub6_sub7_sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15_sub16 {
- ; CHECK-NEXT: internal [[COPY1]].sub20_lo16_sub20_hi16_sub21_lo16_sub21_hi16_sub22_lo16_sub22_hi16_sub23_lo16_sub23_hi16_sub24_lo16_sub24_hi16_sub25_lo16_sub25_hi16_sub26_lo16_sub26_hi16_sub27_lo16_sub27_hi16_sub28_lo16_sub28_hi16_sub29_lo16_sub29_hi16_sub30_lo16_sub30_hi16_sub31_lo16_sub31_hi16:av_1024_align2 = COPY [[COPY]].sub20_lo16_sub20_hi16_sub21_lo16_sub21_hi16_sub22_lo16_sub22_hi16_sub23_lo16_sub23_hi16_sub24_lo16_sub24_hi16_sub25_lo16_sub25_hi16_sub26_lo16_sub26_hi16_sub27_lo16_sub27_hi16_sub28_lo16_sub28_hi16_sub29_lo16_sub29_hi16_sub30_lo16_sub30_hi16_sub31_lo16_sub31_hi16
- ; CHECK-NEXT: internal [[COPY1]].sub17_sub18_sub19:av_1024_align2 = COPY [[COPY]].sub17_sub18_sub19
+ ; CHECK-NEXT: undef [[COPY1:%[0-9]+]].sub1_sub2_sub3_sub4_sub5_sub6_sub7_sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15_sub16:av_1024_align2 = lr-split COPY [[COPY]].sub1_sub2_sub3_sub4_sub5_sub6_sub7_sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15_sub16 {
+ ; CHECK-NEXT: internal [[COPY1]].sub20_lo16_sub20_hi16_sub21_lo16_sub21_hi16_sub22_lo16_sub22_hi16_sub23_lo16_sub23_hi16_sub24_lo16_sub24_hi16_sub25_lo16_sub25_hi16_sub26_lo16_sub26_hi16_sub27_lo16_sub27_hi16_sub28_lo16_sub28_hi16_sub29_lo16_sub29_hi16_sub30_lo16_sub30_hi16_sub31_lo16_sub31_hi16:av_1024_align2 = lr-split COPY [[COPY]].sub20_lo16_sub20_hi16_sub21_lo16_sub21_hi16_sub22_lo16_sub22_hi16_sub23_lo16_sub23_hi16_sub24_lo16_sub24_hi16_sub25_lo16_sub25_hi16_sub26_lo16_sub26_hi16_sub27_lo16_sub27_hi16_sub28_lo16_sub28_hi16_sub29_lo16_sub29_hi16_sub30_lo16_sub30_hi16_sub31_lo16_sub31_hi16
+ ; CHECK-NEXT: internal [[COPY1]].sub17_sub18_sub19:av_1024_align2 = lr-split COPY [[COPY]].sub17_sub18_sub19
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY1:%[0-9]+]].sub0:av_1024_align2 = IMPLICIT_DEF
; CHECK-NEXT: S_NOP 0, implicit [[COPY1]].sub0
diff --git a/llvm/test/CodeGen/AMDGPU/greedy-global-heuristic.mir b/llvm/test/CodeGen/AMDGPU/greedy-global-heuristic.mir
index 6f1e5b89db884..1f2e91bc9cc74 100644
--- a/llvm/test/CodeGen/AMDGPU/greedy-global-heuristic.mir
+++ b/llvm/test/CodeGen/AMDGPU/greedy-global-heuristic.mir
@@ -133,14 +133,14 @@ body: |
; CHECK-NEXT: S_NOP 0
; CHECK-NEXT: S_NOP 0
; CHECK-NEXT: S_NOP 0
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_128 = COPY %31
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_128 = lr-split COPY %31
; CHECK-NEXT: S_NOP 0, implicit %31
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY %29
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = lr-split COPY %29
; CHECK-NEXT: S_NOP 0, implicit %29
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vreg_128 = COPY %27
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vreg_128 = lr-split COPY %27
; CHECK-NEXT: S_NOP 0, implicit %27
; CHECK-NEXT: [[SI_SPILL_V128_RESTORE1:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.1, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.1, align 4, addrspace 5)
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vreg_128 = COPY [[SI_SPILL_V128_RESTORE1]]
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE1]]
; CHECK-NEXT: S_NOP 0, implicit [[SI_SPILL_V128_RESTORE1]]
; CHECK-NEXT: [[SI_SPILL_V128_RESTORE2:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.0, align 4, addrspace 5)
; CHECK-NEXT: S_NOP 0, implicit [[SI_SPILL_V128_RESTORE2]]
diff --git a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir
index 45c185b6154ea..caa566983530a 100644
--- a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir
+++ b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir
@@ -159,7 +159,7 @@ body: |
; CHECK-NEXT: liveins: $vgpr8, $vgpr31, $sgpr8_sgpr9, $vgpr0_vgpr1:0x0000000000000003, $vgpr2_vgpr3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: renamable $vgpr1 = COPY renamable $vgpr8
- ; CHECK-NEXT: renamable $agpr0_agpr1 = COPY killed renamable $vgpr2_vgpr3
+ ; CHECK-NEXT: renamable $agpr0_agpr1 = lr-split COPY killed renamable $vgpr2_vgpr3
; CHECK-NEXT: renamable $vgpr2 = COPY renamable $vgpr1
; CHECK-NEXT: renamable $vgpr3 = COPY renamable $vgpr8
; CHECK-NEXT: renamable $vgpr6 = COPY renamable $vgpr1
@@ -207,7 +207,7 @@ body: |
; CHECK-NEXT: renamable $vgpr28 = COPY renamable $vgpr12
; CHECK-NEXT: renamable $vgpr4_vgpr5 = V_MOV_B64_PSEUDO 0, implicit $exec
; CHECK-NEXT: renamable $vgpr4_vgpr5 = V_ADD_F64_e64 0, killed $vgpr28_vgpr29, 0, killed $vgpr4_vgpr5, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: renamable $vgpr28_vgpr29 = COPY killed renamable $agpr0_agpr1
+ ; CHECK-NEXT: renamable $vgpr28_vgpr29 = lr-split COPY killed renamable $agpr0_agpr1
; CHECK-NEXT: renamable $vgpr4_vgpr5 = V_ADD_F64_e64 0, killed $vgpr4_vgpr5, 0, killed $vgpr28_vgpr29, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: dead renamable $vgpr4 = COPY killed renamable $vgpr11
; CHECK-NEXT: dead renamable $vgpr4 = COPY killed renamable $vgpr14
@@ -366,7 +366,7 @@ body: |
; CHECK-NEXT: bb.2:
; CHECK-NEXT: liveins: $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17:0x00000000FFFFFFFF
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17
+ ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = lr-split COPY killed renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
@@ -455,7 +455,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
- ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35
+ ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = lr-split COPY killed renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
@@ -546,7 +546,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
- ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35
+ ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = lr-split COPY killed renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
@@ -651,7 +651,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
- ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35
+ ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = lr-split COPY killed renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
@@ -770,7 +770,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
- ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35
+ ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = lr-split COPY killed renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
diff --git a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir
index f19f29d5a5adc..d53e671759fa7 100644
--- a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir
+++ b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir
@@ -485,7 +485,7 @@ body: |
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
- ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = lr-split COPY killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
; CHECK-NEXT: INLINEASM &"; use $0 ", sideeffect attdialect, reguse:VReg_512_Align2, killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; CHECK-NEXT: S_ENDPGM 0
bb.0:
@@ -889,7 +889,7 @@ body: |
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = COPY killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = lr-split COPY killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr10_vgpr11_vgpr12_vgpr13, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr14_vgpr15_vgpr16_vgpr17, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr2_vgpr3_vgpr4_vgpr5, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
@@ -962,7 +962,7 @@ body: |
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = COPY killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = lr-split COPY killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr10_vgpr11_vgpr12_vgpr13, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr14_vgpr15_vgpr16_vgpr17, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr2_vgpr3_vgpr4_vgpr5, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
@@ -1033,7 +1033,7 @@ body: |
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
; CHECK-NEXT: renamable $vgpr0_vgpr1 = SI_SPILL_AV64_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s64) from %stack.0, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $agpr16_agpr17 = COPY killed renamable $agpr0_agpr1
+ ; CHECK-NEXT: renamable $agpr16_agpr17 = lr-split COPY killed renamable $agpr0_agpr1
; CHECK-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_e64 killed $vgpr0_vgpr1, $vgpr0_vgpr1, $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
@@ -1044,7 +1044,7 @@ body: |
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = COPY killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = lr-split COPY killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr10_vgpr11_vgpr12_vgpr13, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr14_vgpr15_vgpr16_vgpr17, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr2_vgpr3_vgpr4_vgpr5, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
@@ -1127,8 +1127,8 @@ body: |
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: renamable $vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = COPY killed renamable $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5 = COPY renamable $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: renamable $vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = lr-split COPY killed renamable $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5 = lr-split COPY renamable $agpr0_agpr1_agpr2_agpr3
; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr10_vgpr11_vgpr12_vgpr13, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr14_vgpr15_vgpr16_vgpr17, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $vgpr2_vgpr3_vgpr4_vgpr5, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
@@ -1199,7 +1199,7 @@ body: |
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
; CHECK-NEXT: renamable $vgpr0_vgpr1 = SI_SPILL_AV64_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s64) from %stack.0, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $agpr16_agpr17 = COPY killed renamable $agpr0_agpr1
+ ; CHECK-NEXT: renamable $agpr16_agpr17 = lr-split COPY killed renamable $agpr0_agpr1
; CHECK-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_e64 killed $vgpr0_vgpr1, $vgpr0_vgpr1, $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
@@ -1210,8 +1210,8 @@ body: |
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: renamable $vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = COPY killed renamable $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5 = COPY renamable $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: renamable $vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = lr-split COPY killed renamable $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5 = lr-split COPY renamable $agpr0_agpr1_agpr2_agpr3
; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr10_vgpr11_vgpr12_vgpr13, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr14_vgpr15_vgpr16_vgpr17, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $vgpr2_vgpr3_vgpr4_vgpr5, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
@@ -1286,7 +1286,7 @@ body: |
; CHECK-NEXT: bb.2:
; CHECK-NEXT: liveins: $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33:0x00000000FFFFFFFF
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17
+ ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = lr-split COPY killed renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
@@ -1377,7 +1377,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
- ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
+ ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = lr-split COPY killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
@@ -1469,7 +1469,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
- ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
+ ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = lr-split COPY killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
@@ -1735,7 +1735,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
- ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
+ ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = lr-split COPY killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
diff --git a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av.mir b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av.mir
index ee0d729e2abbf..ed5e907cc1e74 100644
--- a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av.mir
+++ b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av.mir
@@ -39,7 +39,7 @@ body: |
; CHECK-NEXT: bb.2:
; CHECK-NEXT: liveins: $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17:0x00000000FFFFFFFF
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17
+ ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = lr-split COPY killed renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
@@ -133,7 +133,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
- ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
+ ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = lr-split COPY killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
@@ -223,7 +223,7 @@ body: |
; CHECK-NEXT: bb.2:
; CHECK-NEXT: liveins: $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17:0x00000000FFFFFFFF
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17
+ ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = lr-split COPY killed renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
diff --git a/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir b/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir
index 06c3da09eede9..9a46cb1496570 100644
--- a/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir
+++ b/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir
@@ -29,14 +29,14 @@
# CHECK-NEXT: SI_SPILL_AV512_SAVE [[ORIG_REG]], %stack.0, $sgpr32, 0, implicit $exec :: (store (s512) into %stack.0, align 4, addrspace 5)
# CHECK-NEXT: [[RESTORE0:%[0-9]+]]:vreg_512_align2 = SI_SPILL_AV512_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s512) from %stack.0, align 4, addrspace 5)
# CHECK-NEXT: early-clobber $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_16X16X1F32_vgprcd_e64 undef %3:vgpr_32, undef %3:vgpr_32, [[RESTORE0]], 0, 0, 0, implicit $mode, implicit $exec, implicit $mode, implicit $exec
-# CHECK-NEXT: undef [[SPLIT0:%[0-9]+]].sub2_sub3:av_512_align2 = COPY undef $vgpr2_vgpr3 {
-# CHECK-NEXT: internal [[SPLIT0]].sub0:av_512_align2 = COPY undef $vgpr0
+# CHECK-NEXT: undef [[SPLIT0:%[0-9]+]].sub2_sub3:av_512_align2 = lr-split COPY undef $vgpr2_vgpr3 {
+# CHECK-NEXT: internal [[SPLIT0]].sub0:av_512_align2 = lr-split COPY undef $vgpr0
# CHECK-NEXT: }
-# CHECK-NEXT: undef [[SPLIT2:%[0-9]+]].sub2_sub3:av_512_align2 = COPY [[SPLIT0]].sub2_sub3 {
-# CHECK-NEXT: internal [[SPLIT2]].sub0:av_512_align2 = COPY [[SPLIT0]].sub0
+# CHECK-NEXT: undef [[SPLIT2:%[0-9]+]].sub2_sub3:av_512_align2 = lr-split COPY [[SPLIT0]].sub2_sub3 {
+# CHECK-NEXT: internal [[SPLIT2]].sub0:av_512_align2 = lr-split COPY [[SPLIT0]].sub0
# CHECK-NEXT: }
# CHECK-NEXT: [[RESTORE2:%[0-9]+]]:av_512_align2 = SI_SPILL_AV512_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s512) from %stack.0, align 4, addrspace 5)
-# CHECK-NEXT: [[MFMA_USE1:%[0-9]+]].sub0_sub1:vreg_512_align2 = COPY [[RESTORE2]].sub0_sub1
+# CHECK-NEXT: [[MFMA_USE1:%[0-9]+]].sub0_sub1:vreg_512_align2 = lr-split COPY [[RESTORE2]].sub0_sub1
# CHECK-NEXT: [[MFMA_USE1]].sub2:vreg_512_align2 = COPY [[SPLIT2]].sub3
# CHECK-NEXT: [[MFMA_USE1]].sub3:vreg_512_align2 = COPY [[SPLIT2]].sub2
# CHECK-NEXT: [[MFMA_USE1]].sub4:vreg_512_align2 = COPY [[SPLIT2]].sub0
@@ -102,34 +102,34 @@ body: |
# CHECK-NEXT: SI_SPILL_AV512_SAVE [[ORIG_REG]], %stack.0, $sgpr32, 0, implicit $exec :: (store (s512) into %stack.0, align 4, addrspace 5)
# CHECK-NEXT: [[RESTORE_0:%[0-9]+]]:av_512_align2 = SI_SPILL_AV512_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s512) from %stack.0, align 4, addrspace 5)
# CHECK-NEXT: S_NOP 0, implicit-def early-clobber $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit [[RESTORE_0]].sub0_sub1_sub2_sub3, implicit [[RESTORE_0]].sub4_sub5_sub6_sub7
-# CHECK-NEXT: undef [[SPLIT0:%[0-9]+]].sub2_sub3:av_512_align2 = COPY undef $vgpr2_vgpr3 {
-# CHECK-NEXT: internal [[SPLIT0]].sub0:av_512_align2 = COPY undef $vgpr0
+# CHECK-NEXT: undef [[SPLIT0:%[0-9]+]].sub2_sub3:av_512_align2 = lr-split COPY undef $vgpr2_vgpr3 {
+# CHECK-NEXT: internal [[SPLIT0]].sub0:av_512_align2 = lr-split COPY undef $vgpr0
# CHECK-NEXT: }
-# CHECK-NEXT: undef [[SPLIT1:%[0-9]+]].sub2_sub3:av_512_align2 = COPY [[SPLIT0]].sub2_sub3 {
-# CHECK-NEXT: internal [[SPLIT1]].sub0:av_512_align2 = COPY [[SPLIT0]].sub0
+# CHECK-NEXT: undef [[SPLIT1:%[0-9]+]].sub2_sub3:av_512_align2 = lr-split COPY [[SPLIT0]].sub2_sub3 {
+# CHECK-NEXT: internal [[SPLIT1]].sub0:av_512_align2 = lr-split COPY [[SPLIT0]].sub0
# CHECK-NEXT: }
-# CHECK-NEXT: undef [[SPLIT2:%[0-9]+]].sub2_sub3:av_512_align2 = COPY [[SPLIT1]].sub2_sub3 {
-# CHECK-NEXT: internal [[SPLIT2]].sub0:av_512_align2 = COPY [[SPLIT1]].sub0
+# CHECK-NEXT: undef [[SPLIT2:%[0-9]+]].sub2_sub3:av_512_align2 = lr-split COPY [[SPLIT1]].sub2_sub3 {
+# CHECK-NEXT: internal [[SPLIT2]].sub0:av_512_align2 = lr-split COPY [[SPLIT1]].sub0
# CHECK-NEXT: }
# CHECK-NEXT: SI_SPILL_AV512_SAVE [[SPLIT2]], %stack.1, $sgpr32, 0, implicit $exec :: (store (s512) into %stack.1, align 4, addrspace 5)
# CHECK-NEXT: [[RESTORE_1:%[0-9]+]]:av_512_align2 = SI_SPILL_AV512_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s512) from %stack.0, align 4, addrspace 5)
-# CHECK-NEXT: undef [[SPLIT3:%[0-9]+]].sub0_sub1:av_512_align2 = COPY [[RESTORE_1]].sub0_sub1
+# CHECK-NEXT: undef [[SPLIT3:%[0-9]+]].sub0_sub1:av_512_align2 = lr-split COPY [[RESTORE_1]].sub0_sub1
# CHECK-NEXT: [[RESTORE_2:%[0-9]+]]:av_512_align2 = SI_SPILL_AV512_RESTORE %stack.1, $sgpr32, 0, implicit $exec :: (load (s512) from %stack.1, align 4, addrspace 5)
-# CHECK-NEXT: undef [[SPLIT4:%[0-9]+]].sub2_sub3:av_512_align2 = COPY [[RESTORE_2]].sub2_sub3 {
-# CHECK-NEXT: internal [[SPLIT4]].sub0:av_512_align2 = COPY [[RESTORE_2]].sub0
+# CHECK-NEXT: undef [[SPLIT4:%[0-9]+]].sub2_sub3:av_512_align2 = lr-split COPY [[RESTORE_2]].sub2_sub3 {
+# CHECK-NEXT: internal [[SPLIT4]].sub0:av_512_align2 = lr-split COPY [[RESTORE_2]].sub0
# CHECK-NEXT: }
-# CHECK-NEXT: undef [[SPLIT5:%[0-9]+]].sub2_sub3:av_512_align2 = COPY [[SPLIT4]].sub2_sub3 {
-# CHECK-NEXT: internal [[SPLIT5]].sub0:av_512_align2 = COPY [[SPLIT4]].sub0
+# CHECK-NEXT: undef [[SPLIT5:%[0-9]+]].sub2_sub3:av_512_align2 = lr-split COPY [[SPLIT4]].sub2_sub3 {
+# CHECK-NEXT: internal [[SPLIT5]].sub0:av_512_align2 = lr-split COPY [[SPLIT4]].sub0
# CHECK-NEXT: }
# CHECK-NEXT: [[SPLIT3]].sub2:av_512_align2 = COPY [[SPLIT5]].sub3
-# CHECK-NEXT: undef [[SPLIT6:%[0-9]+]].sub0_sub1_sub2:av_512_align2 = COPY [[SPLIT3]].sub0_sub1_sub2
-# CHECK-NEXT: undef [[SPLIT7:%[0-9]+]].sub0_sub1_sub2:av_512_align2 = COPY [[SPLIT6]].sub0_sub1_sub2
-# CHECK-NEXT: undef [[SPLIT8:%[0-9]+]].sub0:av_512_align2 = COPY [[SPLIT5]].sub0 {
-# CHECK-NEXT: internal [[SPLIT8]].sub2:av_512_align2 = COPY [[SPLIT5]].sub2
+# CHECK-NEXT: undef [[SPLIT6:%[0-9]+]].sub0_sub1_sub2:av_512_align2 = lr-split COPY [[SPLIT3]].sub0_sub1_sub2
+# CHECK-NEXT: undef [[SPLIT7:%[0-9]+]].sub0_sub1_sub2:av_512_align2 = lr-split COPY [[SPLIT6]].sub0_sub1_sub2
+# CHECK-NEXT: undef [[SPLIT8:%[0-9]+]].sub0:av_512_align2 = lr-split COPY [[SPLIT5]].sub0 {
+# CHECK-NEXT: internal [[SPLIT8]].sub2:av_512_align2 = lr-split COPY [[SPLIT5]].sub2
# CHECK-NEXT: }
# CHECK-NEXT: [[SPLIT7]].sub3:av_512_align2 = COPY [[SPLIT8]].sub2
-# CHECK-NEXT: undef [[SPLIT9:%[0-9]+]].sub0_sub1_sub2_sub3:av_512_align2 = COPY [[SPLIT7]].sub0_sub1_sub2_sub3
-# CHECK-NEXT: undef [[LAST_USE:%[0-9]+]].sub0_sub1_sub2_sub3:vreg_512_align2 = COPY [[SPLIT9]].sub0_sub1_sub2_sub3
+# CHECK-NEXT: undef [[SPLIT9:%[0-9]+]].sub0_sub1_sub2_sub3:av_512_align2 = lr-split COPY [[SPLIT7]].sub0_sub1_sub2_sub3
+# CHECK-NEXT: undef [[LAST_USE:%[0-9]+]].sub0_sub1_sub2_sub3:vreg_512_align2 = lr-split COPY [[SPLIT9]].sub0_sub1_sub2_sub3
# CHECK-NEXT: [[LAST_USE]].sub4:vreg_512_align2 = COPY [[SPLIT8]].sub0
# CHECK-NEXT: [[LAST_USE]].sub5:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
# CHECK-NEXT: [[LAST_USE]].sub6:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
index 215200c770245..c2798d0616a46 100644
--- a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
+++ b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
@@ -66,10 +66,10 @@ body: |
; CHECK-NEXT: bb.5:
; CHECK-NEXT: liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x00000000FFFFFFFF
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27 = COPY killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+ ; CHECK-NEXT: renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27 = lr-split COPY killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = IMPLICIT_DEF
; CHECK-NEXT: dead [[IMAGE_SAMPLE_LZ_V1_V2_4:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], killed renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, undef renamable $sgpr24_sgpr25_sgpr26_sgpr27, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = COPY killed renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27
+ ; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = lr-split COPY killed renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27
; CHECK-NEXT: S_BRANCH %bb.7
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.6:
diff --git a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
index b8818c5550ad4..8dff55e1c5f08 100644
--- a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
+++ b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
@@ -63,10 +63,10 @@ body: |
; CHECK-NEXT: bb.4:
; CHECK-NEXT: liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x00000000FFFFFFFF
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27 = COPY killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+ ; CHECK-NEXT: renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27 = lr-split COPY killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = IMPLICIT_DEF
; CHECK-NEXT: dead [[IMAGE_SAMPLE_LZ_V1_V2_4:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], killed renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, undef renamable $sgpr24_sgpr25_sgpr26_sgpr27, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = COPY killed renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27
+ ; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = lr-split COPY killed renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27
; CHECK-NEXT: S_BRANCH %bb.6
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.5:
diff --git a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
index 0f940c8da3915..a23d5dac49011 100644
--- a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
@@ -13,10 +13,10 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
; REGALLOC-GFX908-NEXT: {{ $}}
; REGALLOC-GFX908-NEXT: INLINEASM &"; use $0", sideeffect attdialect, reguse:AGPR_32, undef %6:agpr_32
; REGALLOC-GFX908-NEXT: INLINEASM &"; def $0", sideeffect attdialect, regdef:VReg_128, def %25
- ; REGALLOC-GFX908-NEXT: [[COPY:%[0-9]+]]:av_128 = COPY %25
+ ; REGALLOC-GFX908-NEXT: [[COPY:%[0-9]+]]:av_128 = lr-split COPY %25
; REGALLOC-GFX908-NEXT: INLINEASM &"; def $0", sideeffect attdialect, regdef:VReg_64, def %27
; REGALLOC-GFX908-NEXT: SI_SPILL_AV64_SAVE %27, %stack.0, $sgpr32, 0, implicit $exec :: (store (s64) into %stack.0, align 4, addrspace 5)
- ; REGALLOC-GFX908-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY [[COPY]]
+ ; REGALLOC-GFX908-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = lr-split COPY [[COPY]]
; REGALLOC-GFX908-NEXT: GLOBAL_STORE_DWORDX4 undef %15:vreg_64, [[COPY1]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
; REGALLOC-GFX908-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
; REGALLOC-GFX908-NEXT: [[COPY2:%[0-9]+]]:areg_128 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3
@@ -38,11 +38,11 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
; PEI-GFX908-NEXT: $sgpr13 = S_ADDC_U32 $sgpr13, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr12_sgpr13_sgpr14_sgpr15
; PEI-GFX908-NEXT: INLINEASM &"; use $0", sideeffect attdialect, reguse:AGPR_32, undef renamable $agpr0
; PEI-GFX908-NEXT: INLINEASM &"; def $0", sideeffect attdialect, regdef:VReg_128, def renamable $vgpr0_vgpr1_vgpr2_vgpr3
- ; PEI-GFX908-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit $exec
+ ; PEI-GFX908-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = lr-split COPY killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit $exec
; PEI-GFX908-NEXT: INLINEASM &"; def $0", sideeffect attdialect, regdef:VReg_64, def renamable $vgpr0_vgpr1
; PEI-GFX908-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $vgpr0_vgpr1 :: ("amdgpu-thread-private" store (s32) into %stack.0, addrspace 5)
; PEI-GFX908-NEXT: $agpr4 = V_ACCVGPR_WRITE_B32_e64 killed $vgpr1, implicit $exec, implicit killed $vgpr0_vgpr1
- ; PEI-GFX908-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = COPY killed renamable $agpr0_agpr1_agpr2_agpr3, implicit $exec
+ ; PEI-GFX908-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = lr-split COPY killed renamable $agpr0_agpr1_agpr2_agpr3, implicit $exec
; PEI-GFX908-NEXT: GLOBAL_STORE_DWORDX4 undef renamable $vgpr0_vgpr1, killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
; PEI-GFX908-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
; PEI-GFX908-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec
@@ -62,9 +62,9 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
; REGALLOC-GFX90A-NEXT: {{ $}}
; REGALLOC-GFX90A-NEXT: INLINEASM &"; use $0", sideeffect attdialect, reguse:AGPR_32, undef %6:agpr_32
; REGALLOC-GFX90A-NEXT: INLINEASM &"; def $0", sideeffect attdialect, regdef:VReg_128_Align2, def %23
- ; REGALLOC-GFX90A-NEXT: [[COPY:%[0-9]+]]:av_128_align2 = COPY %23
+ ; REGALLOC-GFX90A-NEXT: [[COPY:%[0-9]+]]:av_128_align2 = lr-split COPY %23
; REGALLOC-GFX90A-NEXT: INLINEASM &"; def $0", sideeffect attdialect, regdef:VReg_64_Align2, def %21
- ; REGALLOC-GFX90A-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY %21
+ ; REGALLOC-GFX90A-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = lr-split COPY %21
; REGALLOC-GFX90A-NEXT: GLOBAL_STORE_DWORDX4 undef %15:vreg_64_align2, [[COPY]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
; REGALLOC-GFX90A-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
; REGALLOC-GFX90A-NEXT: [[COPY2:%[0-9]+]]:areg_128_align2 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3
@@ -81,7 +81,7 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
; PEI-GFX90A-NEXT: {{ $}}
; PEI-GFX90A-NEXT: INLINEASM &"; use $0", sideeffect attdialect, reguse:AGPR_32, undef renamable $agpr0
; PEI-GFX90A-NEXT: INLINEASM &"; def $0", sideeffect attdialect, regdef:VReg_128_Align2, def renamable $vgpr0_vgpr1_vgpr2_vgpr3
- ; PEI-GFX90A-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit $exec
+ ; PEI-GFX90A-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = lr-split COPY killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit $exec
; PEI-GFX90A-NEXT: INLINEASM &"; def $0", sideeffect attdialect, regdef:VReg_64_Align2, def renamable $vgpr2_vgpr3
; PEI-GFX90A-NEXT: GLOBAL_STORE_DWORDX4 undef renamable $vgpr0_vgpr1, killed renamable $agpr0_agpr1_agpr2_agpr3, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
; PEI-GFX90A-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
diff --git a/llvm/test/CodeGen/AMDGPU/ra-inserted-scalar-instructions.mir b/llvm/test/CodeGen/AMDGPU/ra-inserted-scalar-instructions.mir
index d406f2932dc96..a781cfcd3f9ec 100644
--- a/llvm/test/CodeGen/AMDGPU/ra-inserted-scalar-instructions.mir
+++ b/llvm/test/CodeGen/AMDGPU/ra-inserted-scalar-instructions.mir
@@ -222,12 +222,12 @@ body: |
; GCN-NEXT: [[S_LOAD_DWORD_IMM3:%[0-9]+]]:sgpr_32 = S_LOAD_DWORD_IMM [[SI_SPILL_S64_RESTORE]], 0, 0 :: ("amdgpu-noclobber" load (s32), align 8, addrspace 1)
; GCN-NEXT: SI_SPILL_S32_SAVE [[S_LOAD_DWORD_IMM3]], %stack.7, implicit $exec, implicit $sgpr32 :: (store (s32) into %stack.7, addrspace 5)
; GCN-NEXT: SI_SPILL_S64_SAVE [[V_READFIRSTLANE_B32_]], %stack.0, implicit $exec, implicit $sgpr32 :: (store (s64) into %stack.0, align 4, addrspace 5)
- ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_64 = COPY [[V_READFIRSTLANE_B32_]]
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_64 = lr-split COPY [[V_READFIRSTLANE_B32_]]
; GCN-NEXT: dead [[S_LOAD_DWORD_IMM4:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY1]], 0, 0 :: ("amdgpu-noclobber" load (s32), addrspace 1)
; GCN-NEXT: [[S_MOV_B64_1:%[0-9]+]]:sreg_64 = S_MOV_B64 0
; GCN-NEXT: [[S_LOAD_DWORD_IMM5:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[S_MOV_B64_1]], 0, 0 :: ("amdgpu-noclobber" load (s32), addrspace 1)
; GCN-NEXT: [[SI_SPILL_S64_RESTORE1:%[0-9]+]]:sgpr_64 = SI_SPILL_S64_RESTORE %stack.2, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.2, align 4, addrspace 5)
- ; GCN-NEXT: undef [[COPY2:%[0-9]+]].sub1:sgpr_64 = COPY [[SI_SPILL_S64_RESTORE1]].sub1
+ ; GCN-NEXT: undef [[COPY2:%[0-9]+]].sub1:sgpr_64 = lr-split COPY [[SI_SPILL_S64_RESTORE1]].sub1
; GCN-NEXT: [[COPY2:%[0-9]+]].sub0:sgpr_64 = S_MOV_B32 1
; GCN-NEXT: S_CBRANCH_SCC1 %bb.10, implicit undef $scc
; GCN-NEXT: S_BRANCH %bb.7
@@ -241,7 +241,7 @@ body: |
; GCN-NEXT: [[DEF3:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
; GCN-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
; GCN-NEXT: $vcc = COPY [[DEF3]]
- ; GCN-NEXT: S_CBRANCH_VCCNZ %bb.9, implicit $vcc
+ ; GCN-NEXT: S_CBRANCH_VCCNZ %bb.9, implicit $vcc_lo
; GCN-NEXT: S_BRANCH %bb.8
; GCN-NEXT: {{ $}}
; GCN-NEXT: bb.8:
@@ -252,7 +252,7 @@ body: |
; GCN-NEXT: bb.9:
; GCN-NEXT: successors: %bb.10(0x80000000)
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sgpr_32 = COPY [[S_MOV_B32_4]]
+ ; GCN-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sgpr_32 = lr-split COPY [[S_MOV_B32_4]]
; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_64 = SI_SPILL_S64_RESTORE %stack.2, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.2, align 4, addrspace 5)
; GCN-NEXT: {{ $}}
; GCN-NEXT: bb.10:
@@ -268,10 +268,10 @@ body: |
; GCN-NEXT: S_CMP_EQ_U32 [[SI_SPILL_S64_RESTORE2]].sub1, 0, implicit-def $scc
; GCN-NEXT: dead [[DEF5:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
; GCN-NEXT: [[SI_SPILL_S256_RESTORE:%[0-9]+]]:sgpr_256 = SI_SPILL_S256_RESTORE %stack.20, implicit $exec, implicit $sgpr32 :: (load (s256) from %stack.20, align 4, addrspace 5)
- ; GCN-NEXT: undef [[COPY3:%[0-9]+]].sub0:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE]].sub0 {
- ; GCN-NEXT: internal [[COPY3]].sub2:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE]].sub2
- ; GCN-NEXT: internal [[COPY3]].sub4:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE]].sub4
- ; GCN-NEXT: internal [[COPY3]].sub7:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE]].sub7
+ ; GCN-NEXT: undef [[COPY3:%[0-9]+]].sub0:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE]].sub0 {
+ ; GCN-NEXT: internal [[COPY3]].sub2:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE]].sub2
+ ; GCN-NEXT: internal [[COPY3]].sub4:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE]].sub4
+ ; GCN-NEXT: internal [[COPY3]].sub7:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE]].sub7
; GCN-NEXT: }
; GCN-NEXT: dead [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 [[COPY3]].sub7, [[S_LOAD_DWORD_IMM5]], implicit-def dead $scc
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_1:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY3]].sub0, 0, implicit $mode, implicit $exec
@@ -285,7 +285,7 @@ body: |
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_8:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[S_LOAD_DWORDX8_IMM]].sub5, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_9:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[S_LOAD_DWORDX8_IMM]].sub6, 0, implicit $mode, implicit $exec
; GCN-NEXT: [[SI_SPILL_S128_RESTORE:%[0-9]+]]:sgpr_128 = SI_SPILL_S128_RESTORE %stack.14, implicit $exec, implicit $sgpr32 :: (load (s128) from %stack.14, align 4, addrspace 5)
- ; GCN-NEXT: undef [[COPY4:%[0-9]+]].sub0_sub1_sub2:sgpr_128 = COPY [[SI_SPILL_S128_RESTORE]].sub0_sub1_sub2
+ ; GCN-NEXT: undef [[COPY4:%[0-9]+]].sub0_sub1_sub2:sgpr_128 = lr-split COPY [[SI_SPILL_S128_RESTORE]].sub0_sub1_sub2
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_10:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY4]].sub0, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_11:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY4]].sub1, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_12:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY4]].sub2, 0, implicit $mode, implicit $exec
@@ -298,17 +298,17 @@ body: |
; GCN-NEXT: dead [[DEF11:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
; GCN-NEXT: dead [[S_AND_B32_2:%[0-9]+]]:sreg_32 = S_AND_B32 undef [[DEF11]], undef [[DEF11]], implicit-def dead $scc
; GCN-NEXT: [[SI_SPILL_S256_RESTORE1:%[0-9]+]]:sgpr_256 = SI_SPILL_S256_RESTORE %stack.16, implicit $exec, implicit $sgpr32 :: (load (s256) from %stack.16, align 4, addrspace 5)
- ; GCN-NEXT: undef [[COPY5:%[0-9]+]].sub0:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE1]].sub0 {
- ; GCN-NEXT: internal [[COPY5]].sub2:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE1]].sub2
- ; GCN-NEXT: internal [[COPY5]].sub5:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE1]].sub5
- ; GCN-NEXT: internal [[COPY5]].sub7:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE1]].sub7
+ ; GCN-NEXT: undef [[COPY5:%[0-9]+]].sub0:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE1]].sub0 {
+ ; GCN-NEXT: internal [[COPY5]].sub2:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE1]].sub2
+ ; GCN-NEXT: internal [[COPY5]].sub5:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE1]].sub5
+ ; GCN-NEXT: internal [[COPY5]].sub7:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE1]].sub7
; GCN-NEXT: }
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_13:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY5]].sub0, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[S_AND_B32_3:%[0-9]+]]:sreg_32 = S_AND_B32 undef [[V_CMP_GT_F32_e64_8]], undef [[V_CMP_GT_F32_e64_9]], implicit-def dead $scc
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_14:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY5]].sub2, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[S_OR_B32_2:%[0-9]+]]:sreg_32 = S_OR_B32 [[COPY5]].sub5, [[COPY5]].sub7, implicit-def dead $scc
; GCN-NEXT: [[SI_SPILL_S256_RESTORE2:%[0-9]+]]:sgpr_256 = SI_SPILL_S256_RESTORE %stack.10, implicit $exec, implicit $sgpr32 :: (load (s256) from %stack.10, align 4, addrspace 5)
- ; GCN-NEXT: undef [[COPY6:%[0-9]+]].lo16_hi16_sub1_lo16_sub1_hi16_sub2_lo16_sub2_hi16_sub3_lo16_sub3_hi16_sub4_lo16_sub4_hi16_sub5_lo16_sub5_hi16_sub6_lo16_sub6_hi16:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE2]].lo16_hi16_sub1_lo16_sub1_hi16_sub2_lo16_sub2_hi16_sub3_lo16_sub3_hi16_sub4_lo16_sub4_hi16_sub5_lo16_sub5_hi16_sub6_lo16_sub6_hi16
+ ; GCN-NEXT: undef [[COPY6:%[0-9]+]].lo16_hi16_sub1_lo16_sub1_hi16_sub2_lo16_sub2_hi16_sub3_lo16_sub3_hi16_sub4_lo16_sub4_hi16_sub5_lo16_sub5_hi16_sub6_lo16_sub6_hi16:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE2]].lo16_hi16_sub1_lo16_sub1_hi16_sub2_lo16_sub2_hi16_sub3_lo16_sub3_hi16_sub4_lo16_sub4_hi16_sub5_lo16_sub5_hi16_sub6_lo16_sub6_hi16
; GCN-NEXT: dead [[S_OR_B32_3:%[0-9]+]]:sreg_32 = S_OR_B32 [[COPY6]].sub0, [[COPY6]].sub1, implicit-def dead $scc
; GCN-NEXT: dead [[S_OR_B32_4:%[0-9]+]]:sreg_32 = S_OR_B32 [[COPY6]].sub2, undef [[S_OR_B32_3]], implicit-def dead $scc
; GCN-NEXT: [[SI_SPILL_S32_RESTORE2:%[0-9]+]]:sreg_32_xm0_xexec = SI_SPILL_S32_RESTORE %stack.9, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.9, addrspace 5)
@@ -320,10 +320,10 @@ body: |
; GCN-NEXT: [[SI_SPILL_S32_RESTORE3:%[0-9]+]]:sreg_32_xm0_xexec = SI_SPILL_S32_RESTORE %stack.11, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.11, addrspace 5)
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_19:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[SI_SPILL_S32_RESTORE3]], 0, implicit $mode, implicit $exec
; GCN-NEXT: [[SI_SPILL_S256_RESTORE3:%[0-9]+]]:sgpr_256 = SI_SPILL_S256_RESTORE %stack.13, implicit $exec, implicit $sgpr32 :: (load (s256) from %stack.13, align 4, addrspace 5)
- ; GCN-NEXT: undef [[COPY7:%[0-9]+]].sub0:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE3]].sub0 {
- ; GCN-NEXT: internal [[COPY7]].sub2:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE3]].sub2
- ; GCN-NEXT: internal [[COPY7]].sub4:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE3]].sub4
- ; GCN-NEXT: internal [[COPY7]].sub7:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE3]].sub7
+ ; GCN-NEXT: undef [[COPY7:%[0-9]+]].sub0:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE3]].sub0 {
+ ; GCN-NEXT: internal [[COPY7]].sub2:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE3]].sub2
+ ; GCN-NEXT: internal [[COPY7]].sub4:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE3]].sub4
+ ; GCN-NEXT: internal [[COPY7]].sub7:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE3]].sub7
; GCN-NEXT: }
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_20:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY7]].sub0, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_21:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY7]].sub2, 0, implicit $mode, implicit $exec
@@ -331,14 +331,14 @@ body: |
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_22:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY7]].sub4, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[S_AND_B32_5:%[0-9]+]]:sreg_32 = S_AND_B32 undef [[DEF12]], undef [[V_CMP_GT_F32_e64_20]], implicit-def dead $scc
; GCN-NEXT: S_CMP_EQ_U32 [[COPY7]].sub7, 0, implicit-def $scc
- ; GCN-NEXT: undef [[COPY8:%[0-9]+]].sub0:sgpr_512 = COPY [[S_LOAD_DWORDX16_IMM]].sub0 {
- ; GCN-NEXT: internal [[COPY8]].sub2:sgpr_512 = COPY [[S_LOAD_DWORDX16_IMM]].sub2
- ; GCN-NEXT: internal [[COPY8]].sub4:sgpr_512 = COPY [[S_LOAD_DWORDX16_IMM]].sub4
- ; GCN-NEXT: internal [[COPY8]].sub6:sgpr_512 = COPY [[S_LOAD_DWORDX16_IMM]].sub6
- ; GCN-NEXT: internal [[COPY8]].sub9:sgpr_512 = COPY [[S_LOAD_DWORDX16_IMM]].sub9
- ; GCN-NEXT: internal [[COPY8]].sub10:sgpr_512 = COPY [[S_LOAD_DWORDX16_IMM]].sub10
- ; GCN-NEXT: internal [[COPY8]].sub13:sgpr_512 = COPY [[S_LOAD_DWORDX16_IMM]].sub13
- ; GCN-NEXT: internal [[COPY8]].sub14:sgpr_512 = COPY [[S_LOAD_DWORDX16_IMM]].sub14
+ ; GCN-NEXT: undef [[COPY8:%[0-9]+]].sub0:sgpr_512 = lr-split COPY [[S_LOAD_DWORDX16_IMM]].sub0 {
+ ; GCN-NEXT: internal [[COPY8]].sub2:sgpr_512 = lr-split COPY [[S_LOAD_DWORDX16_IMM]].sub2
+ ; GCN-NEXT: internal [[COPY8]].sub4:sgpr_512 = lr-split COPY [[S_LOAD_DWORDX16_IMM]].sub4
+ ; GCN-NEXT: internal [[COPY8]].sub6:sgpr_512 = lr-split COPY [[S_LOAD_DWORDX16_IMM]].sub6
+ ; GCN-NEXT: internal [[COPY8]].sub9:sgpr_512 = lr-split COPY [[S_LOAD_DWORDX16_IMM]].sub9
+ ; GCN-NEXT: internal [[COPY8]].sub10:sgpr_512 = lr-split COPY [[S_LOAD_DWORDX16_IMM]].sub10
+ ; GCN-NEXT: internal [[COPY8]].sub13:sgpr_512 = lr-split COPY [[S_LOAD_DWORDX16_IMM]].sub13
+ ; GCN-NEXT: internal [[COPY8]].sub14:sgpr_512 = lr-split COPY [[S_LOAD_DWORDX16_IMM]].sub14
; GCN-NEXT: }
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_23:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY8]].sub0, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_24:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY8]].sub2, 0, implicit $mode, implicit $exec
@@ -349,13 +349,13 @@ body: |
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_27:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY8]].sub13, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_28:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY8]].sub14, 0, implicit $mode, implicit $exec
; GCN-NEXT: [[SI_SPILL_S512_RESTORE:%[0-9]+]]:sgpr_512 = SI_SPILL_S512_RESTORE %stack.12, implicit $exec, implicit $sgpr32 :: (load (s512) from %stack.12, align 4, addrspace 5)
- ; GCN-NEXT: undef [[COPY9:%[0-9]+]].sub1:sgpr_512 = COPY [[SI_SPILL_S512_RESTORE]].sub1 {
- ; GCN-NEXT: internal [[COPY9]].sub5:sgpr_512 = COPY [[SI_SPILL_S512_RESTORE]].sub5
- ; GCN-NEXT: internal [[COPY9]].sub6:sgpr_512 = COPY [[SI_SPILL_S512_RESTORE]].sub6
- ; GCN-NEXT: internal [[COPY9]].sub9:sgpr_512 = COPY [[SI_SPILL_S512_RESTORE]].sub9
- ; GCN-NEXT: internal [[COPY9]].sub10:sgpr_512 = COPY [[SI_SPILL_S512_RESTORE]].sub10
- ; GCN-NEXT: internal [[COPY9]].sub12:sgpr_512 = COPY [[SI_SPILL_S512_RESTORE]].sub12
- ; GCN-NEXT: internal [[COPY9]].sub15:sgpr_512 = COPY [[SI_SPILL_S512_RESTORE]].sub15
+ ; GCN-NEXT: undef [[COPY9:%[0-9]+]].sub1:sgpr_512 = lr-split COPY [[SI_SPILL_S512_RESTORE]].sub1 {
+ ; GCN-NEXT: internal [[COPY9]].sub5:sgpr_512 = lr-split COPY [[SI_SPILL_S512_RESTORE]].sub5
+ ; GCN-NEXT: internal [[COPY9]].sub6:sgpr_512 = lr-split COPY [[SI_SPILL_S512_RESTORE]].sub6
+ ; GCN-NEXT: internal [[COPY9]].sub9:sgpr_512 = lr-split COPY [[SI_SPILL_S512_RESTORE]].sub9
+ ; GCN-NEXT: internal [[COPY9]].sub10:sgpr_512 = lr-split COPY [[SI_SPILL_S512_RESTORE]].sub10
+ ; GCN-NEXT: internal [[COPY9]].sub12:sgpr_512 = lr-split COPY [[SI_SPILL_S512_RESTORE]].sub12
+ ; GCN-NEXT: internal [[COPY9]].sub15:sgpr_512 = lr-split COPY [[SI_SPILL_S512_RESTORE]].sub15
; GCN-NEXT: }
; GCN-NEXT: S_CMP_EQ_U32 [[COPY9]].sub1, 0, implicit-def $scc
; GCN-NEXT: dead [[DEF13:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
@@ -368,7 +368,7 @@ body: |
; GCN-NEXT: dead [[S_AND_B32_7:%[0-9]+]]:sreg_32 = S_AND_B32 undef [[DEF15]], undef [[DEF14]], implicit-def dead $scc
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_33:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY9]].sub12, 0, implicit $mode, implicit $exec
; GCN-NEXT: [[SI_SPILL_S256_RESTORE4:%[0-9]+]]:sgpr_256 = SI_SPILL_S256_RESTORE %stack.6, implicit $exec, implicit $sgpr32 :: (load (s256) from %stack.6, align 4, addrspace 5)
- ; GCN-NEXT: undef [[COPY10:%[0-9]+]].lo16_hi16_sub1_lo16_sub1_hi16_sub2_lo16_sub2_hi16_sub3_lo16_sub3_hi16_sub4_lo16_sub4_hi16_sub5_lo16_sub5_hi16_sub6_lo16_sub6_hi16:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE4]].lo16_hi16_sub1_lo16_sub1_hi16_sub2_lo16_sub2_hi16_sub3_lo16_sub3_hi16_sub4_lo16_sub4_hi16_sub5_lo16_sub5_hi16_sub6_lo16_sub6_hi16
+ ; GCN-NEXT: undef [[COPY10:%[0-9]+]].lo16_hi16_sub1_lo16_sub1_hi16_sub2_lo16_sub2_hi16_sub3_lo16_sub3_hi16_sub4_lo16_sub4_hi16_sub5_lo16_sub5_hi16_sub6_lo16_sub6_hi16:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE4]].lo16_hi16_sub1_lo16_sub1_hi16_sub2_lo16_sub2_hi16_sub3_lo16_sub3_hi16_sub4_lo16_sub4_hi16_sub5_lo16_sub5_hi16_sub6_lo16_sub6_hi16
; GCN-NEXT: dead [[S_OR_B32_6:%[0-9]+]]:sreg_32 = S_OR_B32 [[COPY10]].sub0, [[COPY9]].sub15, implicit-def dead $scc
; GCN-NEXT: dead [[DEF16:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_34:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY10]].sub1, 0, implicit $mode, implicit $exec
@@ -381,7 +381,7 @@ body: |
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_39:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY10]].sub6, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[S_AND_B32_8:%[0-9]+]]:sreg_32 = S_AND_B32 undef [[DEF18]], undef [[DEF17]], implicit-def dead $scc
; GCN-NEXT: [[SI_SPILL_S256_RESTORE5:%[0-9]+]]:sgpr_256 = SI_SPILL_S256_RESTORE %stack.4, implicit $exec, implicit $sgpr32 :: (load (s256) from %stack.4, align 4, addrspace 5)
- ; GCN-NEXT: undef [[COPY11:%[0-9]+]].sub0_sub1_sub2_sub3_sub4_sub5:sgpr_256 = COPY [[SI_SPILL_S256_RESTORE5]].sub0_sub1_sub2_sub3_sub4_sub5
+ ; GCN-NEXT: undef [[COPY11:%[0-9]+]].sub0_sub1_sub2_sub3_sub4_sub5:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE5]].sub0_sub1_sub2_sub3_sub4_sub5
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_40:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY11]].sub0, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_41:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY11]].sub1, 0, implicit $mode, implicit $exec
; GCN-NEXT: [[SI_SPILL_S32_RESTORE4:%[0-9]+]]:sreg_32_xm0_xexec = SI_SPILL_S32_RESTORE %stack.3, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.3, addrspace 5)
@@ -401,7 +401,7 @@ body: |
; GCN-NEXT: dead [[S_AND_B32_10:%[0-9]+]]:sreg_32 = S_AND_B32 [[DEF19]], undef [[S_LOAD_DWORD_IMM6]], implicit-def dead $scc
; GCN-NEXT: dead [[S_AND_B32_11:%[0-9]+]]:sreg_32 = S_AND_B32 undef [[S_AND_B32_10]], [[S_MOV_B32_2]], implicit-def dead $scc
; GCN-NEXT: $vcc = COPY undef [[S_AND_B32_11]]
- ; GCN-NEXT: S_CBRANCH_VCCNZ %bb.12, implicit $vcc
+ ; GCN-NEXT: S_CBRANCH_VCCNZ %bb.12, implicit $vcc_lo
; GCN-NEXT: S_BRANCH %bb.11
; GCN-NEXT: {{ $}}
; GCN-NEXT: bb.11:
diff --git a/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir b/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir
index e7eefafe31203..273cc7a50085a 100644
--- a/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir
+++ b/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir
@@ -84,24 +84,24 @@ body: |
; CHECK-NEXT: renamable $sgpr49 = COPY renamable $sgpr72
; CHECK-NEXT: renamable $sgpr50 = COPY renamable $sgpr72
; CHECK-NEXT: renamable $sgpr51 = COPY killed renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 = COPY killed renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+ ; CHECK-NEXT: renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 = lr-split COPY killed renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
; CHECK-NEXT: renamable $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87 = SI_SPILL_S1024_RESTORE %stack.2, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
; CHECK-NEXT: renamable $sgpr56 = COPY killed renamable $sgpr72
; CHECK-NEXT: renamable $sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91 = SI_SPILL_S1024_RESTORE %stack.2, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = COPY killed renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55
- ; CHECK-NEXT: renamable $sgpr52 = COPY renamable $sgpr56
+ ; CHECK-NEXT: renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = lr-split COPY killed renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55
+ ; CHECK-NEXT: renamable $sgpr52 = lr-split COPY renamable $sgpr56
; CHECK-NEXT: renamable $sgpr53 = COPY killed renamable $sgpr76
- ; CHECK-NEXT: renamable $sgpr56_sgpr57 = COPY renamable $sgpr52_sgpr53
- ; CHECK-NEXT: renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 = COPY killed renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+ ; CHECK-NEXT: renamable $sgpr56_sgpr57 = lr-split COPY renamable $sgpr52_sgpr53
+ ; CHECK-NEXT: renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 = lr-split COPY killed renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
; CHECK-NEXT: renamable $sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91 = SI_SPILL_S1024_RESTORE %stack.2, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = COPY killed renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55
- ; CHECK-NEXT: renamable $sgpr52_sgpr53 = COPY renamable $sgpr56_sgpr57
+ ; CHECK-NEXT: renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = lr-split COPY killed renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55
+ ; CHECK-NEXT: renamable $sgpr52_sgpr53 = lr-split COPY renamable $sgpr56_sgpr57
; CHECK-NEXT: renamable $sgpr54 = COPY killed renamable $sgpr76
- ; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47 = COPY killed renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
- ; CHECK-NEXT: renamable $sgpr48_sgpr49_sgpr50 = COPY renamable $sgpr52_sgpr53_sgpr54
+ ; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47 = lr-split COPY killed renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+ ; CHECK-NEXT: renamable $sgpr48_sgpr49_sgpr50 = lr-split COPY renamable $sgpr52_sgpr53_sgpr54
; CHECK-NEXT: renamable $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83 = SI_SPILL_S1024_RESTORE %stack.2, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr52_sgpr53_sgpr54 = COPY renamable $sgpr48_sgpr49_sgpr50
- ; CHECK-NEXT: renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = COPY killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47
+ ; CHECK-NEXT: renamable $sgpr52_sgpr53_sgpr54 = lr-split COPY renamable $sgpr48_sgpr49_sgpr50
+ ; CHECK-NEXT: renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = lr-split COPY killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47
; CHECK-NEXT: renamable $sgpr55 = COPY killed renamable $sgpr68
; CHECK-NEXT: renamable $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87 = SI_SPILL_S1024_RESTORE %stack.2, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
; CHECK-NEXT: renamable $sgpr56 = COPY killed renamable $sgpr72
@@ -129,9 +129,9 @@ body: |
; CHECK-NEXT: renamable $sgpr67 = COPY killed renamable $sgpr84
; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
; CHECK-NEXT: renamable $sgpr68 = COPY killed renamable $sgpr84
- ; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = COPY killed renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55
- ; CHECK-NEXT: renamable $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63 = COPY renamable $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
- ; CHECK-NEXT: renamable $sgpr64 = COPY renamable $sgpr68
+ ; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = lr-split COPY killed renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55
+ ; CHECK-NEXT: renamable $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63 = lr-split COPY renamable $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+ ; CHECK-NEXT: renamable $sgpr64 = lr-split COPY renamable $sgpr68
; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
; CHECK-NEXT: renamable $sgpr65 = COPY killed renamable $sgpr84
; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
@@ -146,7 +146,7 @@ body: |
; CHECK-NEXT: successors: %bb.4(0x80000000)
; CHECK-NEXT: liveins: $sgpr16
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $sgpr60 = COPY killed renamable $sgpr16
+ ; CHECK-NEXT: renamable $sgpr60 = lr-split COPY killed renamable $sgpr16
; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; CHECK-NEXT: dead $sgpr30_sgpr31 = SI_CALL undef renamable $sgpr4_sgpr5, 0, CustomRegMask($sgpr60,$sgpr62)
; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
@@ -205,28 +205,28 @@ body: |
; CHECK-NEXT: dead renamable $sgpr12_sgpr13 = V_CMP_NE_U32_e64 1, [[V_CNDMASK_B32_e64_1]], implicit $exec
; CHECK-NEXT: renamable $sgpr82 = S_ADD_U32 renamable $sgpr8, 32, implicit-def dead $scc
; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
- ; CHECK-NEXT: renamable $sgpr68_sgpr69 = COPY killed renamable $sgpr4_sgpr5
+ ; CHECK-NEXT: renamable $sgpr68_sgpr69 = lr-split COPY killed renamable $sgpr4_sgpr5
; CHECK-NEXT: $sgpr4_sgpr5 = COPY renamable $sgpr68_sgpr69
- ; CHECK-NEXT: renamable $sgpr70_sgpr71 = COPY killed renamable $sgpr6_sgpr7
+ ; CHECK-NEXT: renamable $sgpr70_sgpr71 = lr-split COPY killed renamable $sgpr6_sgpr7
; CHECK-NEXT: $sgpr6_sgpr7 = COPY renamable $sgpr70_sgpr71
- ; CHECK-NEXT: renamable $sgpr80_sgpr81 = COPY killed renamable $sgpr10_sgpr11
+ ; CHECK-NEXT: renamable $sgpr80_sgpr81 = lr-split COPY killed renamable $sgpr10_sgpr11
; CHECK-NEXT: $sgpr10_sgpr11 = COPY renamable $sgpr80_sgpr81
; CHECK-NEXT: $sgpr12 = SI_SPILL_S32_RESTORE %stack.1, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.1, addrspace 5)
; CHECK-NEXT: $sgpr13 = SI_SPILL_S32_RESTORE %stack.0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.0, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr84 = COPY killed renamable $sgpr8
- ; CHECK-NEXT: renamable $sgpr33 = COPY killed renamable $sgpr16
- ; CHECK-NEXT: renamable $sgpr48_sgpr49 = COPY killed renamable $sgpr14_sgpr15
+ ; CHECK-NEXT: renamable $sgpr84 = lr-split COPY killed renamable $sgpr8
+ ; CHECK-NEXT: renamable $sgpr33 = lr-split COPY killed renamable $sgpr16
+ ; CHECK-NEXT: renamable $sgpr48_sgpr49 = lr-split COPY killed renamable $sgpr14_sgpr15
; CHECK-NEXT: dead $sgpr30_sgpr31 = SI_CALL undef renamable $sgpr12_sgpr13, 0, csr_amdgpu_gfx90ainsts, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13
; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; CHECK-NEXT: $sgpr8_sgpr9 = COPY renamable $sgpr82_sgpr83
; CHECK-NEXT: dead $sgpr30_sgpr31 = SI_CALL undef renamable $sgpr12_sgpr13, 0, csr_amdgpu_gfx90ainsts, implicit $sgpr8_sgpr9
- ; CHECK-NEXT: renamable $sgpr14_sgpr15 = COPY killed renamable $sgpr48_sgpr49
- ; CHECK-NEXT: renamable $sgpr16 = COPY killed renamable $sgpr33
- ; CHECK-NEXT: renamable $sgpr4_sgpr5 = COPY killed renamable $sgpr68_sgpr69
- ; CHECK-NEXT: renamable $sgpr6_sgpr7 = COPY killed renamable $sgpr70_sgpr71
- ; CHECK-NEXT: renamable $sgpr8 = COPY killed renamable $sgpr84
- ; CHECK-NEXT: renamable $sgpr10_sgpr11 = COPY killed renamable $sgpr80_sgpr81
+ ; CHECK-NEXT: renamable $sgpr14_sgpr15 = lr-split COPY killed renamable $sgpr48_sgpr49
+ ; CHECK-NEXT: renamable $sgpr16 = lr-split COPY killed renamable $sgpr33
+ ; CHECK-NEXT: renamable $sgpr4_sgpr5 = lr-split COPY killed renamable $sgpr68_sgpr69
+ ; CHECK-NEXT: renamable $sgpr6_sgpr7 = lr-split COPY killed renamable $sgpr70_sgpr71
+ ; CHECK-NEXT: renamable $sgpr8 = lr-split COPY killed renamable $sgpr84
+ ; CHECK-NEXT: renamable $sgpr10_sgpr11 = lr-split COPY killed renamable $sgpr80_sgpr81
; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; CHECK-NEXT: $exec = S_MOV_B64_term renamable $sgpr66_sgpr67
; CHECK-NEXT: S_CBRANCH_EXECZ %bb.10, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/regalloc-fail-unsatisfiable-overlapping-tuple-hints.mir b/llvm/test/CodeGen/AMDGPU/regalloc-fail-unsatisfiable-overlapping-tuple-hints.mir
index 09be927dc952e..92532b1820578 100644
--- a/llvm/test/CodeGen/AMDGPU/regalloc-fail-unsatisfiable-overlapping-tuple-hints.mir
+++ b/llvm/test/CodeGen/AMDGPU/regalloc-fail-unsatisfiable-overlapping-tuple-hints.mir
@@ -52,7 +52,7 @@ body: |
; CHECK-NEXT: [[SI_SPILL_V256_RESTORE:%[0-9]+]]:vreg_256 = SI_SPILL_V256_RESTORE %stack.1, $sgpr32, 0, implicit $exec :: (load (s256) from %stack.1, align 4, addrspace 5)
; CHECK-NEXT: [[SI_SPILL_V256_RESTORE1:%[0-9]+]]:vreg_256 = SI_SPILL_V256_RESTORE %stack.3, $sgpr32, 0, implicit $exec :: (load (s256) from %stack.3, align 4, addrspace 5)
; CHECK-NEXT: S_NOP 0, implicit [[SI_SPILL_V256_RESTORE]], implicit [[SI_SPILL_V256_RESTORE1]], implicit %4
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_256 = COPY [[SI_SPILL_V256_RESTORE1]]
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_256 = lr-split COPY [[SI_SPILL_V256_RESTORE1]]
; CHECK-NEXT: S_CBRANCH_EXECNZ %bb.2, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
diff --git a/llvm/test/CodeGen/AMDGPU/regalloc-undef-copy-fold.mir b/llvm/test/CodeGen/AMDGPU/regalloc-undef-copy-fold.mir
index ed8f897b36eb8..ed6053df12f93 100644
--- a/llvm/test/CodeGen/AMDGPU/regalloc-undef-copy-fold.mir
+++ b/llvm/test/CodeGen/AMDGPU/regalloc-undef-copy-fold.mir
@@ -44,7 +44,7 @@ body: |
; CHECK-NEXT: KILL undef $vgpr0_vgpr1_vgpr2_vgpr3
; CHECK-NEXT: SI_SPILL_AV160_SAVE %2, %stack.1, $sgpr32, 0, implicit $exec :: (store (s160) into %stack.1, align 4, addrspace 5)
; CHECK-NEXT: SI_SPILL_AV256_SAVE %1, %stack.3, $sgpr32, 0, implicit $exec :: (store (s256) into %stack.3, align 4, addrspace 5)
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_512 = COPY %10
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_512 = lr-split COPY %10
; CHECK-NEXT: SI_SPILL_V512_SAVE [[COPY]], %stack.0, $sgpr32, 0, implicit $exec :: (store (s512) into %stack.0, align 4, addrspace 5)
; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_512 = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
; CHECK-NEXT: SI_SPILL_AV512_SAVE [[COPY1]], %stack.6, $sgpr32, 0, implicit $exec :: (store (s512) into %stack.6, align 4, addrspace 5)
diff --git a/llvm/test/CodeGen/AMDGPU/split-liverange-overlapping-copies.mir b/llvm/test/CodeGen/AMDGPU/split-liverange-overlapping-copies.mir
index 9352216ccde7c..a90d20a8a161c 100644
--- a/llvm/test/CodeGen/AMDGPU/split-liverange-overlapping-copies.mir
+++ b/llvm/test/CodeGen/AMDGPU/split-liverange-overlapping-copies.mir
@@ -40,9 +40,9 @@ body: |
; CHECK-NEXT: bb.2:
; CHECK-NEXT: successors: %bb.3(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: undef [[COPY1:%[0-9]+]].sub1_sub2_sub3_sub4_sub5_sub6_sub7_sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15_sub16:av_1024_align2 = COPY [[COPY]].sub1_sub2_sub3_sub4_sub5_sub6_sub7_sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15_sub16 {
- ; CHECK-NEXT: internal [[COPY1]].sub20_lo16_sub20_hi16_sub21_lo16_sub21_hi16_sub22_lo16_sub22_hi16_sub23_lo16_sub23_hi16_sub24_lo16_sub24_hi16_sub25_lo16_sub25_hi16_sub26_lo16_sub26_hi16_sub27_lo16_sub27_hi16_sub28_lo16_sub28_hi16_sub29_lo16_sub29_hi16_sub30_lo16_sub30_hi16_sub31_lo16_sub31_hi16:av_1024_align2 = COPY [[COPY]].sub20_lo16_sub20_hi16_sub21_lo16_sub21_hi16_sub22_lo16_sub22_hi16_sub23_lo16_sub23_hi16_sub24_lo16_sub24_hi16_sub25_lo16_sub25_hi16_sub26_lo16_sub26_hi16_sub27_lo16_sub27_hi16_sub28_lo16_sub28_hi16_sub29_lo16_sub29_hi16_sub30_lo16_sub30_hi16_sub31_lo16_sub31_hi16
- ; CHECK-NEXT: internal [[COPY1]].sub17_sub18_sub19:av_1024_align2 = COPY [[COPY]].sub17_sub18_sub19
+ ; CHECK-NEXT: undef [[COPY1:%[0-9]+]].sub1_sub2_sub3_sub4_sub5_sub6_sub7_sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15_sub16:av_1024_align2 = lr-split COPY [[COPY]].sub1_sub2_sub3_sub4_sub5_sub6_sub7_sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15_sub16 {
+ ; CHECK-NEXT: internal [[COPY1]].sub20_lo16_sub20_hi16_sub21_lo16_sub21_hi16_sub22_lo16_sub22_hi16_sub23_lo16_sub23_hi16_sub24_lo16_sub24_hi16_sub25_lo16_sub25_hi16_sub26_lo16_sub26_hi16_sub27_lo16_sub27_hi16_sub28_lo16_sub28_hi16_sub29_lo16_sub29_hi16_sub30_lo16_sub30_hi16_sub31_lo16_sub31_hi16:av_1024_align2 = lr-split COPY [[COPY]].sub20_lo16_sub20_hi16_sub21_lo16_sub21_hi16_sub22_lo16_sub22_hi16_sub23_lo16_sub23_hi16_sub24_lo16_sub24_hi16_sub25_lo16_sub25_hi16_sub26_lo16_sub26_hi16_sub27_lo16_sub27_hi16_sub28_lo16_sub28_hi16_sub29_lo16_sub29_hi16_sub30_lo16_sub30_hi16_sub31_lo16_sub31_hi16
+ ; CHECK-NEXT: internal [[COPY1]].sub17_sub18_sub19:av_1024_align2 = lr-split COPY [[COPY]].sub17_sub18_sub19
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY1:%[0-9]+]].sub0:av_1024_align2 = IMPLICIT_DEF
; CHECK-NEXT: S_NOP 0, implicit [[COPY1]].sub0
@@ -116,9 +116,9 @@ body: |
; CHECK-NEXT: bb.2:
; CHECK-NEXT: successors: %bb.3(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: undef [[COPY1:%[0-9]+]].sub1_sub2_sub3_sub4_sub5_sub6_sub7_sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15_sub16:av_1024 = COPY [[COPY]].sub1_sub2_sub3_sub4_sub5_sub6_sub7_sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15_sub16 {
- ; CHECK-NEXT: internal [[COPY1]].sub17_lo16_sub17_hi16_sub18_lo16_sub18_hi16_sub19_lo16_sub19_hi16_sub20_lo16_sub20_hi16_sub21_lo16_sub21_hi16_sub22_lo16_sub22_hi16_sub23_lo16_sub23_hi16_sub24_lo16_sub24_hi16_sub25_lo16_sub25_hi16_sub26_lo16_sub26_hi16_sub27_lo16_sub27_hi16_sub28_lo16_sub28_hi16:av_1024 = COPY [[COPY]].sub17_lo16_sub17_hi16_sub18_lo16_sub18_hi16_sub19_lo16_sub19_hi16_sub20_lo16_sub20_hi16_sub21_lo16_sub21_hi16_sub22_lo16_sub22_hi16_sub23_lo16_sub23_hi16_sub24_lo16_sub24_hi16_sub25_lo16_sub25_hi16_sub26_lo16_sub26_hi16_sub27_lo16_sub27_hi16_sub28_lo16_sub28_hi16
- ; CHECK-NEXT: internal [[COPY1]].sub29_sub30:av_1024 = COPY [[COPY]].sub29_sub30
+ ; CHECK-NEXT: undef [[COPY1:%[0-9]+]].sub1_sub2_sub3_sub4_sub5_sub6_sub7_sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15_sub16:av_1024 = lr-split COPY [[COPY]].sub1_sub2_sub3_sub4_sub5_sub6_sub7_sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15_sub16 {
+ ; CHECK-NEXT: internal [[COPY1]].sub17_lo16_sub17_hi16_sub18_lo16_sub18_hi16_sub19_lo16_sub19_hi16_sub20_lo16_sub20_hi16_sub21_lo16_sub21_hi16_sub22_lo16_sub22_hi16_sub23_lo16_sub23_hi16_sub24_lo16_sub24_hi16_sub25_lo16_sub25_hi16_sub26_lo16_sub26_hi16_sub27_lo16_sub27_hi16_sub28_lo16_sub28_hi16:av_1024 = lr-split COPY [[COPY]].sub17_lo16_sub17_hi16_sub18_lo16_sub18_hi16_sub19_lo16_sub19_hi16_sub20_lo16_sub20_hi16_sub21_lo16_sub21_hi16_sub22_lo16_sub22_hi16_sub23_lo16_sub23_hi16_sub24_lo16_sub24_hi16_sub25_lo16_sub25_hi16_sub26_lo16_sub26_hi16_sub27_lo16_sub27_hi16_sub28_lo16_sub28_hi16
+ ; CHECK-NEXT: internal [[COPY1]].sub29_sub30:av_1024 = lr-split COPY [[COPY]].sub29_sub30
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY1:%[0-9]+]].sub0:av_1024 = IMPLICIT_DEF
; CHECK-NEXT: [[COPY1:%[0-9]+]].sub31:av_1024 = IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-copy-bundle.mir b/llvm/test/CodeGen/AMDGPU/splitkit-copy-bundle.mir
index 359152e9d2b45..0214018c680d1 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-copy-bundle.mir
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-copy-bundle.mir
@@ -25,7 +25,7 @@ body: |
; RA-NEXT: successors: %bb.2(0x80000000)
; RA-NEXT: {{ $}}
; RA-NEXT: [[SI_SPILL_S1024_RESTORE:%[0-9]+]]:sgpr_1024 = SI_SPILL_S1024_RESTORE %stack.0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.0, align 4, addrspace 5)
- ; RA-NEXT: undef [[COPY:%[0-9]+]].sub0_sub1:sgpr_1024 = COPY [[SI_SPILL_S1024_RESTORE]].sub0_sub1
+ ; RA-NEXT: undef [[COPY:%[0-9]+]].sub0_sub1:sgpr_1024 = lr-split COPY [[SI_SPILL_S1024_RESTORE]].sub0_sub1
; RA-NEXT: [[COPY:%[0-9]+]].sub2:sgpr_1024 = COPY [[COPY]].sub0
; RA-NEXT: [[COPY:%[0-9]+]].sub3:sgpr_1024 = COPY [[COPY]].sub1
; RA-NEXT: [[COPY:%[0-9]+]].sub4:sgpr_1024 = COPY [[COPY]].sub0
@@ -56,7 +56,7 @@ body: |
; RA-NEXT: [[COPY:%[0-9]+]].sub29:sgpr_1024 = COPY [[COPY]].sub1
; RA-NEXT: SI_SPILL_S1024_SAVE [[COPY]], %stack.0, implicit $exec, implicit $sgpr32 :: (store (s1024) into %stack.0, align 4, addrspace 5)
; RA-NEXT: [[SI_SPILL_S1024_RESTORE1:%[0-9]+]]:sgpr_1024 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
- ; RA-NEXT: undef [[COPY1:%[0-9]+]].sub0:sgpr_1024 = COPY [[SI_SPILL_S1024_RESTORE1]].sub0
+ ; RA-NEXT: undef [[COPY1:%[0-9]+]].sub0:sgpr_1024 = lr-split COPY [[SI_SPILL_S1024_RESTORE1]].sub0
; RA-NEXT: [[COPY1:%[0-9]+]].sub1:sgpr_1024 = COPY [[COPY1]].sub0
; RA-NEXT: [[COPY1:%[0-9]+]].sub2:sgpr_1024 = COPY [[COPY1]].sub0
; RA-NEXT: [[COPY1:%[0-9]+]].sub3:sgpr_1024 = COPY [[COPY1]].sub0
@@ -279,36 +279,36 @@ body: |
; RA-NEXT: [[DEF2:%[0-9]+]].sub8:sgpr_512 = S_MOV_B32 -1
; RA-NEXT: [[DEF2:%[0-9]+]].sub13:sgpr_512 = S_MOV_B32 -1
; RA-NEXT: [[DEF2:%[0-9]+]].sub14:sgpr_512 = S_MOV_B32 -1
- ; RA-NEXT: undef [[COPY:%[0-9]+]].sub4_sub5:sgpr_512 = COPY [[DEF2]].sub4_sub5 {
- ; RA-NEXT: internal [[COPY]].sub10_sub11:sgpr_512 = COPY [[DEF2]].sub10_sub11
- ; RA-NEXT: internal [[COPY]].sub7:sgpr_512 = COPY [[DEF2]].sub7
- ; RA-NEXT: internal [[COPY]].sub8:sgpr_512 = COPY [[DEF2]].sub8
- ; RA-NEXT: internal [[COPY]].sub13:sgpr_512 = COPY [[DEF2]].sub13
- ; RA-NEXT: internal [[COPY]].sub14:sgpr_512 = COPY [[DEF2]].sub14
+ ; RA-NEXT: undef [[COPY:%[0-9]+]].sub4_sub5:sgpr_512 = lr-split COPY [[DEF2]].sub4_sub5 {
+ ; RA-NEXT: internal [[COPY]].sub10_sub11:sgpr_512 = lr-split COPY [[DEF2]].sub10_sub11
+ ; RA-NEXT: internal [[COPY]].sub7:sgpr_512 = lr-split COPY [[DEF2]].sub7
+ ; RA-NEXT: internal [[COPY]].sub8:sgpr_512 = lr-split COPY [[DEF2]].sub8
+ ; RA-NEXT: internal [[COPY]].sub13:sgpr_512 = lr-split COPY [[DEF2]].sub13
+ ; RA-NEXT: internal [[COPY]].sub14:sgpr_512 = lr-split COPY [[DEF2]].sub14
; RA-NEXT: }
- ; RA-NEXT: undef [[COPY1:%[0-9]+]].sub4_sub5:sgpr_512 = COPY [[COPY]].sub4_sub5 {
- ; RA-NEXT: internal [[COPY1]].sub10_sub11:sgpr_512 = COPY [[COPY]].sub10_sub11
- ; RA-NEXT: internal [[COPY1]].sub7:sgpr_512 = COPY [[COPY]].sub7
- ; RA-NEXT: internal [[COPY1]].sub8:sgpr_512 = COPY [[COPY]].sub8
- ; RA-NEXT: internal [[COPY1]].sub13:sgpr_512 = COPY [[COPY]].sub13
- ; RA-NEXT: internal [[COPY1]].sub14:sgpr_512 = COPY [[COPY]].sub14
+ ; RA-NEXT: undef [[COPY1:%[0-9]+]].sub4_sub5:sgpr_512 = lr-split COPY [[COPY]].sub4_sub5 {
+ ; RA-NEXT: internal [[COPY1]].sub10_sub11:sgpr_512 = lr-split COPY [[COPY]].sub10_sub11
+ ; RA-NEXT: internal [[COPY1]].sub7:sgpr_512 = lr-split COPY [[COPY]].sub7
+ ; RA-NEXT: internal [[COPY1]].sub8:sgpr_512 = lr-split COPY [[COPY]].sub8
+ ; RA-NEXT: internal [[COPY1]].sub13:sgpr_512 = lr-split COPY [[COPY]].sub13
+ ; RA-NEXT: internal [[COPY1]].sub14:sgpr_512 = lr-split COPY [[COPY]].sub14
; RA-NEXT: }
; RA-NEXT: SI_SPILL_S512_SAVE [[COPY1]], %stack.0, implicit $exec, implicit $sgpr32 :: (store (s512) into %stack.0, align 4, addrspace 5)
; RA-NEXT: S_NOP 0, implicit-def $sgpr8, implicit-def $sgpr12, implicit-def $sgpr16, implicit-def $sgpr20, implicit-def $sgpr24, implicit-def $sgpr28, implicit-def $sgpr32, implicit-def $sgpr36, implicit-def $sgpr40, implicit-def $sgpr44, implicit-def $sgpr48, implicit-def $sgpr52, implicit-def $sgpr56, implicit-def $sgpr60, implicit-def $sgpr64, implicit-def $sgpr68, implicit-def $sgpr72, implicit-def $sgpr74, implicit-def $sgpr78, implicit-def $sgpr82, implicit-def $sgpr86, implicit-def $sgpr90, implicit-def $sgpr94, implicit-def $sgpr98
; RA-NEXT: [[SI_SPILL_S512_RESTORE:%[0-9]+]]:sgpr_512 = SI_SPILL_S512_RESTORE %stack.0, implicit $exec, implicit $sgpr32 :: (load (s512) from %stack.0, align 4, addrspace 5)
- ; RA-NEXT: undef [[COPY2:%[0-9]+]].sub4_sub5:sgpr_512 = COPY [[SI_SPILL_S512_RESTORE]].sub4_sub5 {
- ; RA-NEXT: internal [[COPY2]].sub10_sub11:sgpr_512 = COPY [[SI_SPILL_S512_RESTORE]].sub10_sub11
- ; RA-NEXT: internal [[COPY2]].sub7:sgpr_512 = COPY [[SI_SPILL_S512_RESTORE]].sub7
- ; RA-NEXT: internal [[COPY2]].sub8:sgpr_512 = COPY [[SI_SPILL_S512_RESTORE]].sub8
- ; RA-NEXT: internal [[COPY2]].sub13:sgpr_512 = COPY [[SI_SPILL_S512_RESTORE]].sub13
- ; RA-NEXT: internal [[COPY2]].sub14:sgpr_512 = COPY [[SI_SPILL_S512_RESTORE]].sub14
+ ; RA-NEXT: undef [[COPY2:%[0-9]+]].sub4_sub5:sgpr_512 = lr-split COPY [[SI_SPILL_S512_RESTORE]].sub4_sub5 {
+ ; RA-NEXT: internal [[COPY2]].sub10_sub11:sgpr_512 = lr-split COPY [[SI_SPILL_S512_RESTORE]].sub10_sub11
+ ; RA-NEXT: internal [[COPY2]].sub7:sgpr_512 = lr-split COPY [[SI_SPILL_S512_RESTORE]].sub7
+ ; RA-NEXT: internal [[COPY2]].sub8:sgpr_512 = lr-split COPY [[SI_SPILL_S512_RESTORE]].sub8
+ ; RA-NEXT: internal [[COPY2]].sub13:sgpr_512 = lr-split COPY [[SI_SPILL_S512_RESTORE]].sub13
+ ; RA-NEXT: internal [[COPY2]].sub14:sgpr_512 = lr-split COPY [[SI_SPILL_S512_RESTORE]].sub14
; RA-NEXT: }
- ; RA-NEXT: undef [[COPY3:%[0-9]+]].sub4_sub5:sgpr_512 = COPY [[COPY2]].sub4_sub5 {
- ; RA-NEXT: internal [[COPY3]].sub10_sub11:sgpr_512 = COPY [[COPY2]].sub10_sub11
- ; RA-NEXT: internal [[COPY3]].sub7:sgpr_512 = COPY [[COPY2]].sub7
- ; RA-NEXT: internal [[COPY3]].sub8:sgpr_512 = COPY [[COPY2]].sub8
- ; RA-NEXT: internal [[COPY3]].sub13:sgpr_512 = COPY [[COPY2]].sub13
- ; RA-NEXT: internal [[COPY3]].sub14:sgpr_512 = COPY [[COPY2]].sub14
+ ; RA-NEXT: undef [[COPY3:%[0-9]+]].sub4_sub5:sgpr_512 = lr-split COPY [[COPY2]].sub4_sub5 {
+ ; RA-NEXT: internal [[COPY3]].sub10_sub11:sgpr_512 = lr-split COPY [[COPY2]].sub10_sub11
+ ; RA-NEXT: internal [[COPY3]].sub7:sgpr_512 = lr-split COPY [[COPY2]].sub7
+ ; RA-NEXT: internal [[COPY3]].sub8:sgpr_512 = lr-split COPY [[COPY2]].sub8
+ ; RA-NEXT: internal [[COPY3]].sub13:sgpr_512 = lr-split COPY [[COPY2]].sub13
+ ; RA-NEXT: internal [[COPY3]].sub14:sgpr_512 = lr-split COPY [[COPY2]].sub14
; RA-NEXT: }
; RA-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR [[DEF]], [[COPY3]].sub4, 0 :: (dereferenceable invariant load (s32))
; RA-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR1:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR [[DEF]], [[COPY3]].sub5, 0 :: (dereferenceable invariant load (s32))
@@ -333,12 +333,12 @@ body: |
; VR-NEXT: SI_SPILL_S512_SAVE killed renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27, %stack.0, implicit $exec, implicit $sgpr32 :: (store (s512) into %stack.0, align 4, addrspace 5)
; VR-NEXT: S_NOP 0, implicit-def $sgpr8, implicit-def $sgpr12, implicit-def $sgpr16, implicit-def $sgpr20, implicit-def $sgpr24, implicit-def $sgpr28, implicit-def $sgpr32, implicit-def $sgpr36, implicit-def $sgpr40, implicit-def $sgpr44, implicit-def $sgpr48, implicit-def $sgpr52, implicit-def $sgpr56, implicit-def $sgpr60, implicit-def $sgpr64, implicit-def $sgpr68, implicit-def $sgpr72, implicit-def $sgpr74, implicit-def $sgpr78, implicit-def $sgpr82, implicit-def $sgpr86, implicit-def $sgpr90, implicit-def $sgpr94, implicit-def $sgpr98
; VR-NEXT: renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27 = SI_SPILL_S512_RESTORE %stack.0, implicit $exec, implicit $sgpr32 :: (load (s512) from %stack.0, align 4, addrspace 5)
- ; VR-NEXT: renamable $sgpr12_sgpr13 = COPY killed renamable $sgpr16_sgpr17
- ; VR-NEXT: renamable $sgpr15 = COPY killed renamable $sgpr19
- ; VR-NEXT: renamable $sgpr18_sgpr19 = COPY killed renamable $sgpr22_sgpr23
- ; VR-NEXT: renamable $sgpr16 = COPY killed renamable $sgpr20
- ; VR-NEXT: renamable $sgpr21 = COPY killed renamable $sgpr25
- ; VR-NEXT: renamable $sgpr22 = COPY killed renamable $sgpr26
+ ; VR-NEXT: renamable $sgpr12_sgpr13 = lr-split COPY killed renamable $sgpr16_sgpr17
+ ; VR-NEXT: renamable $sgpr15 = lr-split COPY killed renamable $sgpr19
+ ; VR-NEXT: renamable $sgpr18_sgpr19 = lr-split COPY killed renamable $sgpr22_sgpr23
+ ; VR-NEXT: renamable $sgpr16 = lr-split COPY killed renamable $sgpr20
+ ; VR-NEXT: renamable $sgpr21 = lr-split COPY killed renamable $sgpr25
+ ; VR-NEXT: renamable $sgpr22 = lr-split COPY killed renamable $sgpr26
; VR-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7 = IMPLICIT_DEF
; VR-NEXT: renamable $sgpr8 = S_BUFFER_LOAD_DWORD_SGPR renamable $sgpr4_sgpr5_sgpr6_sgpr7, killed renamable $sgpr12, 0 :: (dereferenceable invariant load (s32))
; VR-NEXT: renamable $sgpr9 = S_BUFFER_LOAD_DWORD_SGPR renamable $sgpr4_sgpr5_sgpr6_sgpr7, killed renamable $sgpr13, 0 :: (dereferenceable invariant load (s32))
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-copy-live-lanes.mir b/llvm/test/CodeGen/AMDGPU/splitkit-copy-live-lanes.mir
index e1e40ccf70e8f..d27d9ae829e91 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-copy-live-lanes.mir
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-copy-live-lanes.mir
@@ -62,143 +62,143 @@ body: |
; CHECK-NEXT: undef [[V_LSHRREV_B32_e32_25:%[0-9]+]].sub2:vreg_128 = V_LSHRREV_B32_e32 16, [[BUFFER_LOAD_DWORDX4_OFFSET7]].sub3, implicit $exec
; CHECK-NEXT: undef [[V_LSHRREV_B32_e32_26:%[0-9]+]].sub2:vreg_128 = V_LSHRREV_B32_e32 16, [[BUFFER_LOAD_DWORDX4_OFFSET7]].sub2, implicit $exec
; CHECK-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
- ; CHECK-NEXT: undef [[COPY2:%[0-9]+]].sub2:vreg_128 = COPY [[V_LSHRREV_B32_e32_]].sub2
+ ; CHECK-NEXT: undef [[COPY2:%[0-9]+]].sub2:vreg_128 = lr-split COPY [[V_LSHRREV_B32_e32_]].sub2
; CHECK-NEXT: [[COPY2:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET]].sub1, implicit $exec
- ; CHECK-NEXT: undef [[COPY3:%[0-9]+]].sub0:vreg_128 = COPY [[COPY2]].sub0 {
- ; CHECK-NEXT: internal [[COPY3]].sub2:vreg_128 = COPY [[COPY2]].sub2
+ ; CHECK-NEXT: undef [[COPY3:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY2]].sub0 {
+ ; CHECK-NEXT: internal [[COPY3]].sub2:vreg_128 = lr-split COPY [[COPY2]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: SI_SPILL_V128_SAVE [[COPY3]], %stack.0, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.0, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY4:%[0-9]+]].sub2:vreg_128 = COPY [[V_LSHRREV_B32_e32_1]].sub2
+ ; CHECK-NEXT: undef [[COPY4:%[0-9]+]].sub2:vreg_128 = lr-split COPY [[V_LSHRREV_B32_e32_1]].sub2
; CHECK-NEXT: [[COPY4:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET]].sub0, implicit $exec
- ; CHECK-NEXT: undef [[COPY5:%[0-9]+]].sub0:vreg_128 = COPY [[COPY4]].sub0 {
- ; CHECK-NEXT: internal [[COPY5]].sub2:vreg_128 = COPY [[COPY4]].sub2
+ ; CHECK-NEXT: undef [[COPY5:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY4]].sub0 {
+ ; CHECK-NEXT: internal [[COPY5]].sub2:vreg_128 = lr-split COPY [[COPY4]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: SI_SPILL_V128_SAVE [[COPY5]], %stack.1, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.1, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY6:%[0-9]+]].sub2:vreg_128 = COPY [[V_LSHRREV_B32_e32_2]].sub2
+ ; CHECK-NEXT: undef [[COPY6:%[0-9]+]].sub2:vreg_128 = lr-split COPY [[V_LSHRREV_B32_e32_2]].sub2
; CHECK-NEXT: [[COPY6:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET]].sub3, implicit $exec
- ; CHECK-NEXT: undef [[COPY7:%[0-9]+]].sub0:vreg_128 = COPY [[COPY6]].sub0 {
- ; CHECK-NEXT: internal [[COPY7]].sub2:vreg_128 = COPY [[COPY6]].sub2
+ ; CHECK-NEXT: undef [[COPY7:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY6]].sub0 {
+ ; CHECK-NEXT: internal [[COPY7]].sub2:vreg_128 = lr-split COPY [[COPY6]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: SI_SPILL_V128_SAVE [[COPY7]], %stack.2, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY8:%[0-9]+]].sub2:vreg_128 = COPY [[V_LSHRREV_B32_e32_3]].sub2
+ ; CHECK-NEXT: undef [[COPY8:%[0-9]+]].sub2:vreg_128 = lr-split COPY [[V_LSHRREV_B32_e32_3]].sub2
; CHECK-NEXT: [[COPY8:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET]].sub2, implicit $exec
- ; CHECK-NEXT: undef [[COPY9:%[0-9]+]].sub0:vreg_128 = COPY [[COPY8]].sub0 {
- ; CHECK-NEXT: internal [[COPY9]].sub2:vreg_128 = COPY [[COPY8]].sub2
+ ; CHECK-NEXT: undef [[COPY9:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY8]].sub0 {
+ ; CHECK-NEXT: internal [[COPY9]].sub2:vreg_128 = lr-split COPY [[COPY8]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: SI_SPILL_V128_SAVE [[COPY9]], %stack.3, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.3, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY10:%[0-9]+]].sub2:vreg_128 = COPY [[V_LSHRREV_B32_e32_4]].sub2
+ ; CHECK-NEXT: undef [[COPY10:%[0-9]+]].sub2:vreg_128 = lr-split COPY [[V_LSHRREV_B32_e32_4]].sub2
; CHECK-NEXT: [[COPY10:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET1]].sub1, implicit $exec
- ; CHECK-NEXT: undef [[COPY11:%[0-9]+]].sub0:vreg_128 = COPY [[COPY10]].sub0 {
- ; CHECK-NEXT: internal [[COPY11]].sub2:vreg_128 = COPY [[COPY10]].sub2
+ ; CHECK-NEXT: undef [[COPY11:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY10]].sub0 {
+ ; CHECK-NEXT: internal [[COPY11]].sub2:vreg_128 = lr-split COPY [[COPY10]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: SI_SPILL_V128_SAVE [[COPY11]], %stack.4, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.4, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY12:%[0-9]+]].sub2:vreg_128 = COPY [[V_LSHRREV_B32_e32_5]].sub2
+ ; CHECK-NEXT: undef [[COPY12:%[0-9]+]].sub2:vreg_128 = lr-split COPY [[V_LSHRREV_B32_e32_5]].sub2
; CHECK-NEXT: [[COPY12:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET1]].sub0, implicit $exec
- ; CHECK-NEXT: undef [[COPY13:%[0-9]+]].sub0:vreg_128 = COPY [[COPY12]].sub0 {
- ; CHECK-NEXT: internal [[COPY13]].sub2:vreg_128 = COPY [[COPY12]].sub2
+ ; CHECK-NEXT: undef [[COPY13:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY12]].sub0 {
+ ; CHECK-NEXT: internal [[COPY13]].sub2:vreg_128 = lr-split COPY [[COPY12]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: SI_SPILL_V128_SAVE [[COPY13]], %stack.5, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.5, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY14:%[0-9]+]].sub2:vreg_128 = COPY [[V_LSHRREV_B32_e32_6]].sub2
+ ; CHECK-NEXT: undef [[COPY14:%[0-9]+]].sub2:vreg_128 = lr-split COPY [[V_LSHRREV_B32_e32_6]].sub2
; CHECK-NEXT: [[COPY14:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET1]].sub3, implicit $exec
- ; CHECK-NEXT: undef [[COPY15:%[0-9]+]].sub0:vreg_128 = COPY [[COPY14]].sub0 {
- ; CHECK-NEXT: internal [[COPY15]].sub2:vreg_128 = COPY [[COPY14]].sub2
+ ; CHECK-NEXT: undef [[COPY15:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY14]].sub0 {
+ ; CHECK-NEXT: internal [[COPY15]].sub2:vreg_128 = lr-split COPY [[COPY14]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: SI_SPILL_V128_SAVE [[COPY15]], %stack.7, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.7, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY16:%[0-9]+]].sub2:vreg_128 = COPY [[V_LSHRREV_B32_e32_7]].sub2
+ ; CHECK-NEXT: undef [[COPY16:%[0-9]+]].sub2:vreg_128 = lr-split COPY [[V_LSHRREV_B32_e32_7]].sub2
; CHECK-NEXT: [[COPY16:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET1]].sub2, implicit $exec
- ; CHECK-NEXT: undef [[COPY17:%[0-9]+]].sub0:vreg_128 = COPY [[COPY16]].sub0 {
- ; CHECK-NEXT: internal [[COPY17]].sub2:vreg_128 = COPY [[COPY16]].sub2
+ ; CHECK-NEXT: undef [[COPY17:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY16]].sub0 {
+ ; CHECK-NEXT: internal [[COPY17]].sub2:vreg_128 = lr-split COPY [[COPY16]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: SI_SPILL_V128_SAVE [[COPY17]], %stack.6, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.6, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY18:%[0-9]+]].sub2:vreg_128 = COPY [[V_LSHRREV_B32_e32_8]].sub2
+ ; CHECK-NEXT: undef [[COPY18:%[0-9]+]].sub2:vreg_128 = lr-split COPY [[V_LSHRREV_B32_e32_8]].sub2
; CHECK-NEXT: [[COPY18:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET2]].sub1, implicit $exec
- ; CHECK-NEXT: undef [[COPY19:%[0-9]+]].sub0:vreg_128 = COPY [[COPY18]].sub0 {
- ; CHECK-NEXT: internal [[COPY19]].sub2:vreg_128 = COPY [[COPY18]].sub2
+ ; CHECK-NEXT: undef [[COPY19:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY18]].sub0 {
+ ; CHECK-NEXT: internal [[COPY19]].sub2:vreg_128 = lr-split COPY [[COPY18]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY20:%[0-9]+]].sub2:vreg_128 = COPY [[V_LSHRREV_B32_e32_9]].sub2
+ ; CHECK-NEXT: undef [[COPY20:%[0-9]+]].sub2:vreg_128 = lr-split COPY [[V_LSHRREV_B32_e32_9]].sub2
; CHECK-NEXT: [[COPY20:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET2]].sub0, implicit $exec
- ; CHECK-NEXT: undef [[COPY21:%[0-9]+]].sub0:vreg_128 = COPY [[COPY20]].sub0 {
- ; CHECK-NEXT: internal [[COPY21]].sub2:vreg_128 = COPY [[COPY20]].sub2
+ ; CHECK-NEXT: undef [[COPY21:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY20]].sub0 {
+ ; CHECK-NEXT: internal [[COPY21]].sub2:vreg_128 = lr-split COPY [[COPY20]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: [[V_LSHRREV_B32_e32_10:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET2]].sub3, implicit $exec
- ; CHECK-NEXT: undef [[COPY22:%[0-9]+]].sub0:vreg_128 = COPY [[V_LSHRREV_B32_e32_10]].sub0 {
- ; CHECK-NEXT: internal [[COPY22]].sub2:vreg_128 = COPY [[V_LSHRREV_B32_e32_10]].sub2
+ ; CHECK-NEXT: undef [[COPY22:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[V_LSHRREV_B32_e32_10]].sub0 {
+ ; CHECK-NEXT: internal [[COPY22]].sub2:vreg_128 = lr-split COPY [[V_LSHRREV_B32_e32_10]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY23:%[0-9]+]].sub0:vreg_128 = COPY [[COPY22]].sub0 {
- ; CHECK-NEXT: internal [[COPY23]].sub2:vreg_128 = COPY [[COPY22]].sub2
+ ; CHECK-NEXT: undef [[COPY23:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY22]].sub0 {
+ ; CHECK-NEXT: internal [[COPY23]].sub2:vreg_128 = lr-split COPY [[COPY22]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: SI_SPILL_V128_SAVE [[COPY23]], %stack.8, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.8, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY24:%[0-9]+]].sub2:vreg_128 = COPY [[V_LSHRREV_B32_e32_11]].sub2
+ ; CHECK-NEXT: undef [[COPY24:%[0-9]+]].sub2:vreg_128 = lr-split COPY [[V_LSHRREV_B32_e32_11]].sub2
; CHECK-NEXT: [[COPY24:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET2]].sub2, implicit $exec
- ; CHECK-NEXT: undef [[COPY25:%[0-9]+]].sub0:vreg_128 = COPY [[COPY24]].sub0 {
- ; CHECK-NEXT: internal [[COPY25]].sub2:vreg_128 = COPY [[COPY24]].sub2
+ ; CHECK-NEXT: undef [[COPY25:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY24]].sub0 {
+ ; CHECK-NEXT: internal [[COPY25]].sub2:vreg_128 = lr-split COPY [[COPY24]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: SI_SPILL_V128_SAVE [[COPY25]], %stack.11, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.11, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY26:%[0-9]+]].sub2:vreg_128 = COPY [[V_LSHRREV_B32_e32_12]].sub2
+ ; CHECK-NEXT: undef [[COPY26:%[0-9]+]].sub2:vreg_128 = lr-split COPY [[V_LSHRREV_B32_e32_12]].sub2
; CHECK-NEXT: [[COPY26:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET3]].sub1, implicit $exec
- ; CHECK-NEXT: undef [[COPY27:%[0-9]+]].sub0:vreg_128 = COPY [[COPY26]].sub0 {
- ; CHECK-NEXT: internal [[COPY27]].sub2:vreg_128 = COPY [[COPY26]].sub2
+ ; CHECK-NEXT: undef [[COPY27:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY26]].sub0 {
+ ; CHECK-NEXT: internal [[COPY27]].sub2:vreg_128 = lr-split COPY [[COPY26]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: SI_SPILL_V128_SAVE [[COPY27]], %stack.9, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.9, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY28:%[0-9]+]].sub2:vreg_128 = COPY [[V_LSHRREV_B32_e32_13]].sub2
+ ; CHECK-NEXT: undef [[COPY28:%[0-9]+]].sub2:vreg_128 = lr-split COPY [[V_LSHRREV_B32_e32_13]].sub2
; CHECK-NEXT: [[COPY28:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET3]].sub0, implicit $exec
- ; CHECK-NEXT: undef [[COPY29:%[0-9]+]].sub0:vreg_128 = COPY [[COPY28]].sub0 {
- ; CHECK-NEXT: internal [[COPY29]].sub2:vreg_128 = COPY [[COPY28]].sub2
+ ; CHECK-NEXT: undef [[COPY29:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY28]].sub0 {
+ ; CHECK-NEXT: internal [[COPY29]].sub2:vreg_128 = lr-split COPY [[COPY28]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY30:%[0-9]+]].sub2:vreg_128 = COPY [[V_LSHRREV_B32_e32_14]].sub2
+ ; CHECK-NEXT: undef [[COPY30:%[0-9]+]].sub2:vreg_128 = lr-split COPY [[V_LSHRREV_B32_e32_14]].sub2
; CHECK-NEXT: [[COPY30:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET3]].sub3, implicit $exec
- ; CHECK-NEXT: undef [[COPY31:%[0-9]+]].sub0:vreg_128 = COPY [[COPY30]].sub0 {
- ; CHECK-NEXT: internal [[COPY31]].sub2:vreg_128 = COPY [[COPY30]].sub2
+ ; CHECK-NEXT: undef [[COPY31:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY30]].sub0 {
+ ; CHECK-NEXT: internal [[COPY31]].sub2:vreg_128 = lr-split COPY [[COPY30]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: SI_SPILL_V128_SAVE [[COPY31]], %stack.10, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.10, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY32:%[0-9]+]].sub2:vreg_128 = COPY [[V_LSHRREV_B32_e32_15]].sub2
+ ; CHECK-NEXT: undef [[COPY32:%[0-9]+]].sub2:vreg_128 = lr-split COPY [[V_LSHRREV_B32_e32_15]].sub2
; CHECK-NEXT: [[COPY32:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET3]].sub2, implicit $exec
- ; CHECK-NEXT: undef [[COPY33:%[0-9]+]].sub0:vreg_128 = COPY [[COPY32]].sub0 {
- ; CHECK-NEXT: internal [[COPY33]].sub2:vreg_128 = COPY [[COPY32]].sub2
+ ; CHECK-NEXT: undef [[COPY33:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY32]].sub0 {
+ ; CHECK-NEXT: internal [[COPY33]].sub2:vreg_128 = lr-split COPY [[COPY32]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY34:%[0-9]+]].sub2:vreg_128 = COPY [[V_LSHRREV_B32_e32_16]].sub2
+ ; CHECK-NEXT: undef [[COPY34:%[0-9]+]].sub2:vreg_128 = lr-split COPY [[V_LSHRREV_B32_e32_16]].sub2
; CHECK-NEXT: [[COPY34:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET4]].sub1, implicit $exec
- ; CHECK-NEXT: undef [[COPY35:%[0-9]+]].sub0:vreg_128 = COPY [[COPY34]].sub0 {
- ; CHECK-NEXT: internal [[COPY35]].sub2:vreg_128 = COPY [[COPY34]].sub2
+ ; CHECK-NEXT: undef [[COPY35:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY34]].sub0 {
+ ; CHECK-NEXT: internal [[COPY35]].sub2:vreg_128 = lr-split COPY [[COPY34]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY36:%[0-9]+]].sub2:vreg_128 = COPY [[V_LSHRREV_B32_e32_17]].sub2
+ ; CHECK-NEXT: undef [[COPY36:%[0-9]+]].sub2:vreg_128 = lr-split COPY [[V_LSHRREV_B32_e32_17]].sub2
; CHECK-NEXT: [[COPY36:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET4]].sub0, implicit $exec
- ; CHECK-NEXT: undef [[COPY37:%[0-9]+]].sub0:vreg_128 = COPY [[COPY36]].sub0 {
- ; CHECK-NEXT: internal [[COPY37]].sub2:vreg_128 = COPY [[COPY36]].sub2
+ ; CHECK-NEXT: undef [[COPY37:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY36]].sub0 {
+ ; CHECK-NEXT: internal [[COPY37]].sub2:vreg_128 = lr-split COPY [[COPY36]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY38:%[0-9]+]].sub2:vreg_128 = COPY [[V_LSHRREV_B32_e32_18]].sub2
+ ; CHECK-NEXT: undef [[COPY38:%[0-9]+]].sub2:vreg_128 = lr-split COPY [[V_LSHRREV_B32_e32_18]].sub2
; CHECK-NEXT: [[COPY38:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET4]].sub3, implicit $exec
- ; CHECK-NEXT: undef [[COPY39:%[0-9]+]].sub0:vreg_128 = COPY [[COPY38]].sub0 {
- ; CHECK-NEXT: internal [[COPY39]].sub2:vreg_128 = COPY [[COPY38]].sub2
+ ; CHECK-NEXT: undef [[COPY39:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY38]].sub0 {
+ ; CHECK-NEXT: internal [[COPY39]].sub2:vreg_128 = lr-split COPY [[COPY38]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY40:%[0-9]+]].sub2:vreg_128 = COPY [[V_LSHRREV_B32_e32_19]].sub2
+ ; CHECK-NEXT: undef [[COPY40:%[0-9]+]].sub2:vreg_128 = lr-split COPY [[V_LSHRREV_B32_e32_19]].sub2
; CHECK-NEXT: [[COPY40:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET4]].sub2, implicit $exec
- ; CHECK-NEXT: undef [[COPY41:%[0-9]+]].sub0:vreg_128 = COPY [[COPY40]].sub0 {
- ; CHECK-NEXT: internal [[COPY41]].sub2:vreg_128 = COPY [[COPY40]].sub2
+ ; CHECK-NEXT: undef [[COPY41:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY40]].sub0 {
+ ; CHECK-NEXT: internal [[COPY41]].sub2:vreg_128 = lr-split COPY [[COPY40]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[V_LSHRREV_B32_e32_27:%[0-9]+]].sub2:vreg_128 = V_LSHRREV_B32_e32 16, [[BUFFER_LOAD_DWORDX4_OFFSET5]].sub1, implicit $exec
- ; CHECK-NEXT: undef [[COPY42:%[0-9]+]].sub2:vreg_128 = COPY [[V_LSHRREV_B32_e32_27]].sub2
+ ; CHECK-NEXT: undef [[COPY42:%[0-9]+]].sub2:vreg_128 = lr-split COPY [[V_LSHRREV_B32_e32_27]].sub2
; CHECK-NEXT: [[COPY42:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET5]].sub1, implicit $exec
- ; CHECK-NEXT: undef [[COPY43:%[0-9]+]].sub0:vreg_128 = COPY [[COPY42]].sub0 {
- ; CHECK-NEXT: internal [[COPY43]].sub2:vreg_128 = COPY [[COPY42]].sub2
+ ; CHECK-NEXT: undef [[COPY43:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY42]].sub0 {
+ ; CHECK-NEXT: internal [[COPY43]].sub2:vreg_128 = lr-split COPY [[COPY42]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[V_LSHRREV_B32_e32_28:%[0-9]+]].sub2:vreg_128 = V_LSHRREV_B32_e32 16, [[BUFFER_LOAD_DWORDX4_OFFSET5]].sub0, implicit $exec
- ; CHECK-NEXT: undef [[COPY44:%[0-9]+]].sub2:vreg_128 = COPY [[V_LSHRREV_B32_e32_28]].sub2
+ ; CHECK-NEXT: undef [[COPY44:%[0-9]+]].sub2:vreg_128 = lr-split COPY [[V_LSHRREV_B32_e32_28]].sub2
; CHECK-NEXT: [[COPY44:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET5]].sub0, implicit $exec
- ; CHECK-NEXT: undef [[COPY45:%[0-9]+]].sub0:vreg_128 = COPY [[COPY44]].sub0 {
- ; CHECK-NEXT: internal [[COPY45]].sub2:vreg_128 = COPY [[COPY44]].sub2
+ ; CHECK-NEXT: undef [[COPY45:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY44]].sub0 {
+ ; CHECK-NEXT: internal [[COPY45]].sub2:vreg_128 = lr-split COPY [[COPY44]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[V_LSHRREV_B32_e32_29:%[0-9]+]].sub2:vreg_128 = V_LSHRREV_B32_e32 16, [[BUFFER_LOAD_DWORDX4_OFFSET5]].sub3, implicit $exec
- ; CHECK-NEXT: undef [[COPY46:%[0-9]+]].sub2:vreg_128 = COPY [[V_LSHRREV_B32_e32_29]].sub2
+ ; CHECK-NEXT: undef [[COPY46:%[0-9]+]].sub2:vreg_128 = lr-split COPY [[V_LSHRREV_B32_e32_29]].sub2
; CHECK-NEXT: [[COPY46:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET5]].sub3, implicit $exec
- ; CHECK-NEXT: undef [[COPY47:%[0-9]+]].sub0:vreg_128 = COPY [[COPY46]].sub0 {
- ; CHECK-NEXT: internal [[COPY47]].sub2:vreg_128 = COPY [[COPY46]].sub2
+ ; CHECK-NEXT: undef [[COPY47:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY46]].sub0 {
+ ; CHECK-NEXT: internal [[COPY47]].sub2:vreg_128 = lr-split COPY [[COPY46]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[V_LSHRREV_B32_e32_30:%[0-9]+]].sub2:vreg_128 = V_LSHRREV_B32_e32 16, [[BUFFER_LOAD_DWORDX4_OFFSET5]].sub2, implicit $exec
- ; CHECK-NEXT: undef [[COPY48:%[0-9]+]].sub2:vreg_128 = COPY [[V_LSHRREV_B32_e32_30]].sub2
+ ; CHECK-NEXT: undef [[COPY48:%[0-9]+]].sub2:vreg_128 = lr-split COPY [[V_LSHRREV_B32_e32_30]].sub2
; CHECK-NEXT: [[COPY48:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET5]].sub2, implicit $exec
; CHECK-NEXT: undef [[V_LSHRREV_B32_e32_31:%[0-9]+]].sub2:vreg_128 = V_LSHRREV_B32_e32 16, [[BUFFER_LOAD_DWORDX4_OFFSET6]].sub1, implicit $exec
- ; CHECK-NEXT: undef [[COPY49:%[0-9]+]].sub2:vreg_128 = COPY [[V_LSHRREV_B32_e32_31]].sub2
+ ; CHECK-NEXT: undef [[COPY49:%[0-9]+]].sub2:vreg_128 = lr-split COPY [[V_LSHRREV_B32_e32_31]].sub2
; CHECK-NEXT: [[COPY49:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET6]].sub1, implicit $exec
; CHECK-NEXT: [[V_LSHRREV_B32_e32_20:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET6]].sub0, implicit $exec
; CHECK-NEXT: [[V_LSHRREV_B32_e32_21:%[0-9]+]].sub0:vreg_128 = V_AND_B32_e32 [[S_MOV_B32_1]], [[BUFFER_LOAD_DWORDX4_OFFSET6]].sub3, implicit $exec
@@ -228,233 +228,233 @@ body: |
; CHECK-NEXT: [[V_LSHRREV_B32_e32_20:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[V_LSHRREV_B32_e32_20:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[V_LSHRREV_B32_e32_20]], [[S_MOV_B32_]], 0, 384, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
- ; CHECK-NEXT: undef [[COPY50:%[0-9]+]].sub0:vreg_128 = COPY [[COPY49]].sub0 {
- ; CHECK-NEXT: internal [[COPY50]].sub2:vreg_128 = COPY [[COPY49]].sub2
+ ; CHECK-NEXT: undef [[COPY50:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY49]].sub0 {
+ ; CHECK-NEXT: internal [[COPY50]].sub2:vreg_128 = lr-split COPY [[COPY49]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY50:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY50:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY50]], [[S_MOV_B32_]], 0, 400, 0, 0, implicit $exec :: (store (s128), addrspace 1)
- ; CHECK-NEXT: undef [[COPY51:%[0-9]+]].sub0:vreg_128 = COPY [[COPY48]].sub0 {
- ; CHECK-NEXT: internal [[COPY51]].sub2:vreg_128 = COPY [[COPY48]].sub2
+ ; CHECK-NEXT: undef [[COPY51:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY48]].sub0 {
+ ; CHECK-NEXT: internal [[COPY51]].sub2:vreg_128 = lr-split COPY [[COPY48]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY51:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY51:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY51]], [[S_MOV_B32_]], 0, 352, 0, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
- ; CHECK-NEXT: undef [[COPY52:%[0-9]+]].sub0:vreg_128 = COPY [[COPY47]].sub0 {
- ; CHECK-NEXT: internal [[COPY52]].sub2:vreg_128 = COPY [[COPY47]].sub2
+ ; CHECK-NEXT: undef [[COPY52:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY47]].sub0 {
+ ; CHECK-NEXT: internal [[COPY52]].sub2:vreg_128 = lr-split COPY [[COPY47]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY53:%[0-9]+]].sub0:vreg_128 = COPY [[COPY52]].sub0 {
- ; CHECK-NEXT: internal [[COPY53]].sub2:vreg_128 = COPY [[COPY52]].sub2
+ ; CHECK-NEXT: undef [[COPY53:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY52]].sub0 {
+ ; CHECK-NEXT: internal [[COPY53]].sub2:vreg_128 = lr-split COPY [[COPY52]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY53:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY53:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY53]], [[S_MOV_B32_]], 0, 368, 0, 0, implicit $exec :: (store (s128), addrspace 1)
- ; CHECK-NEXT: undef [[COPY54:%[0-9]+]].sub0:vreg_128 = COPY [[COPY45]].sub0 {
- ; CHECK-NEXT: internal [[COPY54]].sub2:vreg_128 = COPY [[COPY45]].sub2
+ ; CHECK-NEXT: undef [[COPY54:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY45]].sub0 {
+ ; CHECK-NEXT: internal [[COPY54]].sub2:vreg_128 = lr-split COPY [[COPY45]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY55:%[0-9]+]].sub0:vreg_128 = COPY [[COPY54]].sub0 {
- ; CHECK-NEXT: internal [[COPY55]].sub2:vreg_128 = COPY [[COPY54]].sub2
+ ; CHECK-NEXT: undef [[COPY55:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY54]].sub0 {
+ ; CHECK-NEXT: internal [[COPY55]].sub2:vreg_128 = lr-split COPY [[COPY54]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY55:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY55:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY55]], [[S_MOV_B32_]], 0, 320, 0, 0, implicit $exec :: (store (s128), align 64, addrspace 1)
- ; CHECK-NEXT: undef [[COPY56:%[0-9]+]].sub0:vreg_128 = COPY [[COPY43]].sub0 {
- ; CHECK-NEXT: internal [[COPY56]].sub2:vreg_128 = COPY [[COPY43]].sub2
+ ; CHECK-NEXT: undef [[COPY56:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY43]].sub0 {
+ ; CHECK-NEXT: internal [[COPY56]].sub2:vreg_128 = lr-split COPY [[COPY43]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY57:%[0-9]+]].sub0:vreg_128 = COPY [[COPY56]].sub0 {
- ; CHECK-NEXT: internal [[COPY57]].sub2:vreg_128 = COPY [[COPY56]].sub2
+ ; CHECK-NEXT: undef [[COPY57:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY56]].sub0 {
+ ; CHECK-NEXT: internal [[COPY57]].sub2:vreg_128 = lr-split COPY [[COPY56]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY57:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY57:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY57]], [[S_MOV_B32_]], 0, 336, 0, 0, implicit $exec :: (store (s128), addrspace 1)
- ; CHECK-NEXT: undef [[COPY58:%[0-9]+]].sub0:vreg_128 = COPY [[COPY41]].sub0 {
- ; CHECK-NEXT: internal [[COPY58]].sub2:vreg_128 = COPY [[COPY41]].sub2
+ ; CHECK-NEXT: undef [[COPY58:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY41]].sub0 {
+ ; CHECK-NEXT: internal [[COPY58]].sub2:vreg_128 = lr-split COPY [[COPY41]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY59:%[0-9]+]].sub0:vreg_128 = COPY [[COPY58]].sub0 {
- ; CHECK-NEXT: internal [[COPY59]].sub2:vreg_128 = COPY [[COPY58]].sub2
+ ; CHECK-NEXT: undef [[COPY59:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY58]].sub0 {
+ ; CHECK-NEXT: internal [[COPY59]].sub2:vreg_128 = lr-split COPY [[COPY58]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY59:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY59:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY59]], [[S_MOV_B32_]], 0, 288, 0, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
- ; CHECK-NEXT: undef [[COPY60:%[0-9]+]].sub0:vreg_128 = COPY [[COPY39]].sub0 {
- ; CHECK-NEXT: internal [[COPY60]].sub2:vreg_128 = COPY [[COPY39]].sub2
+ ; CHECK-NEXT: undef [[COPY60:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY39]].sub0 {
+ ; CHECK-NEXT: internal [[COPY60]].sub2:vreg_128 = lr-split COPY [[COPY39]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY61:%[0-9]+]].sub0:vreg_128 = COPY [[COPY60]].sub0 {
- ; CHECK-NEXT: internal [[COPY61]].sub2:vreg_128 = COPY [[COPY60]].sub2
+ ; CHECK-NEXT: undef [[COPY61:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY60]].sub0 {
+ ; CHECK-NEXT: internal [[COPY61]].sub2:vreg_128 = lr-split COPY [[COPY60]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY61:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY61:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY61]], [[S_MOV_B32_]], 0, 304, 0, 0, implicit $exec :: (store (s128), addrspace 1)
- ; CHECK-NEXT: undef [[COPY62:%[0-9]+]].sub0:vreg_128 = COPY [[COPY37]].sub0 {
- ; CHECK-NEXT: internal [[COPY62]].sub2:vreg_128 = COPY [[COPY37]].sub2
+ ; CHECK-NEXT: undef [[COPY62:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY37]].sub0 {
+ ; CHECK-NEXT: internal [[COPY62]].sub2:vreg_128 = lr-split COPY [[COPY37]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY63:%[0-9]+]].sub0:vreg_128 = COPY [[COPY62]].sub0 {
- ; CHECK-NEXT: internal [[COPY63]].sub2:vreg_128 = COPY [[COPY62]].sub2
+ ; CHECK-NEXT: undef [[COPY63:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY62]].sub0 {
+ ; CHECK-NEXT: internal [[COPY63]].sub2:vreg_128 = lr-split COPY [[COPY62]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY63:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY63:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY63]], [[S_MOV_B32_]], 0, 256, 0, 0, implicit $exec :: (store (s128), align 256, addrspace 1)
- ; CHECK-NEXT: undef [[COPY64:%[0-9]+]].sub0:vreg_128 = COPY [[COPY35]].sub0 {
- ; CHECK-NEXT: internal [[COPY64]].sub2:vreg_128 = COPY [[COPY35]].sub2
+ ; CHECK-NEXT: undef [[COPY64:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY35]].sub0 {
+ ; CHECK-NEXT: internal [[COPY64]].sub2:vreg_128 = lr-split COPY [[COPY35]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY65:%[0-9]+]].sub0:vreg_128 = COPY [[COPY64]].sub0 {
- ; CHECK-NEXT: internal [[COPY65]].sub2:vreg_128 = COPY [[COPY64]].sub2
+ ; CHECK-NEXT: undef [[COPY65:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY64]].sub0 {
+ ; CHECK-NEXT: internal [[COPY65]].sub2:vreg_128 = lr-split COPY [[COPY64]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY65:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY65:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY65]], [[S_MOV_B32_]], 0, 272, 0, 0, implicit $exec :: (store (s128), addrspace 1)
- ; CHECK-NEXT: undef [[COPY66:%[0-9]+]].sub0:vreg_128 = COPY [[COPY33]].sub0 {
- ; CHECK-NEXT: internal [[COPY66]].sub2:vreg_128 = COPY [[COPY33]].sub2
+ ; CHECK-NEXT: undef [[COPY66:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY33]].sub0 {
+ ; CHECK-NEXT: internal [[COPY66]].sub2:vreg_128 = lr-split COPY [[COPY33]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY67:%[0-9]+]].sub0:vreg_128 = COPY [[COPY66]].sub0 {
- ; CHECK-NEXT: internal [[COPY67]].sub2:vreg_128 = COPY [[COPY66]].sub2
+ ; CHECK-NEXT: undef [[COPY67:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY66]].sub0 {
+ ; CHECK-NEXT: internal [[COPY67]].sub2:vreg_128 = lr-split COPY [[COPY66]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY67:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY67:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY67]], [[S_MOV_B32_]], 0, 224, 0, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
; CHECK-NEXT: [[SI_SPILL_V128_RESTORE:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.10, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.10, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY68:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V128_RESTORE]].sub0 {
- ; CHECK-NEXT: internal [[COPY68]].sub2:vreg_128 = COPY [[SI_SPILL_V128_RESTORE]].sub2
+ ; CHECK-NEXT: undef [[COPY68:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE]].sub0 {
+ ; CHECK-NEXT: internal [[COPY68]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY69:%[0-9]+]].sub0:vreg_128 = COPY [[COPY68]].sub0 {
- ; CHECK-NEXT: internal [[COPY69]].sub2:vreg_128 = COPY [[COPY68]].sub2
+ ; CHECK-NEXT: undef [[COPY69:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY68]].sub0 {
+ ; CHECK-NEXT: internal [[COPY69]].sub2:vreg_128 = lr-split COPY [[COPY68]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY69:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY69:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY69]], [[S_MOV_B32_]], 0, 240, 0, 0, implicit $exec :: (store (s128), addrspace 1)
- ; CHECK-NEXT: undef [[COPY70:%[0-9]+]].sub0:vreg_128 = COPY [[COPY29]].sub0 {
- ; CHECK-NEXT: internal [[COPY70]].sub2:vreg_128 = COPY [[COPY29]].sub2
+ ; CHECK-NEXT: undef [[COPY70:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY29]].sub0 {
+ ; CHECK-NEXT: internal [[COPY70]].sub2:vreg_128 = lr-split COPY [[COPY29]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY71:%[0-9]+]].sub0:vreg_128 = COPY [[COPY70]].sub0 {
- ; CHECK-NEXT: internal [[COPY71]].sub2:vreg_128 = COPY [[COPY70]].sub2
+ ; CHECK-NEXT: undef [[COPY71:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY70]].sub0 {
+ ; CHECK-NEXT: internal [[COPY71]].sub2:vreg_128 = lr-split COPY [[COPY70]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY71:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY71:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY71]], [[S_MOV_B32_]], 0, 192, 0, 0, implicit $exec :: (store (s128), align 64, addrspace 1)
; CHECK-NEXT: [[SI_SPILL_V128_RESTORE1:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.9, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.9, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY72:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V128_RESTORE1]].sub0 {
- ; CHECK-NEXT: internal [[COPY72]].sub2:vreg_128 = COPY [[SI_SPILL_V128_RESTORE1]].sub2
+ ; CHECK-NEXT: undef [[COPY72:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE1]].sub0 {
+ ; CHECK-NEXT: internal [[COPY72]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE1]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY73:%[0-9]+]].sub0:vreg_128 = COPY [[COPY72]].sub0 {
- ; CHECK-NEXT: internal [[COPY73]].sub2:vreg_128 = COPY [[COPY72]].sub2
+ ; CHECK-NEXT: undef [[COPY73:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY72]].sub0 {
+ ; CHECK-NEXT: internal [[COPY73]].sub2:vreg_128 = lr-split COPY [[COPY72]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY73:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY73:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY73]], [[S_MOV_B32_]], 0, 208, 0, 0, implicit $exec :: (store (s128), addrspace 1)
; CHECK-NEXT: [[SI_SPILL_V128_RESTORE2:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.11, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.11, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY74:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V128_RESTORE2]].sub0 {
- ; CHECK-NEXT: internal [[COPY74]].sub2:vreg_128 = COPY [[SI_SPILL_V128_RESTORE2]].sub2
+ ; CHECK-NEXT: undef [[COPY74:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE2]].sub0 {
+ ; CHECK-NEXT: internal [[COPY74]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE2]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY75:%[0-9]+]].sub0:vreg_128 = COPY [[COPY74]].sub0 {
- ; CHECK-NEXT: internal [[COPY75]].sub2:vreg_128 = COPY [[COPY74]].sub2
+ ; CHECK-NEXT: undef [[COPY75:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY74]].sub0 {
+ ; CHECK-NEXT: internal [[COPY75]].sub2:vreg_128 = lr-split COPY [[COPY74]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY75:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY75:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY75]], [[S_MOV_B32_]], 0, 160, 0, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
; CHECK-NEXT: [[SI_SPILL_V128_RESTORE3:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.8, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.8, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY76:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V128_RESTORE3]].sub0 {
- ; CHECK-NEXT: internal [[COPY76]].sub2:vreg_128 = COPY [[SI_SPILL_V128_RESTORE3]].sub2
+ ; CHECK-NEXT: undef [[COPY76:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE3]].sub0 {
+ ; CHECK-NEXT: internal [[COPY76]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE3]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY77:%[0-9]+]].sub0:vreg_128 = COPY [[COPY76]].sub0 {
- ; CHECK-NEXT: internal [[COPY77]].sub2:vreg_128 = COPY [[COPY76]].sub2
+ ; CHECK-NEXT: undef [[COPY77:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY76]].sub0 {
+ ; CHECK-NEXT: internal [[COPY77]].sub2:vreg_128 = lr-split COPY [[COPY76]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY77:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY77:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY77]], [[S_MOV_B32_]], 0, 176, 0, 0, implicit $exec :: (store (s128), addrspace 1)
- ; CHECK-NEXT: undef [[COPY78:%[0-9]+]].sub0:vreg_128 = COPY [[COPY21]].sub0 {
- ; CHECK-NEXT: internal [[COPY78]].sub2:vreg_128 = COPY [[COPY21]].sub2
+ ; CHECK-NEXT: undef [[COPY78:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY21]].sub0 {
+ ; CHECK-NEXT: internal [[COPY78]].sub2:vreg_128 = lr-split COPY [[COPY21]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY79:%[0-9]+]].sub0:vreg_128 = COPY [[COPY78]].sub0 {
- ; CHECK-NEXT: internal [[COPY79]].sub2:vreg_128 = COPY [[COPY78]].sub2
+ ; CHECK-NEXT: undef [[COPY79:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY78]].sub0 {
+ ; CHECK-NEXT: internal [[COPY79]].sub2:vreg_128 = lr-split COPY [[COPY78]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY79:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY79:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY79]], [[S_MOV_B32_]], 0, 128, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
- ; CHECK-NEXT: undef [[COPY80:%[0-9]+]].sub0:vreg_128 = COPY [[COPY19]].sub0 {
- ; CHECK-NEXT: internal [[COPY80]].sub2:vreg_128 = COPY [[COPY19]].sub2
+ ; CHECK-NEXT: undef [[COPY80:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY19]].sub0 {
+ ; CHECK-NEXT: internal [[COPY80]].sub2:vreg_128 = lr-split COPY [[COPY19]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY81:%[0-9]+]].sub0:vreg_128 = COPY [[COPY80]].sub0 {
- ; CHECK-NEXT: internal [[COPY81]].sub2:vreg_128 = COPY [[COPY80]].sub2
+ ; CHECK-NEXT: undef [[COPY81:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY80]].sub0 {
+ ; CHECK-NEXT: internal [[COPY81]].sub2:vreg_128 = lr-split COPY [[COPY80]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY81:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY81:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY81]], [[S_MOV_B32_]], 0, 144, 0, 0, implicit $exec :: (store (s128), addrspace 1)
; CHECK-NEXT: [[SI_SPILL_V128_RESTORE4:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.6, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.6, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY82:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V128_RESTORE4]].sub0 {
- ; CHECK-NEXT: internal [[COPY82]].sub2:vreg_128 = COPY [[SI_SPILL_V128_RESTORE4]].sub2
+ ; CHECK-NEXT: undef [[COPY82:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE4]].sub0 {
+ ; CHECK-NEXT: internal [[COPY82]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE4]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY83:%[0-9]+]].sub0:vreg_128 = COPY [[COPY82]].sub0 {
- ; CHECK-NEXT: internal [[COPY83]].sub2:vreg_128 = COPY [[COPY82]].sub2
+ ; CHECK-NEXT: undef [[COPY83:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY82]].sub0 {
+ ; CHECK-NEXT: internal [[COPY83]].sub2:vreg_128 = lr-split COPY [[COPY82]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY83:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY83:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY83]], [[S_MOV_B32_]], 0, 96, 0, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
; CHECK-NEXT: [[SI_SPILL_V128_RESTORE5:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.7, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.7, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY84:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V128_RESTORE5]].sub0 {
- ; CHECK-NEXT: internal [[COPY84]].sub2:vreg_128 = COPY [[SI_SPILL_V128_RESTORE5]].sub2
+ ; CHECK-NEXT: undef [[COPY84:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE5]].sub0 {
+ ; CHECK-NEXT: internal [[COPY84]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE5]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY85:%[0-9]+]].sub0:vreg_128 = COPY [[COPY84]].sub0 {
- ; CHECK-NEXT: internal [[COPY85]].sub2:vreg_128 = COPY [[COPY84]].sub2
+ ; CHECK-NEXT: undef [[COPY85:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY84]].sub0 {
+ ; CHECK-NEXT: internal [[COPY85]].sub2:vreg_128 = lr-split COPY [[COPY84]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY85:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY85:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY85]], [[S_MOV_B32_]], 0, 112, 0, 0, implicit $exec :: (store (s128), addrspace 1)
; CHECK-NEXT: [[SI_SPILL_V128_RESTORE6:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.5, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.5, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY86:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V128_RESTORE6]].sub0 {
- ; CHECK-NEXT: internal [[COPY86]].sub2:vreg_128 = COPY [[SI_SPILL_V128_RESTORE6]].sub2
+ ; CHECK-NEXT: undef [[COPY86:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE6]].sub0 {
+ ; CHECK-NEXT: internal [[COPY86]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE6]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY87:%[0-9]+]].sub0:vreg_128 = COPY [[COPY86]].sub0 {
- ; CHECK-NEXT: internal [[COPY87]].sub2:vreg_128 = COPY [[COPY86]].sub2
+ ; CHECK-NEXT: undef [[COPY87:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY86]].sub0 {
+ ; CHECK-NEXT: internal [[COPY87]].sub2:vreg_128 = lr-split COPY [[COPY86]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY87:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY87:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY87]], [[S_MOV_B32_]], 0, 64, 0, 0, implicit $exec :: (store (s128), align 64, addrspace 1)
; CHECK-NEXT: [[SI_SPILL_V128_RESTORE7:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.4, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.4, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY88:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V128_RESTORE7]].sub0 {
- ; CHECK-NEXT: internal [[COPY88]].sub2:vreg_128 = COPY [[SI_SPILL_V128_RESTORE7]].sub2
+ ; CHECK-NEXT: undef [[COPY88:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE7]].sub0 {
+ ; CHECK-NEXT: internal [[COPY88]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE7]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY89:%[0-9]+]].sub0:vreg_128 = COPY [[COPY88]].sub0 {
- ; CHECK-NEXT: internal [[COPY89]].sub2:vreg_128 = COPY [[COPY88]].sub2
+ ; CHECK-NEXT: undef [[COPY89:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY88]].sub0 {
+ ; CHECK-NEXT: internal [[COPY89]].sub2:vreg_128 = lr-split COPY [[COPY88]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY89:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY89:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY89]], [[S_MOV_B32_]], 0, 80, 0, 0, implicit $exec :: (store (s128), addrspace 1)
; CHECK-NEXT: [[SI_SPILL_V128_RESTORE8:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.3, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.3, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY90:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V128_RESTORE8]].sub0 {
- ; CHECK-NEXT: internal [[COPY90]].sub2:vreg_128 = COPY [[SI_SPILL_V128_RESTORE8]].sub2
+ ; CHECK-NEXT: undef [[COPY90:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE8]].sub0 {
+ ; CHECK-NEXT: internal [[COPY90]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE8]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY91:%[0-9]+]].sub0:vreg_128 = COPY [[COPY90]].sub0 {
- ; CHECK-NEXT: internal [[COPY91]].sub2:vreg_128 = COPY [[COPY90]].sub2
+ ; CHECK-NEXT: undef [[COPY91:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY90]].sub0 {
+ ; CHECK-NEXT: internal [[COPY91]].sub2:vreg_128 = lr-split COPY [[COPY90]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY91:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY91:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY91]], [[S_MOV_B32_]], 0, 32, 0, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
; CHECK-NEXT: [[SI_SPILL_V128_RESTORE9:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY92:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V128_RESTORE9]].sub0 {
- ; CHECK-NEXT: internal [[COPY92]].sub2:vreg_128 = COPY [[SI_SPILL_V128_RESTORE9]].sub2
+ ; CHECK-NEXT: undef [[COPY92:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE9]].sub0 {
+ ; CHECK-NEXT: internal [[COPY92]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE9]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY93:%[0-9]+]].sub0:vreg_128 = COPY [[COPY92]].sub0 {
- ; CHECK-NEXT: internal [[COPY93]].sub2:vreg_128 = COPY [[COPY92]].sub2
+ ; CHECK-NEXT: undef [[COPY93:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY92]].sub0 {
+ ; CHECK-NEXT: internal [[COPY93]].sub2:vreg_128 = lr-split COPY [[COPY92]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY93:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY93:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY93]], [[S_MOV_B32_]], 0, 48, 0, 0, implicit $exec :: (store (s128), addrspace 1)
; CHECK-NEXT: [[SI_SPILL_V128_RESTORE10:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.1, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.1, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY94:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V128_RESTORE10]].sub0 {
- ; CHECK-NEXT: internal [[COPY94]].sub2:vreg_128 = COPY [[SI_SPILL_V128_RESTORE10]].sub2
+ ; CHECK-NEXT: undef [[COPY94:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE10]].sub0 {
+ ; CHECK-NEXT: internal [[COPY94]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE10]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY95:%[0-9]+]].sub0:vreg_128 = COPY [[COPY94]].sub0 {
- ; CHECK-NEXT: internal [[COPY95]].sub2:vreg_128 = COPY [[COPY94]].sub2
+ ; CHECK-NEXT: undef [[COPY95:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY94]].sub0 {
+ ; CHECK-NEXT: internal [[COPY95]].sub2:vreg_128 = lr-split COPY [[COPY94]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY95:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY95:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY95]], [[S_MOV_B32_]], 0, 0, 0, 0, implicit $exec :: (store (s128), align 512, addrspace 1)
; CHECK-NEXT: [[SI_SPILL_V128_RESTORE11:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.0, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY96:%[0-9]+]].sub0:vreg_128 = COPY [[SI_SPILL_V128_RESTORE11]].sub0 {
- ; CHECK-NEXT: internal [[COPY96]].sub2:vreg_128 = COPY [[SI_SPILL_V128_RESTORE11]].sub2
+ ; CHECK-NEXT: undef [[COPY96:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE11]].sub0 {
+ ; CHECK-NEXT: internal [[COPY96]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE11]].sub2
; CHECK-NEXT: }
- ; CHECK-NEXT: undef [[COPY97:%[0-9]+]].sub0:vreg_128 = COPY [[COPY96]].sub0 {
- ; CHECK-NEXT: internal [[COPY97]].sub2:vreg_128 = COPY [[COPY96]].sub2
+ ; CHECK-NEXT: undef [[COPY97:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY96]].sub0 {
+ ; CHECK-NEXT: internal [[COPY97]].sub2:vreg_128 = lr-split COPY [[COPY96]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: [[COPY97:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY97:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-do-not-undo-subclass-split-with-remat.mir b/llvm/test/CodeGen/AMDGPU/splitkit-do-not-undo-subclass-split-with-remat.mir
index 9b4031826dd22..702be54f2eccf 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-do-not-undo-subclass-split-with-remat.mir
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-do-not-undo-subclass-split-with-remat.mir
@@ -50,8 +50,8 @@ body: |
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1065353216, implicit $exec
; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1073741824, implicit $exec
; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]]:vreg_1024_align2 = V_MFMA_F32_32X32X1F32_mac_vgprcd_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], [[GLOBAL_LOAD_DWORDX4_SADDR]], 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_32 = COPY [[V_MOV_B32_e32_1]]
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_32 = COPY [[V_MOV_B32_e32_]]
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_32 = lr-split COPY [[V_MOV_B32_e32_1]]
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_32 = lr-split COPY [[V_MOV_B32_e32_]]
; CHECK-NEXT: early-clobber %5:vreg_1024_align2 = V_MFMA_F32_32X32X1F32_vgprcd_e64 [[COPY2]], [[COPY1]], [[GLOBAL_LOAD_DWORDX4_SADDR]], 0, 0, 0, implicit $mode, implicit $exec, implicit $mode, implicit $exec
; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub2:vreg_1024_align2 = COPY %5.sub0
; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub3:vreg_1024_align2 = COPY %5.sub1
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit.mir b/llvm/test/CodeGen/AMDGPU/splitkit.mir
index dd3abf6007854..703e9dc7fb1a8 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit.mir
+++ b/llvm/test/CodeGen/AMDGPU/splitkit.mir
@@ -51,8 +51,8 @@ body: |
; CHECK-NEXT: S_NOP 0, implicit-def dead $sgpr0, implicit-def dead $sgpr1
; CHECK-NEXT: S_NOP 0, implicit renamable $sgpr4
; CHECK-NEXT: S_NOP 0, implicit renamable $sgpr6
- ; CHECK-NEXT: renamable $sgpr0 = COPY killed renamable $sgpr4
- ; CHECK-NEXT: renamable $sgpr2 = COPY renamable $sgpr6
+ ; CHECK-NEXT: renamable $sgpr0 = lr-split COPY killed renamable $sgpr4
+ ; CHECK-NEXT: renamable $sgpr2 = lr-split COPY renamable $sgpr6
; CHECK-NEXT: S_NOP 0, implicit-def dead $sgpr4, implicit-def dead $sgpr5, implicit-def dead $sgpr6, implicit-def dead $sgpr7, implicit-def dead $sgpr8, implicit-def dead $sgpr9, implicit-def dead $sgpr10, implicit-def dead $sgpr11, implicit-def dead $sgpr12, implicit-def dead $sgpr13, implicit-def dead $sgpr14, implicit-def dead $sgpr15, implicit-def dead $vcc_lo, implicit-def dead $vcc_hi
; CHECK-NEXT: S_NOP 0, implicit renamable $sgpr0
; CHECK-NEXT: S_NOP 0, implicit killed renamable $sgpr2
diff --git a/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir b/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir
index bc8a383a285b2..8b55224f15727 100644
--- a/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir
+++ b/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir
@@ -44,7 +44,7 @@ body: |
; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = SI_SPILL_AV128_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.2, align 4, addrspace 5)
; CHECK-NEXT: renamable $vgpr0 = SI_SPILL_AV32_RESTORE %stack.3, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.3, addrspace 5)
; CHECK-NEXT: renamable $vgpr6 = nofpexcept V_FMA_F32_e64 1, killed $vgpr0, 0, $vgpr6, 0, killed $vgpr2, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: renamable $vgpr2_vgpr3 = COPY killed renamable $vgpr6_vgpr7
+ ; CHECK-NEXT: renamable $vgpr2_vgpr3 = lr-split COPY killed renamable $vgpr6_vgpr7
; CHECK-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = SI_SPILL_AV128_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.2, align 4, addrspace 5)
; CHECK-NEXT: renamable $vgpr0 = IMPLICIT_DEF
; CHECK-NEXT: renamable $vgpr0 = nofpexcept V_DIV_FIXUP_F32_e64 0, killed $vgpr0, 0, $vgpr4, 0, $vgpr6, 0, 0, implicit $mode, implicit $exec
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/lr-split-flag.mir b/llvm/test/CodeGen/MIR/AMDGPU/lr-split-flag.mir
new file mode 100644
index 0000000000000..ed1680e3545c2
--- /dev/null
+++ b/llvm/test/CodeGen/MIR/AMDGPU/lr-split-flag.mir
@@ -0,0 +1,25 @@
+# RUN: llc -mtriple=amdgcn -mcpu=gfx950 -run-pass=none -o - %s | FileCheck %s
+#
+# Round-trip test for the 'lr-split' MIR text spelling of
+# MachineInstr::LRSplit. The flag is set by SplitKit on copies inserted for
+# live-range splitting; AMDGPU's SIInstrInfo::canAddToBBProlog uses it to
+# recognize WWM_COPY / IMPLICIT_DEF instructions as part of the BB prologue.
+
+---
+name: lr_split_roundtrip
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+registers:
+ - { id: 0, class: vgpr_32, flags: [ WWM_REG ] }
+ - { id: 1, class: vgpr_32, flags: [ WWM_REG ] }
+body: |
+ bb.0:
+ liveins: $vgpr0
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = lr-split WWM_COPY %0:vgpr_32
+ S_ENDPGM 0, implicit %1
+...
+
+# CHECK-LABEL: name: lr_split_roundtrip
+# CHECK: %1:vgpr_32 = lr-split WWM_COPY %0
diff --git a/llvm/test/CodeGen/RISCV/pr176001.ll b/llvm/test/CodeGen/RISCV/pr176001.ll
index d81bb46ff5c7b..ffc1658558975 100644
--- a/llvm/test/CodeGen/RISCV/pr176001.ll
+++ b/llvm/test/CodeGen/RISCV/pr176001.ll
@@ -42,14 +42,14 @@ define <32 x i64> @main(i1 %tobool93.not, <32 x i64> %0, <32 x i64> %1) #0 {
; CHECK-NEXT: successors: %bb.7(0x80000000)
; CHECK-NEXT: liveins: $v8m8:0x0000000000000002, $v16m8:0x0000000000000006
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $v0 = COPY killed renamable $v8
+ ; CHECK-NEXT: renamable $v0 = lr-split COPY killed renamable $v8
; CHECK-NEXT: PseudoBR %bb.7
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.6.cond.false111:
; CHECK-NEXT: successors: %bb.7(0x80000000)
; CHECK-NEXT: liveins: $v8m8:0x0000000000000002, $v16m8:0x0000000000000006
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $v0 = COPY killed renamable $v8
+ ; CHECK-NEXT: renamable $v0 = lr-split COPY killed renamable $v8
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.7.cond.end113:
; CHECK-NEXT: successors: %bb.8(0x40000000), %bb.9(0x40000000)
diff --git a/llvm/test/CodeGen/X86/AMX/amx-greedy-ra-spill-shape.ll b/llvm/test/CodeGen/X86/AMX/amx-greedy-ra-spill-shape.ll
index cba394f9e55d6..9a93b52f2d74c 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-greedy-ra-spill-shape.ll
+++ b/llvm/test/CodeGen/X86/AMX/amx-greedy-ra-spill-shape.ll
@@ -70,7 +70,7 @@ define void @foo(i32 %M, i32 %N, i32 %K, ptr %A, ptr %B_rcr4, ptr %C, i32 %c_row
; CHECK-NEXT: [[COPY6:%[0-9]+]]:gr64_nosp = ADD64rr [[COPY6]], [[MOVSX64rm32_]], implicit-def dead $eflags
; CHECK-NEXT: [[LEA64r:%[0-9]+]]:gr64 = LEA64r [[COPY]], 4, [[COPY6]], 0, $noreg
; CHECK-NEXT: MOV64mr %stack.9, 1, $noreg, 0, $noreg, [[LEA64r]] :: (store (s64) into %stack.9)
- ; CHECK-NEXT: [[COPY7:%[0-9]+]]:gr64_with_sub_8bit = COPY [[MOV32rm2]]
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:gr64_with_sub_8bit = lr-split COPY [[MOV32rm2]]
; CHECK-NEXT: MOV64mr %stack.7, 1, $noreg, 0, $noreg, [[MOVSX64rr32_5]] :: (store (s64) into %stack.7)
; CHECK-NEXT: [[COPY8:%[0-9]+]]:gr64 = COPY [[MOVSX64rr32_5]]
; CHECK-NEXT: [[COPY8:%[0-9]+]]:gr64 = IMUL64rr [[COPY8]], [[MOVSX64rr32_2]], implicit-def dead $eflags
@@ -91,7 +91,7 @@ define void @foo(i32 %M, i32 %N, i32 %K, ptr %A, ptr %B_rcr4, ptr %C, i32 %c_row
; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64_nosp = MOV64rm %stack.3, 1, $noreg, 0, $noreg :: (load (s64) from %stack.3)
; CHECK-NEXT: [[MOV64rm1:%[0-9]+]]:gr64 = MOV64rm %stack.11, 1, $noreg, 0, $noreg :: (load (s64) from %stack.11)
; CHECK-NEXT: [[MOV32rm6:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.3, 1, $noreg, 0, $noreg :: (load (s32) from %fixed-stack.3, align 16)
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:gr32 = COPY [[MOV32rm6]]
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:gr32 = lr-split COPY [[MOV32rm6]]
; CHECK-NEXT: JCC_1 %bb.5, 13, implicit $eflags
; CHECK-NEXT: JMP_1 %bb.3
; CHECK-NEXT: {{ $}}
@@ -128,21 +128,21 @@ define void @foo(i32 %M, i32 %N, i32 %K, ptr %A, ptr %B_rcr4, ptr %C, i32 %c_row
; CHECK-NEXT: [[PTILEZEROV:%[0-9]+]]:tile = PTILEZEROV [[COPY9]].sub_16bit, [[COPY7]].sub_16bit
; CHECK-NEXT: [[PTILELOADDV:%[0-9]+]]:tile = PTILELOADDV [[COPY9]].sub_16bit, [[COPY4]].sub_16bit, [[MOV64rm2]], 1, [[MOVSX64rr32_]], 0, $noreg
; CHECK-NEXT: [[COPY10:%[0-9]+]]:gr64_nosp = MOVSX64rr32 [[COPY10]].sub_32bit
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:gr64 = COPY [[MOVSX64rm32_1]]
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:gr64 = COPY [[MOVSX64rr32_3]]
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:gr64 = COPY [[MOVSX64rr32_2]]
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:gr64 = COPY [[MOVSX64rr32_]]
- ; CHECK-NEXT: [[COPY15:%[0-9]+]]:gr64 = COPY [[COPY7]]
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:gr64 = lr-split COPY [[MOVSX64rm32_1]]
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:gr64 = lr-split COPY [[MOVSX64rr32_3]]
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:gr64 = lr-split COPY [[MOVSX64rr32_2]]
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:gr64 = lr-split COPY [[MOVSX64rr32_]]
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]]:gr64 = lr-split COPY [[COPY7]]
; CHECK-NEXT: [[MOV64rm7:%[0-9]+]]:gr64 = MOV64rm %stack.5, 1, $noreg, 0, $noreg :: (load (s64) from %stack.5)
; CHECK-NEXT: [[LEA64r2:%[0-9]+]]:gr64 = LEA64r [[MOV64rm7]], 1, [[COPY10]], 0, $noreg
; CHECK-NEXT: [[PTILELOADDV1:%[0-9]+]]:tile = PTILELOADDV [[MOV32rm5]].sub_16bit, [[LEA64_32r1]].sub_16bit, [[LEA64r2]], 1, [[MOV64rm]], 0, $noreg
- ; CHECK-NEXT: [[COPY7:%[0-9]+]]:gr64_with_sub_8bit = COPY [[COPY15]]
- ; CHECK-NEXT: [[MOVSX64rr32_:%[0-9]+]]:gr64_nosp = COPY [[COPY14]]
- ; CHECK-NEXT: [[MOVSX64rr32_2:%[0-9]+]]:gr64_nosp = COPY [[COPY13]]
- ; CHECK-NEXT: [[MOVSX64rr32_3:%[0-9]+]]:gr64_nosp = COPY [[COPY12]]
- ; CHECK-NEXT: [[MOVSX64rm32_1:%[0-9]+]]:gr64 = COPY [[COPY11]]
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:gr64_with_sub_8bit = lr-split COPY [[COPY15]]
+ ; CHECK-NEXT: [[MOVSX64rr32_:%[0-9]+]]:gr64_nosp = lr-split COPY [[COPY14]]
+ ; CHECK-NEXT: [[MOVSX64rr32_2:%[0-9]+]]:gr64_nosp = lr-split COPY [[COPY13]]
+ ; CHECK-NEXT: [[MOVSX64rr32_3:%[0-9]+]]:gr64_nosp = lr-split COPY [[COPY12]]
+ ; CHECK-NEXT: [[MOVSX64rm32_1:%[0-9]+]]:gr64 = lr-split COPY [[COPY11]]
; CHECK-NEXT: [[MOV32rm8:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.3, 1, $noreg, 0, $noreg :: (load (s32) from %fixed-stack.3, align 16)
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:gr32 = COPY [[MOV32rm8]]
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:gr32 = lr-split COPY [[MOV32rm8]]
; CHECK-NEXT: [[PTILEZEROV:%[0-9]+]]:tile = PTDPBSSDV [[COPY9]].sub_16bit, [[LEA64_32r1]].sub_16bit, [[COPY4]].sub_16bit, [[PTILEZEROV]], [[PTILELOADDV]], [[PTILELOADDV1]]
; CHECK-NEXT: PTILESTOREDV [[COPY9]].sub_16bit, [[COPY7]].sub_16bit, [[MOV64rm3]], 1, [[MOVSX64rr32_2]], 0, $noreg, [[PTILEZEROV]]
; CHECK-NEXT: [[MOV64rm4:%[0-9]+]]:gr64 = ADD64rr [[MOV64rm4]], [[MOVSX64rr32_3]], implicit-def dead $eflags
diff --git a/llvm/test/CodeGen/X86/apx/foldmemory.mir b/llvm/test/CodeGen/X86/apx/foldmemory.mir
index 3cdaf613e9431..80b100c8d6215 100644
--- a/llvm/test/CodeGen/X86/apx/foldmemory.mir
+++ b/llvm/test/CodeGen/X86/apx/foldmemory.mir
@@ -49,20 +49,20 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
; CHECK-NEXT: $xmm0 = FsFLD0SS
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gr64 = COPY [[COPY3]]
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gr64 = lr-split COPY [[COPY3]]
; CHECK-NEXT: $rdi = COPY [[COPY5]]
- ; CHECK-NEXT: [[COPY6:%[0-9]+]]:gr64 = COPY [[COPY2]]
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:gr64 = lr-split COPY [[COPY2]]
; CHECK-NEXT: dead $esi = MOV32r0 implicit-def dead $eflags, implicit-def $rsi
- ; CHECK-NEXT: [[COPY7:%[0-9]+]]:gr64 = COPY [[COPY1]]
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:gr64 = lr-split COPY [[COPY1]]
; CHECK-NEXT: dead $edx = MOV32r0 implicit-def dead $eflags, implicit-def $rdx
- ; CHECK-NEXT: [[COPY8:%[0-9]+]]:gr32 = COPY [[COPY]]
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:gr32 = lr-split COPY [[COPY]]
; CHECK-NEXT: dead $ecx = MOV32r0 implicit-def dead $eflags, implicit-def $rcx
; CHECK-NEXT: undef [[MOV32r0_1:%[0-9]+]].sub_32bit:gr64_with_sub_8bit = MOV32r0 implicit-def dead $eflags
; CHECK-NEXT: CALL64r [[MOV32r0_1]], csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx, implicit $rcx, implicit killed $xmm0, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr64 = COPY [[COPY5]]
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64 = COPY [[COPY6]]
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY [[COPY7]]
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY [[COPY8]]
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr64 = lr-split COPY [[COPY5]]
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64 = lr-split COPY [[COPY6]]
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = lr-split COPY [[COPY7]]
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = lr-split COPY [[COPY8]]
; CHECK-NEXT: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %stack.2, 1, $noreg, 0, $noreg :: (load (s64) from %stack.2)
; CHECK-NEXT: MOV64mi32 [[MOV64rm]], 8, [[MOV32r0_]], 0, $noreg, 0 :: (store (s64))
diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir
index 23be8bf4a058b..4accaab7f6172 100644
--- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir
+++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir
@@ -289,7 +289,7 @@ body: |
; CHECK-NEXT: successors:
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64 = COPY [[COPY1]]
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64 = lr-split COPY [[COPY1]]
; CHECK-NEXT: dead [[COPY2:%[0-9]+]]:gr64, dead [[COPY:%[0-9]+]]:gr64 = STATEPOINT 2882400000, 0, 1, target-flags(x86-plt) @barney, undef $rdi, 2, 0, 2, 0, 2, 45, 2, 0, 2, 2, 2, 0, 2, 0, 2, 0, 2, 1, 2, 0, 2, 7, 2, 0, 2, 2, 2, 1, 2, 71, 2, 0, 2, 5, 2, 0, 2, 0, [[COPY2]], 2, 7, 2, 0, 2, 7, 2, 0, 2, 7, 2, 0, 2, 7, 2, 0, 2, 2, 2, 5, 2, 1, 2, 0, 2, 2, 2, 0, 2, 0, [[COPY]], 2, 7, 2, 0, 2, 1, 2, 6, 2, 0, 2, 0, 2, 1, 2, 1, 2, 0, [[COPY]], 2, 8, 2, 10, 2, 2, [[COPY2]](tied-def 0), [[COPY]](tied-def 1), 2, 0, 2, 2, 0, 0, 1, 1, csr_64, implicit-def $rsp, implicit-def $ssp
; CHECK-NEXT: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
; CHECK-NEXT: {{ $}}
@@ -301,7 +301,7 @@ body: |
; CHECK-NEXT: EH_LABEL <mcsymbol .Ltmp0>
; CHECK-NEXT: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
; CHECK-NEXT: $edi = COPY [[MOV32rm]]
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr64 = COPY [[COPY1]]
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr64 = lr-split COPY [[COPY1]]
; CHECK-NEXT: dead [[STATEPOINT1:%[0-9]+]]:gr64, [[COPY3:%[0-9]+]]:gr64, dead [[STATEPOINT:%[0-9]+]]:gr64, [[COPY:%[0-9]+]]:gr64 = STATEPOINT 1, 16, 2, undef %38:gr64, $edi, undef $rsi, 2, 0, 2, 0, 2, 35, 2, 0, 2, 2, 2, 0, 2, 0, 2, 0, 2, 1, 2, 0, 2, 7, 2, 0, 2, 2, 2, 1, 2, 71, 2, 0, 2, 5, 2, 0, 2, 0, [[COPY3]], 2, 7, 2, 0, 2, 7, 2, 0, 2, 7, 2, 0, 2, 7, 2, 0, 2, 8, 2, 5, 2, 12, 2, 0, 2, 2, 2, 0, 2, 0, [[COPY]], 2, 7, 2, 0, 2, 4, undef [[STATEPOINT1]](tied-def 0), [[COPY3]](tied-def 1), [[STATEPOINT]](tied-def 2), [[COPY]](tied-def 3), 2, 0, 2, 4, 0, 0, 1, 1, 2, 2, 3, 3, csr_64, implicit-def $rsp, implicit-def $ssp
; CHECK-NEXT: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
; CHECK-NEXT: EH_LABEL <mcsymbol .Ltmp1>
@@ -317,7 +317,7 @@ body: |
; CHECK-NEXT: dead [[MOV64rm1:%[0-9]+]]:gr64, [[COPY3:%[0-9]+]]:gr64 = STATEPOINT 2, 5, 1, undef %50:gr64, undef $rdi, 2, 0, 2, 0, 2, 27, 2, 0, 2, 2, 2, 0, 2, 0, 2, 0, 2, 1, 2, 0, 2, 7, 2, 0, 2, 10, 2, 1, 2, 83, 2, 0, 2, 5, 2, 1, 2, 0, [[COPY3]], 2, 7, 2, 0, 2, 8, 2, 2, 2, 7, 2, 0, 2, 7, 2, 0, 2, 8, 2, 2, 2, 2, [[MOV64rm1]](tied-def 0), [[COPY3]](tied-def 1), 2, 0, 2, 2, 0, 0, 1, 1, csr_64, implicit-def $rsp, implicit-def $ssp
; CHECK-NEXT: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
; CHECK-NEXT: TEST64rr [[COPY3]], [[COPY3]], implicit-def $eflags
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr64 = COPY [[COPY3]]
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr64 = lr-split COPY [[COPY3]]
; CHECK-NEXT: JCC_1 %bb.9, 4, implicit killed $eflags
; CHECK-NEXT: JMP_1 %bb.6
; CHECK-NEXT: {{ $}}
@@ -328,11 +328,11 @@ body: |
; CHECK-NEXT: [[MOV64rm3:%[0-9]+]]:gr64 = MOV64rm [[MOV64rm2]], 1, $noreg, 0, $noreg :: (dereferenceable load unordered (s64) from @global.1)
; CHECK-NEXT: [[MOV32rm1:%[0-9]+]]:gr32 = MOV32rm $noreg, 1, $noreg, 660, $gs :: (load (s32) from `ptr addrspace(256) inttoptr (i64 660 to ptr addrspace(256))`, addrspace 256)
; CHECK-NEXT: [[MOV64rm3:%[0-9]+]]:gr64 = NOT64r [[MOV64rm3]]
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gr64 = COPY [[MOV64rm3]]
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gr64 = lr-split COPY [[MOV64rm3]]
; CHECK-NEXT: [[MOV32rm1:%[0-9]+]]:gr32 = OR32ri [[MOV32rm1]], 268435456, implicit-def dead $eflags
- ; CHECK-NEXT: [[COPY6:%[0-9]+]]:gr32 = COPY [[MOV32rm1]]
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:gr32 = lr-split COPY [[MOV32rm1]]
; CHECK-NEXT: [[DEF:%[0-9]+]]:gr64 = IMPLICIT_DEF
- ; CHECK-NEXT: [[COPY7:%[0-9]+]]:gr64 = COPY [[COPY4]]
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:gr64 = lr-split COPY [[COPY4]]
; CHECK-NEXT: undef [[MOV32r0_1:%[0-9]+]].sub_32bit:gr64_with_sub_8bit = MOV32r0 implicit-def dead $eflags
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.7.bb33:
diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir
index 62f7ee096a3ff..8d38d687edaad 100644
--- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir
+++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir
@@ -205,7 +205,7 @@ body: |
; CHECK-NEXT: successors: %bb.1(0x80000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
- ; CHECK-NEXT: undef %39.sub_32bit:gr64_with_sub_8bit = MOV32r0 implicit-def dead $eflags
+ ; CHECK-NEXT: undef [[MOV32r0_:%[0-9]+]].sub_32bit:gr64_with_sub_8bit = MOV32r0 implicit-def dead $eflags
; CHECK-NEXT: dead $edi = MOV32r0 implicit-def dead $eflags, implicit-def $rdi
; CHECK-NEXT: STATEPOINT 2, 5, 2, undef %24:gr64, $rdi, undef $rsi, 2, 0, 2, 0, 2, 37, 2, 0, 2, 2, 2, 0, 2, 43, 2, 0, 2, 2, 2, 0, 2, 7, 2, 0, 2, 7, 2, 0, 2, 10, 2, 1, 2, 6, 2, 0, 2, 4, 2, 1, 2, 0, 2, 0, 2, 7, 2, 0, 2, 0, 2, 0, 2, 7, 2, 0, 2, 0, 2, 0, 2, 2, 2, 4, 2, 5, 2, 0, 2, 2, 2, 0, 2, 7, 2, 0, 2, 7, 2, 0, 2, 1, 2, 0, 2, 0, 2, 1, 0, 0, csr_64, implicit-def $rsp, implicit-def $ssp
; CHECK-NEXT: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
@@ -213,33 +213,33 @@ body: |
; CHECK-NEXT: [[DEF1:%[0-9]+]]:gr64 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF2:%[0-9]+]]:gr64 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF3:%[0-9]+]]:gr64 = IMPLICIT_DEF
- ; CHECK-NEXT: [[MOV32r0_:%[0-9]+]]:gr32 = MOV32r0 implicit-def dead $eflags
+ ; CHECK-NEXT: [[MOV32r0_1:%[0-9]+]]:gr32 = MOV32r0 implicit-def dead $eflags
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1.bb1:
; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.4(0x40000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: undef %66.sub_32bit:gr64_nosp = COPY [[MOV32r0_]]
- ; CHECK-NEXT: undef %65.sub_32bit:gr64_with_sub_8bit = MOV32rm undef %31:gr64, 1, $noreg, 0, $noreg :: (load unordered (s32) from `ptr addrspace(1) undef`, align 8, addrspace 1)
- ; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm [[DEF]], 1, $noreg, 0, $noreg :: (load unordered (s32) from %ir.tmp2, addrspace 1)
- ; CHECK-NEXT: %65.sub_32bit:gr64_with_sub_8bit = SUB32rr %65.sub_32bit, [[MOV32rm]], implicit-def dead $eflags
- ; CHECK-NEXT: [[LEA64_32r:%[0-9]+]]:gr32 = LEA64_32r %65, 1, %66, 0, $noreg
- ; CHECK-NEXT: MOV32mr %stack.0, 1, $noreg, 0, $noreg, %66.sub_32bit :: (store (s32) into %stack.0)
- ; CHECK-NEXT: MOV32mr %stack.1, 1, $noreg, 0, $noreg, %65.sub_32bit :: (store (s32) into %stack.1)
+ ; CHECK-NEXT: undef [[COPY:%[0-9]+]].sub_32bit:gr64_nosp = COPY [[MOV32r0_1]]
+ ; CHECK-NEXT: undef [[MOV32rm:%[0-9]+]].sub_32bit:gr64_with_sub_8bit = MOV32rm undef %31:gr64, 1, $noreg, 0, $noreg :: (load unordered (s32) from `ptr addrspace(1) undef`, align 8, addrspace 1)
+ ; CHECK-NEXT: [[MOV32rm1:%[0-9]+]]:gr32 = MOV32rm [[DEF]], 1, $noreg, 0, $noreg :: (load unordered (s32) from %ir.tmp2, addrspace 1)
+ ; CHECK-NEXT: [[MOV32rm:%[0-9]+]].sub_32bit:gr64_with_sub_8bit = SUB32rr [[MOV32rm]].sub_32bit, [[MOV32rm1]], implicit-def dead $eflags
+ ; CHECK-NEXT: [[LEA64_32r:%[0-9]+]]:gr32 = LEA64_32r [[MOV32rm]], 1, [[COPY]], 0, $noreg
+ ; CHECK-NEXT: MOV32mr %stack.0, 1, $noreg, 0, $noreg, [[COPY]].sub_32bit :: (store (s32) into %stack.0)
+ ; CHECK-NEXT: MOV32mr %stack.1, 1, $noreg, 0, $noreg, [[MOV32rm]].sub_32bit :: (store (s32) into %stack.1)
; CHECK-NEXT: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
; CHECK-NEXT: $rdi = COPY [[DEF2]]
- ; CHECK-NEXT: $esi = COPY %66.sub_32bit
+ ; CHECK-NEXT: $esi = COPY [[COPY]].sub_32bit
; CHECK-NEXT: $edx = COPY [[LEA64_32r]]
- ; CHECK-NEXT: $r8d = COPY [[MOV32rm]]
- ; CHECK-NEXT: [[STATEPOINT:%[0-9]+]]:gr64, [[STATEPOINT1:%[0-9]+]]:gr64, [[STATEPOINT2:%[0-9]+]]:gr64, [[STATEPOINT3:%[0-9]+]]:gr64_with_sub_8bit = STATEPOINT 2, 5, 5, undef %35:gr64, $rdi, $esi, $edx, undef $rcx, $r8d, 2, 0, 2, 0, 2, 85, 2, 0, 2, 2, 2, 0, 2, 43, 2, 0, 2, 2, 2, 0, 2, 7, 2, 0, 2, 7, 2, 0, 2, 10, 2, 1, 2, 10, 2, 0, 2, 4, 2, 1, 2, 7, 2, 0, 2, 7, 2, 0, 2, 0, [[STATEPOINT3]], 2, 7, 2, 0, 2, 0, [[STATEPOINT3]], 2, 10, 2, 5, 2, 12, 2, 0, 2, 3, 2, 1, 2, 0, [[STATEPOINT3]], 2, 0, [[STATEPOINT3]], 2, 7, 2, 0, 2, 0, [[STATEPOINT3]], 2, 2, 2, 11, 2, 4, 2, 0, 2, 1, 2, 0, 2, 7, 2, 0, 2, 2, 2, 15, 2, 7, 2, 0, 2, 2, 2, 0, 2, 7, 2, 0, 2, 7, 2, 0, 2, 10, 2, 21, 2, 63, 2, 0, 2, 9, 2, 1, 2, 0, [[STATEPOINT2]], 2, 0, [[STATEPOINT1]], 2, 7, 2, 0, 2, 7, 2, 0, 2, 0, [[STATEPOINT]], 2, 3, 1, 4, %stack.0, 0, 2, 3, 2, 4278124286, 2, 3, 1, 4, %stack.1, 0, 2, 7, 2, 0, 2, 0, [[STATEPOINT]], 2, 4, [[STATEPOINT]](tied-def 0), [[STATEPOINT1]](tied-def 1), [[STATEPOINT2]](tied-def 2), [[STATEPOINT3]](tied-def 3), 2, 0, 2, 4, 0, 0, 1, 1, 2, 2, 3, 3, csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store (s32) on %stack.0), (volatile load store (s32) on %stack.1)
+ ; CHECK-NEXT: $r8d = COPY [[MOV32rm1]]
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:gr64, [[DEF2:%[0-9]+]]:gr64, [[DEF3:%[0-9]+]]:gr64, [[MOV32r0_:%[0-9]+]]:gr64_with_sub_8bit = STATEPOINT 2, 5, 5, undef %35:gr64, $rdi, $esi, $edx, undef $rcx, $r8d, 2, 0, 2, 0, 2, 85, 2, 0, 2, 2, 2, 0, 2, 43, 2, 0, 2, 2, 2, 0, 2, 7, 2, 0, 2, 7, 2, 0, 2, 10, 2, 1, 2, 10, 2, 0, 2, 4, 2, 1, 2, 7, 2, 0, 2, 7, 2, 0, 2, 0, [[MOV32r0_]], 2, 7, 2, 0, 2, 0, [[MOV32r0_]], 2, 10, 2, 5, 2, 12, 2, 0, 2, 3, 2, 1, 2, 0, [[MOV32r0_]], 2, 0, [[MOV32r0_]], 2, 7, 2, 0, 2, 0, [[MOV32r0_]], 2, 2, 2, 11, 2, 4, 2, 0, 2, 1, 2, 0, 2, 7, 2, 0, 2, 2, 2, 15, 2, 7, 2, 0, 2, 2, 2, 0, 2, 7, 2, 0, 2, 7, 2, 0, 2, 10, 2, 21, 2, 63, 2, 0, 2, 9, 2, 1, 2, 0, [[DEF3]], 2, 0, [[DEF2]], 2, 7, 2, 0, 2, 7, 2, 0, 2, 0, [[DEF1]], 2, 3, 1, 4, %stack.0, 0, 2, 3, 2, 4278124286, 2, 3, 1, 4, %stack.1, 0, 2, 7, 2, 0, 2, 0, [[DEF1]], 2, 4, [[DEF1]](tied-def 0), [[DEF2]](tied-def 1), [[DEF3]](tied-def 2), [[MOV32r0_]](tied-def 3), 2, 0, 2, 4, 0, 0, 1, 1, 2, 2, 3, 3, csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store (s32) on %stack.0), (volatile load store (s32) on %stack.1)
; CHECK-NEXT: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
- ; CHECK-NEXT: CMP32rr %65.sub_32bit, undef %37:gr32, implicit-def $eflags
+ ; CHECK-NEXT: CMP32rr [[MOV32rm]].sub_32bit, undef %37:gr32, implicit-def $eflags
; CHECK-NEXT: JCC_1 %bb.4, 13, implicit killed $eflags
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
; CHECK-NEXT: successors: %bb.3(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[DEF1:%[0-9]+]]:gr64 = IMPLICIT_DEF
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY [[LEA64_32r]]
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:gr64 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[MOV32r0_1:%[0-9]+]]:gr32 = lr-split COPY [[LEA64_32r]]
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3.bb21:
; CHECK-NEXT: successors: %bb.1(0x80000000)
@@ -253,8 +253,8 @@ body: |
; CHECK-NEXT: EH_LABEL <mcsymbol .Ltmp0>
; CHECK-NEXT: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
; CHECK-NEXT: $ecx = MOV32r0 implicit-def dead $eflags
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY [[LEA64_32r]]
- ; CHECK-NEXT: [[STATEPOINT2]]:gr64, [[STATEPOINT3]]:gr64_with_sub_8bit, [[STATEPOINT]]:gr64, dead [[STATEPOINT1]]:gr64 = STATEPOINT 1, 16, 5, undef %47:gr64, undef $edi, undef $rsi, undef $rdx, $ecx, undef $r8d, 2, 0, 2, 0, 2, 99, 2, 0, 2, 2, 2, 0, 2, 43, 2, 0, 2, 2, 2, 0, 2, 7, 2, 0, 2, 7, 2, 0, 2, 10, 2, 1, 2, 10, 2, 0, 2, 4, 2, 1, 2, 7, 2, 0, 2, 7, 2, 0, 2, 0, [[STATEPOINT3]], 2, 7, 2, 0, 2, 0, [[STATEPOINT3]], 2, 10, 2, 5, 2, 12, 2, 0, 2, 3, 2, 1, 2, 0, [[STATEPOINT3]], 2, 0, [[STATEPOINT3]], 2, 7, 2, 0, 2, 0, [[STATEPOINT3]], 2, 2, 2, 11, 2, 4, 2, 0, 2, 1, 2, 0, 2, 7, 2, 0, 2, 2, 2, 15, 2, 7, 2, 0, 2, 2, 2, 0, 2, 7, 2, 0, 2, 7, 2, 0, 2, 10, 2, 21, 2, 96, 2, 0, 2, 9, 2, 1, 2, 0, [[STATEPOINT2]], 2, 0, [[STATEPOINT1]], 2, 7, 2, 0, 2, 7, 2, 0, 2, 0, [[STATEPOINT]], 2, 3, 1, 4, %stack.0, 0, 2, 3, 2, 4278124286, 2, 7, 2, 0, 2, 7, 2, 0, 2, 0, [[STATEPOINT]], 2, 8, 2, 12, 2, 34, 2, 0, 2, 3, 2, 1, 2, 0, [[STATEPOINT2]], 2, 0, 2, 4278124286, 2, 7, 2, 0, 2, 0, 2, 4278124286, 2, 5, [[STATEPOINT2]](tied-def 0), [[STATEPOINT3]](tied-def 1), [[STATEPOINT]](tied-def 2), [[STATEPOINT1]](tied-def 3), 2, 4278124286, 2, 0, 2, 5, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store (s32) on %stack.0)
+ ; CHECK-NEXT: [[MOV32r0_1:%[0-9]+]]:gr32 = lr-split COPY [[LEA64_32r]]
+ ; CHECK-NEXT: [[DEF3:%[0-9]+]]:gr64, [[MOV32r0_:%[0-9]+]]:gr64_with_sub_8bit, [[DEF1:%[0-9]+]]:gr64, dead [[DEF2:%[0-9]+]]:gr64 = STATEPOINT 1, 16, 5, undef %47:gr64, undef $edi, undef $rsi, undef $rdx, $ecx, undef $r8d, 2, 0, 2, 0, 2, 99, 2, 0, 2, 2, 2, 0, 2, 43, 2, 0, 2, 2, 2, 0, 2, 7, 2, 0, 2, 7, 2, 0, 2, 10, 2, 1, 2, 10, 2, 0, 2, 4, 2, 1, 2, 7, 2, 0, 2, 7, 2, 0, 2, 0, [[MOV32r0_]], 2, 7, 2, 0, 2, 0, [[MOV32r0_]], 2, 10, 2, 5, 2, 12, 2, 0, 2, 3, 2, 1, 2, 0, [[MOV32r0_]], 2, 0, [[MOV32r0_]], 2, 7, 2, 0, 2, 0, [[MOV32r0_]], 2, 2, 2, 11, 2, 4, 2, 0, 2, 1, 2, 0, 2, 7, 2, 0, 2, 2, 2, 15, 2, 7, 2, 0, 2, 2, 2, 0, 2, 7, 2, 0, 2, 7, 2, 0, 2, 10, 2, 21, 2, 96, 2, 0, 2, 9, 2, 1, 2, 0, [[DEF3]], 2, 0, [[DEF2]], 2, 7, 2, 0, 2, 7, 2, 0, 2, 0, [[DEF1]], 2, 3, 1, 4, %stack.0, 0, 2, 3, 2, 4278124286, 2, 7, 2, 0, 2, 7, 2, 0, 2, 0, [[DEF1]], 2, 8, 2, 12, 2, 34, 2, 0, 2, 3, 2, 1, 2, 0, [[DEF3]], 2, 0, 2, 4278124286, 2, 7, 2, 0, 2, 0, 2, 4278124286, 2, 5, [[DEF3]](tied-def 0), [[MOV32r0_]](tied-def 1), [[DEF1]](tied-def 2), [[DEF2]](tied-def 3), 2, 4278124286, 2, 0, 2, 5, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store (s32) on %stack.0)
; CHECK-NEXT: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
; CHECK-NEXT: EH_LABEL <mcsymbol .Ltmp1>
; CHECK-NEXT: JMP_1 %bb.5
@@ -262,14 +262,14 @@ body: |
; CHECK-NEXT: bb.5.bb30:
; CHECK-NEXT: successors: %bb.3(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64 = COPY [[STATEPOINT2]]
- ; CHECK-NEXT: [[ADD64ri8_:%[0-9]+]]:gr64 = nuw ADD64ri8 [[ADD64ri8_]], 28, implicit-def dead $eflags
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY [[DEF3]]
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = nuw ADD64ri8 [[COPY1]], 28, implicit-def dead $eflags
; CHECK-NEXT: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
; CHECK-NEXT: CALL64pcrel32 target-flags(x86-plt) @barney, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr64 = COPY [[ADD64ri8_]]
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:gr64 = lr-split COPY [[COPY1]]
; CHECK-NEXT: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
- ; CHECK-NEXT: [[DEF:%[0-9]+]]:gr64 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF1:%[0-9]+]]:gr64 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:gr64 = IMPLICIT_DEF
; CHECK-NEXT: JMP_1 %bb.3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.6.bb35 (landing-pad):
@@ -278,7 +278,7 @@ body: |
; CHECK-NEXT: EH_LABEL <mcsymbol .Ltmp2>
; CHECK-NEXT: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
; CHECK-NEXT: $edi = MOV32ri 3
- ; CHECK-NEXT: dead [[STATEPOINT3]]:gr64_with_sub_8bit, dead [[DEF]]:gr64 = STATEPOINT 2882400000, 0, 1, target-flags(x86-plt) @wombat, $edi, 2, 0, 2, 2, 2, 97, 2, 0, 2, 2, 2, 0, 2, 43, 2, 0, 2, 2, 2, 0, 2, 7, 2, 0, 2, 7, 2, 0, 2, 10, 2, 1, 2, 10, 2, 0, 2, 4, 2, 1, 2, 7, 2, 0, 2, 7, 2, 0, 2, 0, [[STATEPOINT3]], 2, 7, 2, 0, 2, 0, [[STATEPOINT3]], 2, 10, 2, 5, 2, 12, 2, 0, 2, 3, 2, 1, 2, 0, [[STATEPOINT3]], 2, 0, [[STATEPOINT3]], 2, 7, 2, 0, 2, 0, [[STATEPOINT3]], 2, 2, 2, 11, 2, 4, 2, 0, 2, 1, 2, 0, 2, 7, 2, 0, 2, 2, 2, 15, 2, 7, 2, 0, 2, 2, 2, 0, 2, 7, 2, 0, 2, 7, 2, 0, 2, 10, 2, 21, 2, 96, 2, 0, 2, 9, 2, 1, 2, 0, 2, 4278124286, 2, 0, 2, 4278124286, 2, 7, 2, 0, 2, 7, 2, 0, 2, 0, [[DEF]], 2, 3, [[COPY1]], 2, 3, 2, 4278124286, 2, 7, 2, 0, 2, 7, 2, 0, 2, 0, [[DEF]], 2, 0, 2, 12, 2, 51, 2, 0, 2, 3, 2, 0, 2, 7, 2, 0, 2, 7, 2, 0, 2, 7, 2, 0, 2, 3, [[STATEPOINT3]](tied-def 0), 2, 4278124286, [[DEF]](tied-def 1), 2, 0, 2, 3, 0, 0, 1, 1, 2, 2, csr_64, implicit-def $rsp, implicit-def $ssp
+ ; CHECK-NEXT: dead [[MOV32r0_:%[0-9]+]]:gr64_with_sub_8bit, dead [[DEF1:%[0-9]+]]:gr64 = STATEPOINT 2882400000, 0, 1, target-flags(x86-plt) @wombat, $edi, 2, 0, 2, 2, 2, 97, 2, 0, 2, 2, 2, 0, 2, 43, 2, 0, 2, 2, 2, 0, 2, 7, 2, 0, 2, 7, 2, 0, 2, 10, 2, 1, 2, 10, 2, 0, 2, 4, 2, 1, 2, 7, 2, 0, 2, 7, 2, 0, 2, 0, [[MOV32r0_]], 2, 7, 2, 0, 2, 0, [[MOV32r0_]], 2, 10, 2, 5, 2, 12, 2, 0, 2, 3, 2, 1, 2, 0, [[MOV32r0_]], 2, 0, [[MOV32r0_]], 2, 7, 2, 0, 2, 0, [[MOV32r0_]], 2, 2, 2, 11, 2, 4, 2, 0, 2, 1, 2, 0, 2, 7, 2, 0, 2, 2, 2, 15, 2, 7, 2, 0, 2, 2, 2, 0, 2, 7, 2, 0, 2, 7, 2, 0, 2, 10, 2, 21, 2, 96, 2, 0, 2, 9, 2, 1, 2, 0, 2, 4278124286, 2, 0, 2, 4278124286, 2, 7, 2, 0, 2, 7, 2, 0, 2, 0, [[DEF1]], 2, 3, [[MOV32r0_1]], 2, 3, 2, 4278124286, 2, 7, 2, 0, 2, 7, 2, 0, 2, 0, [[DEF1]], 2, 0, 2, 12, 2, 51, 2, 0, 2, 3, 2, 0, 2, 7, 2, 0, 2, 7, 2, 0, 2, 7, 2, 0, 2, 3, [[MOV32r0_]](tied-def 0), 2, 4278124286, [[DEF1]](tied-def 1), 2, 0, 2, 3, 0, 0, 1, 1, 2, 2, csr_64, implicit-def $rsp, implicit-def $ssp
; CHECK-NEXT: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
bb.0.bb:
successors: %bb.1(0x80000000)
diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir
index ae0a1e03e78e3..01e5966e8d6a3 100644
--- a/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir
+++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir
@@ -34,7 +34,7 @@
# CHECK: dead $esi = MOV32r0 implicit-def dead $eflags, implicit-def $rsi
# CHECK: $ecx = COPY %7
# CHECK: $r8d = MOV32r0 implicit-def dead $eflags
-# CHECK: %37:gr64 = COPY %38
+# CHECK: %37:gr64 = lr-split COPY %38
# CHECK: %17:gr64, %16:gr64, %13:gr64, %37:gr64 = STATEPOINT 1, 16, 5, undef %23:gr64, $edi, $rsi, undef $edx, $ecx, $r8d, 2, 0, 2, 0, 2, 11, 1, 4, %stack.0, 0, %13, 1, 4, %stack.1, 0, 1, 4, %stack.2, 0, 1, 4, %stack.3, 0, 1, 4, %stack.4, 0, 1, 4, %stack.2, 0, %37, 1, 4, %stack.5, 0, %16, %17, 2, 4, %17(tied-def 0), %16(tied-def 1), %13(tied-def 2), %37(tied-def 3), 2, 0, 2, 4, 0, 0, 1, 1, 2, 2, 3, 3, csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def dead $eax :: (volatile load store (s32) on %stack.0), (volatile load store (s32) on %stack.1), (volatile load store (s32) on %stack.2), (volatile load store (s32) on %stack.3), (volatile load store (s32) on %stack.4), (volatile load store (s32) on %stack.5)
# CHECK: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
# CHECK: EH_LABEL <mcsymbol .Ltmp1>
diff --git a/llvm/test/DebugInfo/MIR/X86/dvl-livedebugvars-movements.mir b/llvm/test/DebugInfo/MIR/X86/dvl-livedebugvars-movements.mir
index de69a6f034f18..037dbace062d5 100644
--- a/llvm/test/DebugInfo/MIR/X86/dvl-livedebugvars-movements.mir
+++ b/llvm/test/DebugInfo/MIR/X86/dvl-livedebugvars-movements.mir
@@ -13,7 +13,7 @@
# CHECK-NEXT: DBG_VALUE_LIST !14, !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_arg, 1, DW_OP_plus), $rax, $rbx,
# CHECK-NEXT: CALL64pcrel32 @foo
# CHECK-NEXT: DBG_VALUE_LIST !14, !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_deref, DW_OP_LLVM_arg, 1, DW_OP_plus), %stack.0, $rbx,
-# CHECK-NEXT: $rcx = COPY killed renamable $rbx
+# CHECK-NEXT: $rcx = lr-split COPY killed renamable $rbx
# CHECK-NEXT: DBG_VALUE_LIST !14, !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_deref, DW_OP_LLVM_arg, 1, DW_OP_plus), %stack.0, $rcx,
# CHECK-NEXT: INLINEASM
# CHECK-NEXT: $rax = MOV64rm %stack.0
diff --git a/llvm/test/DebugInfo/MIR/X86/dvl-livedebugvars-stackptr.mir b/llvm/test/DebugInfo/MIR/X86/dvl-livedebugvars-stackptr.mir
index ef495e9bac82d..cf2b3edd010fe 100644
--- a/llvm/test/DebugInfo/MIR/X86/dvl-livedebugvars-stackptr.mir
+++ b/llvm/test/DebugInfo/MIR/X86/dvl-livedebugvars-stackptr.mir
@@ -13,7 +13,7 @@
# CHECK-NEXT: DBG_VALUE_LIST !14, !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_arg, 1, DW_OP_plus), $rax, %stack.0.local1,
# CHECK-NEXT: CALL64pcrel32 @foo
# CHECK-NEXT: DBG_VALUE_LIST !14, !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_deref, DW_OP_LLVM_arg, 1, DW_OP_plus), %stack.1, %stack.0.local1,
-# CHECK-NEXT: $rcx = COPY killed renamable $rbx
+# CHECK-NEXT: $rcx = lr-split COPY killed renamable $rbx
# CHECK-NEXT: INLINEASM
# CHECK-NEXT: $rax = MOV64rm %stack.1
# CHECK-NEXT: DBG_VALUE_LIST !14, !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_arg, 1, DW_OP_plus), $rax, %stack.0.local1,
diff --git a/llvm/utils/UpdateTestChecks/mir.py b/llvm/utils/UpdateTestChecks/mir.py
index b6fa324d0565e..e75d3dd2ff368 100644
--- a/llvm/utils/UpdateTestChecks/mir.py
+++ b/llvm/utils/UpdateTestChecks/mir.py
@@ -21,7 +21,7 @@
MI_FLAGS_STR = (
r"(frame-setup |frame-destroy |nnan |ninf |nsz |arcp |contract |afn "
r"|reassoc |nuw |nsw |exact |nofpexcept |nomerge |unpredictable "
- r"|noconvergent |nneg |disjoint |nusw |samesign |inbounds )*"
+ r"|noconvergent |nneg |disjoint |nusw |samesign |inbounds |lr-split )*"
)
VREG_DEF_FLAGS_STR = r"(?:dead |undef )*"
>From c35cba42e67284a1b6b8609f20782f6d8c692a6c Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <95053726+gandhi56 at users.noreply.github.com>
Date: Mon, 6 Jul 2026 12:55:47 -0400
Subject: [PATCH 20/46] [SandboxIR] Fix pass pipeline parsing after aux
arguments (#207237)
## Summary
- Fix `sandboxir::PassManager::setPassPipeline` to resume scanning pass
names after a pass aux argument when the next token is a delimiter.
- Add a unit test covering flat pipelines like `foo(aux1),bar`.
This fixes parsing of region pass pipelines such as
`bottom-up-vec(top-down),tr-accept` that will be used by the
SandboxVectorizer.
## Test plan
- [x] `ninja SandboxIRTests`
- [x] `./unittests/SandboxIR/SandboxIRTests
--gtest_filter=PassTest.SetPassPipeline`
Made with [Cursor](https://cursor.com)
Co-authored-by: Cursor <cursoragent at cursor.com>
---
llvm/include/llvm/SandboxIR/PassManager.h | 9 +++++++--
llvm/unittests/SandboxIR/PassTest.cpp | 16 ++++++++++++++++
2 files changed, 23 insertions(+), 2 deletions(-)
diff --git a/llvm/include/llvm/SandboxIR/PassManager.h b/llvm/include/llvm/SandboxIR/PassManager.h
index 5b9b4d0562e6d..057f61096e4ce 100644
--- a/llvm/include/llvm/SandboxIR/PassManager.h
+++ b/llvm/include/llvm/SandboxIR/PassManager.h
@@ -235,9 +235,14 @@ class PassManager : public ParentPass {
}
break;
case State::AuxArgsEnded:
- if (C == EndToken || C == PassDelimToken) {
+ if (C == EndToken) {
AddPass(PassName, StringRef(), AuxArg);
- CurrentState = State::ScanArgs;
+ CurrentState = State::ScanName;
+ } else if (C == PassDelimToken) {
+ AddPass(PassName, StringRef(), AuxArg);
+ PassBeginIdx = Idx + 1;
+ AuxArg = StringRef();
+ CurrentState = State::ScanName;
} else if (C == BeginArgsToken) {
++NestedArgs;
ArgsBeginIdx = Idx + 1;
diff --git a/llvm/unittests/SandboxIR/PassTest.cpp b/llvm/unittests/SandboxIR/PassTest.cpp
index 4707137645430..f0931ad66468e 100644
--- a/llvm/unittests/SandboxIR/PassTest.cpp
+++ b/llvm/unittests/SandboxIR/PassTest.cpp
@@ -323,6 +323,22 @@ define void @f() {
EXPECT_EQ(Str,
"foo(aux1)<abc>bar<nested1(aux2)<nested2<nested3()>>>foo(aux3)<>");
+ // A pass with an aux argument followed by another pass in a flat pipeline.
+ std::string AuxArgStr;
+ auto CreatePassWithAuxArg =
+ [&AuxArgStr](llvm::StringRef Name, llvm::StringRef Args,
+ llvm::StringRef AuxArg) -> std::unique_ptr<FunctionPass> {
+ if (Name == "foo")
+ return std::make_unique<FooPass>(AuxArgStr, Args, AuxArg);
+ if (Name == "bar")
+ return std::make_unique<BarPass>(AuxArgStr, Args, AuxArg);
+ return nullptr;
+ };
+ FunctionPassManager FPMWithAuxArg("test-fpm");
+ FPMWithAuxArg.setPassPipeline("foo(aux1),bar", CreatePassWithAuxArg);
+ FPMWithAuxArg.runOnFunction(*F, Analyses::emptyForTesting());
+ EXPECT_EQ(AuxArgStr, "foo(aux1)<>bar<>");
+
// A second call to setPassPipeline will trigger an assertion in debug mode.
#ifndef NDEBUG
EXPECT_DEATH(FPM.setPassPipeline("bar,bar,foo", CreatePass),
>From 20e12152f9b218c42791151f0bca93a5a1763b64 Mon Sep 17 00:00:00 2001
From: Jason Eckhardt <jeckhardt at nvidia.com>
Date: Mon, 6 Jul 2026 11:59:18 -0500
Subject: [PATCH 21/46] [TableGen][CC][NFC] Emit idiomatic guards for
CallingConv. (#207758)
Emit independent guards for the two existing sections rather than the
if-else done today. This is in preparation for adding other sections
that must be includable exclusive of the other sections.
This also eliminates the current asymmetry between the automatic
IfGuardEmitter for the "if" portion vs the manually emitted "else"
section.
---
.../AArch64/AArch64CallingConvention.cpp | 1 +
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 1 +
llvm/lib/Target/AMDGPU/R600ISelLowering.cpp | 1 +
llvm/lib/Target/ARC/ARCISelLowering.cpp | 1 +
llvm/lib/Target/ARM/ARMCallingConv.cpp | 1 +
llvm/lib/Target/AVR/AVRISelLowering.cpp | 1 +
llvm/lib/Target/BPF/BPFISelLowering.cpp | 1 +
llvm/lib/Target/CSKY/CSKYISelLowering.cpp | 1 +
.../Target/Hexagon/HexagonISelLowering.cpp | 1 +
llvm/lib/Target/Lanai/LanaiISelLowering.cpp | 1 +
llvm/lib/Target/M68k/M68kISelLowering.cpp | 1 +
llvm/lib/Target/MSP430/MSP430ISelLowering.cpp | 1 +
llvm/lib/Target/Mips/MipsFastISel.cpp | 1 +
llvm/lib/Target/Mips/MipsISelLowering.cpp | 1 +
llvm/lib/Target/PowerPC/PPCCallingConv.cpp | 1 +
llvm/lib/Target/Sparc/SparcISelLowering.cpp | 1 +
.../Target/SystemZ/SystemZISelLowering.cpp | 1 +
llvm/lib/Target/VE/VEISelLowering.cpp | 1 +
llvm/lib/Target/X86/X86CallingConv.cpp | 1 +
llvm/lib/Target/XCore/XCoreISelLowering.cpp | 1 +
llvm/lib/Target/Xtensa/XtensaISelLowering.cpp | 1 +
llvm/utils/TableGen/CallingConvEmitter.cpp | 24 ++++++++++---------
22 files changed, 34 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64CallingConvention.cpp b/llvm/lib/Target/AArch64/AArch64CallingConvention.cpp
index cc46159915d76..b094a058e50a7 100644
--- a/llvm/lib/Target/AArch64/AArch64CallingConvention.cpp
+++ b/llvm/lib/Target/AArch64/AArch64CallingConvention.cpp
@@ -223,4 +223,5 @@ static bool CC_AArch64_Custom_Block(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
// TableGen provides definitions of the calling convention analysis entry
// points.
+#define GET_CALLING_CONV_IMPL
#include "AArch64GenCallingConv.inc"
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 5010720ca8e76..72f15b054c302 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -30,6 +30,7 @@
using namespace llvm;
+#define GET_CALLING_CONV_IMPL
#include "AMDGPUGenCallingConv.inc"
static cl::opt<bool> AMDGPUBypassSlowDiv(
diff --git a/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp b/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp
index 981a30d4fb356..b413bd4d53cc2 100644
--- a/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp
@@ -26,6 +26,7 @@
using namespace llvm;
+#define GET_CALLING_CONV_IMPL
#include "R600GenCallingConv.inc"
R600TargetLowering::R600TargetLowering(const TargetMachine &TM,
diff --git a/llvm/lib/Target/ARC/ARCISelLowering.cpp b/llvm/lib/Target/ARC/ARCISelLowering.cpp
index cbe9fb6239f7b..6ce05ad5dfef1 100644
--- a/llvm/lib/Target/ARC/ARCISelLowering.cpp
+++ b/llvm/lib/Target/ARC/ARCISelLowering.cpp
@@ -236,6 +236,7 @@ SDValue ARCTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
return DAG.getNode(ARCISD::GAWRAPPER, SDLoc(N), MVT::i32, GA);
}
+#define GET_CALLING_CONV_IMPL
#include "ARCGenCallingConv.inc"
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/ARM/ARMCallingConv.cpp b/llvm/lib/Target/ARM/ARMCallingConv.cpp
index 795476b3f386b..2ba040b8f4e50 100644
--- a/llvm/lib/Target/ARM/ARMCallingConv.cpp
+++ b/llvm/lib/Target/ARM/ARMCallingConv.cpp
@@ -330,4 +330,5 @@ static bool CC_ARM_AAPCS_Common_Custom_f16_Stack(unsigned ValNo, MVT ValVT,
}
// Include the table generated calling convention implementations.
+#define GET_CALLING_CONV_IMPL
#include "ARMGenCallingConv.inc"
diff --git a/llvm/lib/Target/AVR/AVRISelLowering.cpp b/llvm/lib/Target/AVR/AVRISelLowering.cpp
index 89389f7da827b..d485b61c58ee0 100644
--- a/llvm/lib/Target/AVR/AVRISelLowering.cpp
+++ b/llvm/lib/Target/AVR/AVRISelLowering.cpp
@@ -1151,6 +1151,7 @@ bool AVRTargetLowering::isOffsetFoldingLegal(
// Formal Arguments Calling Convention Implementation
//===----------------------------------------------------------------------===//
+#define GET_CALLING_CONV_IMPL
#include "AVRGenCallingConv.inc"
/// Registers for calling conventions, ordered in reverse as required by ABI.
diff --git a/llvm/lib/Target/BPF/BPFISelLowering.cpp b/llvm/lib/Target/BPF/BPFISelLowering.cpp
index 422a1ba5c4dd6..435f0e15a25c5 100644
--- a/llvm/lib/Target/BPF/BPFISelLowering.cpp
+++ b/llvm/lib/Target/BPF/BPFISelLowering.cpp
@@ -387,6 +387,7 @@ SDValue BPFTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
}
// Calling Convention Implementation
+#define GET_CALLING_CONV_IMPL
#include "BPFGenCallingConv.inc"
// Apply AssertSext/AssertZext and truncate based on VA's LocInfo.
diff --git a/llvm/lib/Target/CSKY/CSKYISelLowering.cpp b/llvm/lib/Target/CSKY/CSKYISelLowering.cpp
index 738dcfb167e48..1bc3d5ca4431e 100644
--- a/llvm/lib/Target/CSKY/CSKYISelLowering.cpp
+++ b/llvm/lib/Target/CSKY/CSKYISelLowering.cpp
@@ -29,6 +29,7 @@ using namespace llvm;
STATISTIC(NumTailCalls, "Number of tail calls");
+#define GET_CALLING_CONV_IMPL
#include "CSKYGenCallingConv.inc"
static const MCPhysReg GPRArgRegs[] = {CSKY::R0, CSKY::R1, CSKY::R2, CSKY::R3};
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 6b5e5ef1a3c81..57694bdf8c0c6 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -113,6 +113,7 @@ static bool CC_SkipOdd(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
return false;
}
+#define GET_CALLING_CONV_IMPL
#include "HexagonGenCallingConv.inc"
unsigned HexagonTargetLowering::getVectorTypeBreakdownForCallingConv(
diff --git a/llvm/lib/Target/Lanai/LanaiISelLowering.cpp b/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
index ff35d981d6291..5831bd8685f72 100644
--- a/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
+++ b/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
@@ -351,6 +351,7 @@ void LanaiTargetLowering::LowerAsmOperandForConstraint(
// Calling Convention Implementation
//===----------------------------------------------------------------------===//
+#define GET_CALLING_CONV_IMPL
#include "LanaiGenCallingConv.inc"
static bool CC_Lanai32_VarArg(unsigned ValNo, MVT ValVT, MVT LocVT,
diff --git a/llvm/lib/Target/M68k/M68kISelLowering.cpp b/llvm/lib/Target/M68k/M68kISelLowering.cpp
index 3621580d6ad35..d167e7f6597b6 100644
--- a/llvm/lib/Target/M68k/M68kISelLowering.cpp
+++ b/llvm/lib/Target/M68k/M68kISelLowering.cpp
@@ -230,6 +230,7 @@ MVT M68kTargetLowering::getScalarShiftAmountTy(const DataLayout &DL,
return MVT::getIntegerVT(DL.getPointerSizeInBits(0));
}
+#define GET_CALLING_CONV_IMPL
#include "M68kGenCallingConv.inc"
enum StructReturnType { NotStructReturn, RegStructReturn, StackStructReturn };
diff --git a/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp b/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
index db716f02899b5..04a8a0721e6f5 100644
--- a/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
+++ b/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
@@ -230,6 +230,7 @@ MSP430TargetLowering::getRegForInlineAsmConstraint(
// Calling Convention Implementation
//===----------------------------------------------------------------------===//
+#define GET_CALLING_CONV_IMPL
#include "MSP430GenCallingConv.inc"
/// For each argument in a function store the number of pieces it is composed
diff --git a/llvm/lib/Target/Mips/MipsFastISel.cpp b/llvm/lib/Target/Mips/MipsFastISel.cpp
index 9645fb5293609..7ef770184e78e 100644
--- a/llvm/lib/Target/Mips/MipsFastISel.cpp
+++ b/llvm/lib/Target/Mips/MipsFastISel.cpp
@@ -285,6 +285,7 @@ static bool CC_MipsO32_FP64(unsigned ValNo, MVT ValVT, MVT LocVT,
llvm_unreachable("should not be called");
}
+#define GET_CALLING_CONV_IMPL
#include "MipsGenCallingConv.inc"
CCAssignFn *MipsFastISel::CCAssignFnForCall(CallingConv::ID CC) const {
diff --git a/llvm/lib/Target/Mips/MipsISelLowering.cpp b/llvm/lib/Target/Mips/MipsISelLowering.cpp
index 4389f0c8a649a..076ca31649c43 100644
--- a/llvm/lib/Target/Mips/MipsISelLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsISelLowering.cpp
@@ -3139,6 +3139,7 @@ static bool CC_MipsO32_FP64(unsigned ValNo, MVT ValVT, MVT LocVT,
ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
CCState &State);
+#define GET_CALLING_CONV_IMPL
#include "MipsGenCallingConv.inc"
CCAssignFn *MipsTargetLowering::CCAssignFnForCall() const{
diff --git a/llvm/lib/Target/PowerPC/PPCCallingConv.cpp b/llvm/lib/Target/PowerPC/PPCCallingConv.cpp
index 332b5e07d3d5d..3f14cabe78ceb 100644
--- a/llvm/lib/Target/PowerPC/PPCCallingConv.cpp
+++ b/llvm/lib/Target/PowerPC/PPCCallingConv.cpp
@@ -193,4 +193,5 @@ static bool CC_PPC32_SPE_RetF64(unsigned &ValNo, MVT &ValVT,
return true;
}
+#define GET_CALLING_CONV_IMPL
#include "PPCGenCallingConv.inc"
diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.cpp b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
index a41847a132dfb..c45babaec5dc2 100644
--- a/llvm/lib/Target/Sparc/SparcISelLowering.cpp
+++ b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
@@ -223,6 +223,7 @@ static bool RetCC_Sparc64_Half(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
State);
}
+#define GET_CALLING_CONV_IMPL
#include "SparcGenCallingConv.inc"
// The calling conventions in SparcCallingConv.td are described in terms of the
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index e56a449cd455d..388e44af29aab 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -1855,6 +1855,7 @@ void SystemZTargetLowering::LowerAsmOperandForConstraint(
// Calling conventions
//===----------------------------------------------------------------------===//
+#define GET_CALLING_CONV_IMPL
#include "SystemZGenCallingConv.inc"
const MCPhysReg *SystemZTargetLowering::getScratchRegisters(
diff --git a/llvm/lib/Target/VE/VEISelLowering.cpp b/llvm/lib/Target/VE/VEISelLowering.cpp
index 23eb3034fe3e6..809423c2fb811 100644
--- a/llvm/lib/Target/VE/VEISelLowering.cpp
+++ b/llvm/lib/Target/VE/VEISelLowering.cpp
@@ -42,6 +42,7 @@ using namespace llvm;
// Calling Convention Implementation
//===----------------------------------------------------------------------===//
+#define GET_CALLING_CONV_IMPL
#include "VEGenCallingConv.inc"
CCAssignFn *getReturnCC(CallingConv::ID CallConv) {
diff --git a/llvm/lib/Target/X86/X86CallingConv.cpp b/llvm/lib/Target/X86/X86CallingConv.cpp
index 5d5a705893242..ce2ce1cbc6379 100644
--- a/llvm/lib/Target/X86/X86CallingConv.cpp
+++ b/llvm/lib/Target/X86/X86CallingConv.cpp
@@ -406,4 +406,5 @@ static bool CC_X86_32_I128_FP128(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
}
// Provides entry points of CC_X86 and RetCC_X86.
+#define GET_CALLING_CONV_IMPL
#include "X86GenCallingConv.inc"
diff --git a/llvm/lib/Target/XCore/XCoreISelLowering.cpp b/llvm/lib/Target/XCore/XCoreISelLowering.cpp
index f54862a804a45..c3cbb4b4752d2 100644
--- a/llvm/lib/Target/XCore/XCoreISelLowering.cpp
+++ b/llvm/lib/Target/XCore/XCoreISelLowering.cpp
@@ -892,6 +892,7 @@ LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG) const {
// Calling Convention Implementation
//===----------------------------------------------------------------------===//
+#define GET_CALLING_CONV_IMPL
#include "XCoreGenCallingConv.inc"
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp b/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp
index 062911cd553b6..0c880234af65f 100644
--- a/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp
+++ b/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp
@@ -353,6 +353,7 @@ void XtensaTargetLowering::LowerAsmOperandForConstraint(
// Calling conventions
//===----------------------------------------------------------------------===//
+#define GET_CALLING_CONV_IMPL
#include "XtensaGenCallingConv.inc"
static const MCPhysReg IntRegs[] = {Xtensa::A2, Xtensa::A3, Xtensa::A4,
diff --git a/llvm/utils/TableGen/CallingConvEmitter.cpp b/llvm/utils/TableGen/CallingConvEmitter.cpp
index 369238dd70f7c..0c59f2b47215b 100644
--- a/llvm/utils/TableGen/CallingConvEmitter.cpp
+++ b/llvm/utils/TableGen/CallingConvEmitter.cpp
@@ -79,17 +79,19 @@ void CallingConvEmitter::run(raw_ostream &O) {
// Emit prototypes for all of the non-custom CC's so that they can forward ref
// each other.
Records.getTimer().startTimer("Emit prototypes");
- IfGuardEmitter IfGuard(O, "!defined(GET_CC_REGISTER_LISTS)");
- for (const Record *CC : CCs) {
- if (!CC->getValueAsBit("Custom"))
- emitCCHeader(O, CC, ";\n");
- }
+ {
+ IfDefEmitter IfGuard(O, "GET_CALLING_CONV_IMPL");
+ for (const Record *CC : CCs) {
+ if (!CC->getValueAsBit("Custom"))
+ emitCCHeader(O, CC, ";\n");
+ }
- // Emit each non-custom calling convention description in full.
- Records.getTimer().startTimer("Emit full descriptions");
- for (const Record *CC : CCs) {
- if (!CC->getValueAsBit("Custom"))
- emitCallingConv(CC, O);
+ // Emit each non-custom calling convention description in full.
+ Records.getTimer().startTimer("Emit full descriptions");
+ for (const Record *CC : CCs) {
+ if (!CC->getValueAsBit("Custom"))
+ emitCallingConv(CC, O);
+ }
}
emitArgRegisterLists(O);
@@ -365,7 +367,7 @@ void CallingConvEmitter::emitArgRegisterLists(raw_ostream &O) {
if (AssignedRegsMap.empty())
return;
- O << "\n#else\n\n";
+ IfDefEmitter IfGuard(O, "GET_CC_REGISTER_LISTS");
for (const auto &[RegName, Registers] : AssignedRegsMap) {
if (RegName.empty())
>From 78bc590bbc89aa868bcee35982dd3cf70cebea64 Mon Sep 17 00:00:00 2001
From: Stefan Mada <smada at nvidia.com>
Date: Mon, 6 Jul 2026 10:01:36 -0700
Subject: [PATCH 22/46] [IR][NVVM] Fixed null pointer deference in AsmWriter
pretty printer (#207223)
ASMWriter can deference a null pointer on intrinsics with pretty printed
args when those intrinsics are called with an invalid number of
arguments. This check avoids the null dereference and turns this into a
graceful failure.
---
llvm/lib/IR/AsmWriter.cpp | 2 +-
...insic-immarg-print-mismatched-signature.ll | 22 +++++++++++++++++++
2 files changed, 23 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/NVPTX/intrinsic-immarg-print-mismatched-signature.ll
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index 02d6bcd90add0..e90630a8cae59 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -4644,7 +4644,7 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
Function *CalledFunc = CI->getCalledFunction();
auto PrintArgComment = [&](unsigned ArgNo) {
const auto *ConstArg = dyn_cast<Constant>(CI->getArgOperand(ArgNo));
- if (!ConstArg)
+ if (!ConstArg || !CalledFunc)
return;
std::string ArgComment;
raw_string_ostream ArgCommentStream(ArgComment);
diff --git a/llvm/test/CodeGen/NVPTX/intrinsic-immarg-print-mismatched-signature.ll b/llvm/test/CodeGen/NVPTX/intrinsic-immarg-print-mismatched-signature.ll
new file mode 100644
index 0000000000000..aefd9045c91ad
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/intrinsic-immarg-print-mismatched-signature.ll
@@ -0,0 +1,22 @@
+; NOTE: This test guards AsmWriter's intrinsic immediate-argument pretty-printer
+; against a null-pointer dereference.
+; RUN: llvm-as -disable-verify < %s | llvm-dis | FileCheck %s
+
+declare void @llvm.nvvm.tensormap.replace.elemtype.p0(ptr, i32)
+
+; A well-formed call: the elemtype immediate is pretty-printed.
+define void @valid(ptr %p) {
+ ; CHECK-LABEL: define void @valid(
+ ; CHECK: call void @llvm.nvvm.tensormap.replace.elemtype.p0(ptr %p, /* elemtype=u8 */ i32 0)
+ call void @llvm.nvvm.tensormap.replace.elemtype.p0(ptr %p, i32 0)
+ ret void
+}
+
+; A call with a mismatched signature (an extra operand): getCalledFunction() is
+; null, so no comment is emitted and printing must not crash.
+define void @mismatched_signature(ptr %p) {
+ ; CHECK-LABEL: define void @mismatched_signature(
+ ; CHECK: call void @llvm.nvvm.tensormap.replace.elemtype.p0(ptr %p, i32 0, i32 0)
+ call void @llvm.nvvm.tensormap.replace.elemtype.p0(ptr %p, i32 0, i32 0)
+ ret void
+}
>From 9c823964dd43645162d361b16d2ddedc381f5d2d Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi at nvidia.com>
Date: Mon, 6 Jul 2026 10:05:36 -0700
Subject: [PATCH 23/46] [NFC][LLVM][NVPTX] Initialize pass-ids to zero
(#207547)
Initialize pass::ID values for various passes to canonical zero instead
of 1.
---
llvm/lib/Target/NVPTX/NVPTXLowerAlloca.cpp | 2 +-
llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp | 2 +-
llvm/lib/Target/NVPTX/NVPTXLowerUnreachable.cpp | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXLowerAlloca.cpp b/llvm/lib/Target/NVPTX/NVPTXLowerAlloca.cpp
index 1583d45eeb3a6..ef8df4d70d956 100644
--- a/llvm/lib/Target/NVPTX/NVPTXLowerAlloca.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXLowerAlloca.cpp
@@ -46,7 +46,7 @@ class NVPTXLowerAlloca : public FunctionPass {
};
} // namespace
-char NVPTXLowerAlloca::ID = 1;
+char NVPTXLowerAlloca::ID = 0;
INITIALIZE_PASS(NVPTXLowerAlloca, "nvptx-lower-alloca", "Lower Alloca", false,
false)
diff --git a/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp b/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
index 92ba3d214620d..3e8eb667f3eb1 100644
--- a/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
@@ -83,7 +83,7 @@ class NVPTXLowerArgsLegacyPass : public ModulePass {
};
} // namespace
-char NVPTXLowerArgsLegacyPass::ID = 1;
+char NVPTXLowerArgsLegacyPass::ID = 0;
INITIALIZE_PASS_BEGIN(NVPTXLowerArgsLegacyPass, "nvptx-lower-args",
"Lower arguments (NVPTX)", false, false)
diff --git a/llvm/lib/Target/NVPTX/NVPTXLowerUnreachable.cpp b/llvm/lib/Target/NVPTX/NVPTXLowerUnreachable.cpp
index 00a12bf818897..4807c143e96d4 100644
--- a/llvm/lib/Target/NVPTX/NVPTXLowerUnreachable.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXLowerUnreachable.cpp
@@ -96,7 +96,7 @@ class NVPTXLowerUnreachable : public FunctionPass {
};
} // namespace
-char NVPTXLowerUnreachable::ID = 1;
+char NVPTXLowerUnreachable::ID = 0;
INITIALIZE_PASS(NVPTXLowerUnreachable, "nvptx-lower-unreachable",
"Lower Unreachable", false, false)
>From 2e75acd861aa78755c7a57ab8eca63fde916497a Mon Sep 17 00:00:00 2001
From: Aditya Medhane <sherlockedaditya at gmail.com>
Date: Mon, 6 Jul 2026 22:56:17 +0530
Subject: [PATCH 24/46] [libc][test] Remove internal linkage from
ErrnoSetterMatcher helpers (NFC) (#207707)
`Succeeds`, `Fails`, and `returns` are static function templates in a
header, so any test TU that includes it without using all of them trips
`-Wunused-template`. Dropping static also matches the neighboring EQ/NE
helpers, which are already non-static.
Part of #202945.
---
libc/test/UnitTest/ErrnoSetterMatcher.h | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/libc/test/UnitTest/ErrnoSetterMatcher.h b/libc/test/UnitTest/ErrnoSetterMatcher.h
index 212b7a8f83e74..e883308314087 100644
--- a/libc/test/UnitTest/ErrnoSetterMatcher.h
+++ b/libc/test/UnitTest/ErrnoSetterMatcher.h
@@ -153,15 +153,15 @@ template <typename T> internal::Comparator<T> NE(T val) {
}
template <typename RetT = int>
-static internal::ErrnoSetterMatcher<RetT> Succeeds(RetT ExpectedReturn = 0,
- int ExpectedErrno = 0) {
+internal::ErrnoSetterMatcher<RetT> Succeeds(RetT ExpectedReturn = 0,
+ int ExpectedErrno = 0) {
return internal::ErrnoSetterMatcher<RetT>(EQ(ExpectedReturn),
EQ(ExpectedErrno));
}
template <typename RetT = int>
-static internal::ErrnoSetterMatcher<RetT> Fails(int ExpectedErrno,
- RetT ExpectedReturn = -1) {
+internal::ErrnoSetterMatcher<RetT> Fails(int ExpectedErrno,
+ RetT ExpectedReturn = -1) {
return internal::ErrnoSetterMatcher<RetT>(EQ(ExpectedReturn),
EQ(ExpectedErrno));
}
@@ -180,7 +180,7 @@ template <typename RetT = int> class ErrnoSetterMatcherBuilder {
};
template <typename RetT>
-static ErrnoSetterMatcherBuilder<RetT> returns(internal::Comparator<RetT> cmp) {
+ErrnoSetterMatcherBuilder<RetT> returns(internal::Comparator<RetT> cmp) {
return ErrnoSetterMatcherBuilder<RetT>(cmp);
}
>From 42746081459775eb85879cc09361a316ac9e0ea2 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Mon, 6 Jul 2026 19:28:24 +0200
Subject: [PATCH 25/46] [AMDGPU] Fix printf %s buffer slot oversized when
strlen % 4 == 0 (#207772)
The metadata slot for a constant string %s used alignTo(strlen + 1, 4),
but the store loop only writes ceil(strlen / 4) dwords (no trailing
NUL), so every following argument was read from offset shifted by +4
---
.../lib/Target/AMDGPU/AMDGPUPrintfRuntimeBinding.cpp | 8 ++++++--
llvm/test/CodeGen/AMDGPU/opencl-printf.ll | 12 ++++++------
2 files changed, 12 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPrintfRuntimeBinding.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPrintfRuntimeBinding.cpp
index b0018e0d1c271..98928e26f8de0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPrintfRuntimeBinding.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPrintfRuntimeBinding.cpp
@@ -211,8 +211,12 @@ bool AMDGPUPrintfRuntimeBindingImpl::lowerPrintfForGpu(Module &M) {
ArgSize = 4;
}
}
- if (shouldPrintAsStr(OpConvSpecifiers[ArgCount - 1], ArgType))
- ArgSize = alignTo(getAsConstantStr(Arg).size() + 1, 4);
+ if (shouldPrintAsStr(OpConvSpecifiers[ArgCount - 1], ArgType)) {
+ // Match the store loop below: ceil(len / 4) dwords, or one dword
+ // for the empty string. The trailing NUL is not stored.
+ StringRef Str = getAsConstantStr(Arg);
+ ArgSize = Str.empty() ? 4 : alignTo(Str.size(), 4);
+ }
LLVM_DEBUG(dbgs() << "Printf ArgSize (in buffer) = " << ArgSize
<< " for type: " << *ArgType << '\n');
diff --git a/llvm/test/CodeGen/AMDGPU/opencl-printf.ll b/llvm/test/CodeGen/AMDGPU/opencl-printf.ll
index 9407fb2f66219..17abc6efb49bb 100644
--- a/llvm/test/CodeGen/AMDGPU/opencl-printf.ll
+++ b/llvm/test/CodeGen/AMDGPU/opencl-printf.ll
@@ -1106,7 +1106,7 @@ entry:
define amdgpu_kernel void @test_print_string_literal_size4_nonull_term(i32 %n) {
; GCN-LABEL: @test_print_string_literal_size4_nonull_term(
; GCN-NEXT: entry:
-; GCN-NEXT: [[PRINTF_ALLOC_FN:%.*]] = call ptr addrspace(1) @__printf_alloc(i32 16)
+; GCN-NEXT: [[PRINTF_ALLOC_FN:%.*]] = call ptr addrspace(1) @__printf_alloc(i32 12)
; GCN-NEXT: br label [[ENTRY_SPLIT:%.*]]
; GCN: entry.split:
; GCN-NEXT: [[TMP0:%.*]] = icmp ne ptr addrspace(1) [[PRINTF_ALLOC_FN]], null
@@ -1131,7 +1131,7 @@ entry:
define amdgpu_kernel void @test_print_string_literal_size5(i32 %n) {
; GCN-LABEL: @test_print_string_literal_size5(
; GCN-NEXT: entry:
-; GCN-NEXT: [[PRINTF_ALLOC_FN:%.*]] = call ptr addrspace(1) @__printf_alloc(i32 16)
+; GCN-NEXT: [[PRINTF_ALLOC_FN:%.*]] = call ptr addrspace(1) @__printf_alloc(i32 12)
; GCN-NEXT: br label [[ENTRY_SPLIT:%.*]]
; GCN: entry.split:
; GCN-NEXT: [[TMP0:%.*]] = icmp ne ptr addrspace(1) [[PRINTF_ALLOC_FN]], null
@@ -1237,7 +1237,7 @@ entry:
define amdgpu_kernel void @test_print_string_literal_size9(i32 %n) {
; GCN-LABEL: @test_print_string_literal_size9(
; GCN-NEXT: entry:
-; GCN-NEXT: [[PRINTF_ALLOC_FN:%.*]] = call ptr addrspace(1) @__printf_alloc(i32 20)
+; GCN-NEXT: [[PRINTF_ALLOC_FN:%.*]] = call ptr addrspace(1) @__printf_alloc(i32 16)
; GCN-NEXT: br label [[ENTRY_SPLIT:%.*]]
; GCN: entry.split:
; GCN-NEXT: [[TMP0:%.*]] = icmp ne ptr addrspace(1) [[PRINTF_ALLOC_FN]], null
@@ -1295,7 +1295,7 @@ entry:
define amdgpu_kernel void @test_print_string_literal_size17(i32 %n) {
; GCN-LABEL: @test_print_string_literal_size17(
; GCN-NEXT: entry:
-; GCN-NEXT: [[PRINTF_ALLOC_FN:%.*]] = call ptr addrspace(1) @__printf_alloc(i32 28)
+; GCN-NEXT: [[PRINTF_ALLOC_FN:%.*]] = call ptr addrspace(1) @__printf_alloc(i32 24)
; GCN-NEXT: br label [[ENTRY_SPLIT:%.*]]
; GCN: entry.split:
; GCN-NEXT: [[TMP0:%.*]] = icmp ne ptr addrspace(1) [[PRINTF_ALLOC_FN]], null
@@ -1448,7 +1448,7 @@ entry:
define amdgpu_kernel void @test_print_string_literal_v4i8(i32 %n) {
; GCN-LABEL: @test_print_string_literal_v4i8(
; GCN-NEXT: entry:
-; GCN-NEXT: [[PRINTF_ALLOC_FN:%.*]] = call ptr addrspace(1) @__printf_alloc(i32 16)
+; GCN-NEXT: [[PRINTF_ALLOC_FN:%.*]] = call ptr addrspace(1) @__printf_alloc(i32 12)
; GCN-NEXT: br label [[ENTRY_SPLIT:%.*]]
; GCN: entry.split:
; GCN-NEXT: [[TMP0:%.*]] = icmp ne ptr addrspace(1) [[PRINTF_ALLOC_FN]], null
@@ -1723,7 +1723,7 @@ entry:
define amdgpu_kernel void @test_print_string_indexed(i32 %n) {
; GCN-LABEL: @test_print_string_indexed(
; GCN-NEXT: entry:
-; GCN-NEXT: [[PRINTF_ALLOC_FN:%.*]] = call ptr addrspace(1) @__printf_alloc(i32 28)
+; GCN-NEXT: [[PRINTF_ALLOC_FN:%.*]] = call ptr addrspace(1) @__printf_alloc(i32 24)
; GCN-NEXT: br label [[ENTRY_SPLIT:%.*]]
; GCN: entry.split:
; GCN-NEXT: [[TMP0:%.*]] = icmp ne ptr addrspace(1) [[PRINTF_ALLOC_FN]], null
>From 0f0b1c76a4d84594fef4de8ae5c49aa565d8bab0 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Mon, 6 Jul 2026 13:28:51 -0400
Subject: [PATCH 26/46] [libc++] Run the libcxx-run-benchmarks action on macOS
in addition to Linux (#207780)
The libcxx-run-benchmarks action is triggered when a specially formatted
comment is posted on a PR, requesting benchmarks to be run against that
PR. This patch expands the Github action so it also runs benchmarks on
the macOS CI resources that were added recently.
---
.github/workflows/libcxx-run-benchmarks.yml | 79 ++++++++++++++-------
1 file changed, 54 insertions(+), 25 deletions(-)
diff --git a/.github/workflows/libcxx-run-benchmarks.yml b/.github/workflows/libcxx-run-benchmarks.yml
index 6346eef5bcd33..8529c0937af37 100644
--- a/.github/workflows/libcxx-run-benchmarks.yml
+++ b/.github/workflows/libcxx-run-benchmarks.yml
@@ -18,10 +18,6 @@ on:
- created
- edited
-env:
- CC: clang-22
- CXX: clang++-22
-
jobs:
permission-check:
if: >-
@@ -45,12 +41,12 @@ jobs:
LLVM_TOKEN_GENERATOR_CLIENT_ID: ${{ secrets.LLVM_TOKEN_GENERATOR_CLIENT_ID }}
LLVM_TOKEN_GENERATOR_PRIVATE_KEY: ${{ secrets.LLVM_TOKEN_GENERATOR_PRIVATE_KEY }}
- run-benchmarks:
- permissions:
- pull-requests: write
- runs-on: llvm-premerge-libcxx-next-runners # TODO: This should run on a dedicated set of machines
+ extract-info:
+ runs-on: ubuntu-24.04
needs:
- permission-check
+ permissions:
+ pull-requests: write
steps:
- uses: actions/setup-python at a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
@@ -78,9 +74,7 @@ jobs:
BENCHMARKS=$(echo "$COMMENT_BODY" | sed -nE 's/\/libcxx-bot benchmark (.+)/\1/p')
echo "benchmarks=${BENCHMARKS}" >> ${GITHUB_OUTPUT}
- - name: Update comment with link to the job
- env:
- JOB_CHECK_RUN_ID: ${{ job.check_run_id }}
+ - name: Update comment with link to the run
run: |
source .venv/bin/activate
cat <<EOF | python
@@ -88,26 +82,61 @@ jobs:
repo = github.Github(auth=github.Auth.Token("${{ github.token }}")).get_repo("${{ github.repository }}")
pr = repo.get_pull(${{ github.event.issue.number }})
comment = pr.get_issue_comment(${{ github.event.comment.id }})
- add_text = "> _Running benchmarks in ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/job/${JOB_CHECK_RUN_ID}_"
+ add_text = "> _Running benchmarks in ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}_"
comment.edit('\n\n'.join([comment.body, add_text]))
EOF
+ outputs:
+ pr_base: ${{ steps.vars.outputs.pr_base }}
+ pr_head: ${{ steps.vars.outputs.pr_head }}
+ benchmarks: ${{ steps.vars.outputs.benchmarks }}
+ run-benchmarks:
+ strategy:
+ matrix:
+ include:
+ - platform: macOS 26.5 arm64
+ runner: ["self-hosted", "macOS", "ARM64", "26", "26.5"]
+ cc: clang
+ cxx: clang++
+ - platform: Linux x86_64
+ runner: llvm-premerge-libcxx-next-runners
+ cc: clang-22
+ cxx: clang++-22
+ fail-fast: false
+ permissions:
+ pull-requests: write
+ runs-on: ${{ matrix.runner }}
+ needs:
+ - extract-info
+ env:
+ CC: ${{ matrix.cc }}
+ CXX: ${{ matrix.cxx }}
+ steps:
- uses: actions/checkout at de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
- ref: ${{ steps.vars.outputs.pr_head }}
+ ref: ${{ needs.extract-info.outputs.pr_head }}
fetch-depth: 0
fetch-tags: true # This job requires access to all the Git branches so it can diff against (usually) main
- path: repo # Avoid nuking the workspace, where we have the Python virtualenv
+
+ - uses: actions/setup-python at a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+ with:
+ python-version: '3.14'
+
+ - name: Setup virtual environment
+ run: |
+ python3 -m venv .venv
+ source .venv/bin/activate
+ python -m pip install -r libcxx/utils/requirements.txt
+ python -m pip install pygithub==2.8.1
- name: Build and run baseline
env:
- BENCHMARKS: ${{ steps.vars.outputs.benchmarks }}
- PR_HEAD: ${{ steps.vars.outputs.pr_head }}
- PR_BASE: ${{ steps.vars.outputs.pr_base }}
+ BENCHMARKS: ${{ needs.extract-info.outputs.benchmarks }}
+ PR_HEAD: ${{ needs.extract-info.outputs.pr_head }}
+ PR_BASE: ${{ needs.extract-info.outputs.pr_base }}
run: |
- source .venv/bin/activate && cd repo
- python -m pip install -r libcxx/utils/requirements.txt
+ source .venv/bin/activate
baseline_commit=$(git merge-base $PR_BASE $PR_HEAD)
./libcxx/utils/build-at-commit --commit ${baseline_commit} --install-dir install/baseline -- -DCMAKE_BUILD_TYPE=RelWithDebInfo
./libcxx/utils/test-at-commit --libcxx-installation install/baseline -B benchmarks/baseline -- -sv -j1 --param optimization=speed "$BENCHMARKS"
@@ -115,22 +144,22 @@ jobs:
- name: Build and run candidate
env:
- BENCHMARKS: ${{ steps.vars.outputs.benchmarks }}
- PR_HEAD: ${{ steps.vars.outputs.pr_head }}
+ BENCHMARKS: ${{ needs.extract-info.outputs.benchmarks }}
+ PR_HEAD: ${{ needs.extract-info.outputs.pr_head }}
run: |
- source .venv/bin/activate && cd repo
+ source .venv/bin/activate
./libcxx/utils/build-at-commit --commit $PR_HEAD --install-dir install/candidate -- -DCMAKE_BUILD_TYPE=RelWithDebInfo
./libcxx/utils/test-at-commit --libcxx-installation install/candidate -B benchmarks/candidate -- -sv -j1 --param optimization=speed "$BENCHMARKS"
./libcxx/utils/consolidate-benchmarks benchmarks/candidate | tee candidate.lnt
- name: Compare baseline and candidate runs
run: |
- source .venv/bin/activate && cd repo
+ source .venv/bin/activate
./libcxx/utils/compare-benchmarks baseline.lnt candidate.lnt | tee results.txt
- name: Update comment with results
run: |
- source .venv/bin/activate && cd repo
+ source .venv/bin/activate
cat <<EOF | python
import github
repo = github.Github(auth=github.Auth.Token("${{ github.token }}")).get_repo("${{ github.repository }}")
@@ -144,7 +173,7 @@ jobs:
<details>
<summary>
- Benchmark results:
+ Benchmark results for ${{ matrix.platform }}:
</summary>
\`\`\`
>From 804a081a797c953b86503ead745dfde96f1c61c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= <omeragacan at gmail.com>
Date: Mon, 6 Jul 2026 18:39:43 +0100
Subject: [PATCH 27/46] [GlobalISel][AArch64] Legalize PHIs with vectors
smaller than 64-bits (#207356)
New legalization rule added as the last to not change legalization of
small vectors in existing tests.
---
.../AArch64/GISel/AArch64LegalizerInfo.cpp | 3 +-
.../AArch64/GlobalISel/legalize-phi.mir | 68 +++++++++++++++++++
2 files changed, 70 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index d6bf8af931422..e4e58327ead53 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -156,7 +156,8 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
.clampNumElements(0, v4s16, v8s16)
.clampNumElements(0, v2s32, v4s32)
.clampMaxNumElements(0, s64, 2)
- .clampMaxNumElements(0, p0, 2);
+ .clampMaxNumElements(0, p0, 2)
+ .widenScalarOrEltToNextPow2OrMinSize(0, 8);
getActionDefinitionsBuilder(G_INSERT)
.legalIf(all(typeInSet(0, {s32, s64, p0}), typeInSet(1, {s8, s16, s32}),
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-phi.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-phi.mir
index a94d8cf3094f8..37527cf830fb0 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-phi.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-phi.mir
@@ -946,3 +946,71 @@ body: |
$q0 = COPY %unmerge_1(<2 x p0>)
$q1 = COPY %unmerge_2(<2 x p0>)
RET_ReallyLR implicit $q0, implicit $q1
+...
+---
+name: v4s1
+alignment: 4
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: v4s1
+ ; CHECK: bb.0.entry:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $d0, $w0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $w0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<4 x i16>) = COPY $d0
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i8) = G_CONSTANT i8 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(i32) = G_XOR [[COPY]], [[C1]]
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i16), [[UV1:%[0-9]+]]:_(i16), [[UV2:%[0-9]+]]:_(i16), [[UV3:%[0-9]+]]:_(i16) = G_UNMERGE_VALUES [[COPY1]](<4 x i16>)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i8) = G_TRUNC [[UV]](i16)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(i8) = G_TRUNC [[UV1]](i16)
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(i8) = G_TRUNC [[UV2]](i16)
+ ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(i8) = G_TRUNC [[UV3]](i16)
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(i8) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i8>) = G_BUILD_VECTOR [[TRUNC]](i8), [[TRUNC1]](i8), [[TRUNC2]](i8), [[TRUNC3]](i8), [[DEF]](i8), [[DEF]](i8), [[DEF]](i8), [[DEF]](i8)
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[XOR]], [[C1]]
+ ; CHECK-NEXT: G_BRCOND [[AND]](i32), %bb.2
+ ; CHECK-NEXT: G_BR %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:_(<8 x i8>) = G_PHI %20(<8 x i8>), %bb.2, [[BUILD_VECTOR]](<8 x i8>), %bb.0
+ ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(<8 x i16>) = G_ANYEXT [[PHI]](<8 x i8>)
+ ; CHECK-NEXT: [[UV4:%[0-9]+]]:_(<4 x i16>), [[UV5:%[0-9]+]]:_(<4 x i16>) = G_UNMERGE_VALUES [[ANYEXT]](<8 x i16>)
+ ; CHECK-NEXT: $d0 = COPY [[UV4]](<4 x i16>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $d0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(i8) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<8 x i8>) = G_BUILD_VECTOR [[C]](i8), [[C]](i8), [[C]](i8), [[C]](i8), [[DEF1]](i8), [[DEF1]](i8), [[DEF1]](i8), [[DEF1]](i8)
+ ; CHECK-NEXT: G_BR %bb.1
+ bb.1.entry:
+ successors: %bb.2(0x40000000), %bb.3(0x40000000)
+ liveins: $d0, $w0
+
+ %3:_(i32) = COPY $w0
+ %2:_(i8) = G_TRUNC %3(i32)
+ %4:_(<4 x i16>) = COPY $d0
+ %1:_(<4 x i1>) = G_TRUNC %4(<4 x i16>)
+ %5:_(i8) = G_ASSERT_ZEXT %2, 1
+ %0:_(i1) = G_TRUNC %5(i8)
+ %8:_(i1) = G_CONSTANT i1 false
+ %7:_(<4 x i1>) = G_BUILD_VECTOR %8(i1), %8(i1), %8(i1), %8(i1)
+ %11:_(i1) = G_CONSTANT i1 true
+ %12:_(i1) = G_XOR %0, %11
+ G_BRCOND %12(i1), %bb.3
+ G_BR %bb.2
+
+ bb.2:
+ %9:_(<4 x i1>) = G_PHI %7(<4 x i1>), %bb.3, %1(<4 x i1>), %bb.1
+ %10:_(<4 x i16>) = G_ANYEXT %9(<4 x i1>)
+ $d0 = COPY %10(<4 x i16>)
+ RET_ReallyLR implicit $d0
+
+ bb.3:
+ successors: %bb.2(0x80000000)
+
+ G_BR %bb.2
+...
>From 82d27f67b6aad3bf72e9db610ab5b6984cb02b76 Mon Sep 17 00:00:00 2001
From: Nathan Corbyn <n_corbyn at apple.com>
Date: Mon, 6 Jul 2026 18:48:00 +0100
Subject: [PATCH 28/46] Revert "[ORC] Track __emutls_t definitions in
IRMaterializationUnit" (#207775)
---
clang/test/Interpreter/emulated-tls.cpp | 25 -------------------------
llvm/lib/ExecutionEngine/Orc/Layer.cpp | 1 -
2 files changed, 26 deletions(-)
delete mode 100644 clang/test/Interpreter/emulated-tls.cpp
diff --git a/clang/test/Interpreter/emulated-tls.cpp b/clang/test/Interpreter/emulated-tls.cpp
deleted file mode 100644
index 73afe172ef6d9..0000000000000
--- a/clang/test/Interpreter/emulated-tls.cpp
+++ /dev/null
@@ -1,25 +0,0 @@
-// REQUIRES: host-supports-jit
-// UNSUPPORTED: system-windows
-//
-// An inline function that odr-uses a non-zero-initialized thread_local is
-// emitted as a weak (linkonce_odr) definition into every PartialTranslationUnit
-// that references it. With emulated TLS that set includes an __emutls_t.<var>
-// symbol. When a later PTU re-defines the same weak set, ORC's
-// IRMaterializationUnit::discard() must find each duplicated symbol in its
-// SymbolToDefinition map. The emulated-TLS path used to register __emutls_t.<var>
-// in SymbolFlags but not SymbolToDefinition, so discarding it dereferenced
-// end() -- an assertion failure in +Asserts builds and heap corruption
-// otherwise. Two PTUs each pulling in the same inline worker reproduces it.
-//
-// RUN: cat %s | clang-repl | FileCheck %s
-
-extern "C" int printf(const char *, ...);
-template <int Tag> struct HeavyThing { static thread_local int tls; };
-template <int Tag> thread_local int HeavyThing<Tag>::tls = Tag + 1;
-inline int worker() { return HeavyThing<1>::tls; }
-int callA() { return worker(); }
-int callB() { return worker(); }
-auto r = printf("tls = %d, %d\n", callA(), callB());
-// CHECK: tls = 2, 2
-
-%quit
diff --git a/llvm/lib/ExecutionEngine/Orc/Layer.cpp b/llvm/lib/ExecutionEngine/Orc/Layer.cpp
index 5e95b8c73b482..eb144275da589 100644
--- a/llvm/lib/ExecutionEngine/Orc/Layer.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Layer.cpp
@@ -70,7 +70,6 @@ IRMaterializationUnit::IRMaterializationUnit(
auto EmuTLST = Mangle(("__emutls_t." + GV.getName()).str());
SymbolFlags[EmuTLST] = Flags;
- SymbolToDefinition[EmuTLST] = &GV;
}
continue;
}
>From 96904e83cf69b0a02a5fb7dece9b258f562205e0 Mon Sep 17 00:00:00 2001
From: Christopher Ferris <cferris1000 at users.noreply.github.com>
Date: Mon, 6 Jul 2026 17:54:14 +0000
Subject: [PATCH 29/46] [scudo] Make death tests use SCUDO_XXX_DEATH_TEST.
(#206196)
On Android, we want to disable the tombstone generation on expected
crashes to speed up running the tests. Also, this means that any
tombstone files will be real crashes and it's not necessary to search
for the right tombstone if one of the tests crashes unexpectedly.
In order to do the above, add SCUDO_EXPECT_DEATH_TEST and
SCUDO_ASSERT_DEATH_TEST that will disable tombstoned on Android but do
nothing on all other platforms.
---
.../lib/scudo/standalone/tests/chunk_test.cpp | 4 +-
.../scudo/standalone/tests/combined_test.cpp | 126 +++++++---------
.../lib/scudo/standalone/tests/flags_test.cpp | 33 +++--
.../lib/scudo/standalone/tests/map_test.cpp | 6 +-
.../scudo/standalone/tests/memtag_test.cpp | 47 +++---
.../scudo/standalone/tests/report_test.cpp | 134 ++++++++++--------
.../scudo/standalone/tests/scudo_unit_test.h | 29 ++++
.../scudo/standalone/tests/secondary_test.cpp | 2 +-
.../standalone/tests/wrappers_c_test.cpp | 56 ++++----
.../standalone/tests/wrappers_cpp_test.cpp | 8 +-
10 files changed, 234 insertions(+), 211 deletions(-)
diff --git a/compiler-rt/lib/scudo/standalone/tests/chunk_test.cpp b/compiler-rt/lib/scudo/standalone/tests/chunk_test.cpp
index 1b2c1eb5a7d0c..ded1867f0ff45 100644
--- a/compiler-rt/lib/scudo/standalone/tests/chunk_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/chunk_test.cpp
@@ -33,7 +33,7 @@ TEST(ScudoChunkDeathTest, ChunkBasic) {
scudo::Chunk::loadHeader(Cookie, P, &Header);
EXPECT_TRUE(scudo::Chunk::isValid(Cookie, P, &Header));
EXPECT_FALSE(scudo::Chunk::isValid(InvalidCookie, P, &Header));
- EXPECT_DEATH(scudo::Chunk::loadHeader(InvalidCookie, P, &Header), "");
+ SCUDO_EXPECT_DEATH(scudo::Chunk::loadHeader(InvalidCookie, P, &Header), "");
free(Block);
}
@@ -50,7 +50,7 @@ TEST(ScudoChunkDeathTest, CorruptHeader) {
// Simulate a couple of corrupted bits per byte of header data.
for (scudo::uptr I = 0; I < sizeof(scudo::Chunk::PackedHeader); I++) {
*(reinterpret_cast<scudo::u8 *>(Block) + I) ^= 0x42U;
- EXPECT_DEATH(scudo::Chunk::loadHeader(Cookie, P, &Header), "");
+ SCUDO_EXPECT_DEATH(scudo::Chunk::loadHeader(Cookie, P, &Header), "");
*(reinterpret_cast<scudo::u8 *>(Block) + I) ^= 0x42U;
}
free(Block);
diff --git a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
index 528afc449694d..4116cf840a7f1 100644
--- a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
@@ -32,15 +32,6 @@
static constexpr scudo::Chunk::Origin Origin = scudo::Chunk::Origin::Malloc;
static constexpr scudo::uptr MinAlignLog = FIRST_32_SECOND_64(3U, 4U);
-// Fuchsia complains that the function is not used.
-UNUSED static void disableDebuggerdMaybe() {
-#if SCUDO_ANDROID
- // Disable the debuggerd signal handler on Android, without this we can end
- // up spending a significant amount of time creating tombstones.
- signal(SIGSEGV, SIG_DFL);
-#endif
-}
-
template <class AllocatorT>
bool isPrimaryAllocation(scudo::uptr Size, scudo::uptr Alignment) {
const scudo::uptr MinAlignment = 1UL << SCUDO_MIN_ALIGNMENT_LOG;
@@ -58,23 +49,13 @@ void checkMemoryTaggingMaybe(AllocatorT *Allocator, void *P, scudo::uptr Size,
const scudo::uptr MinAlignment = 1UL << SCUDO_MIN_ALIGNMENT_LOG;
Size = scudo::roundUp(Size, MinAlignment);
if (Allocator->useMemoryTaggingTestOnly()) {
- EXPECT_DEATH(
- {
- disableDebuggerdMaybe();
- reinterpret_cast<char *>(P)[-1] = 'A';
- },
- "");
+ SCUDO_EXPECT_DEATH(reinterpret_cast<char *>(P)[-1] = 'A', "");
}
if (isPrimaryAllocation<AllocatorT>(Size, Alignment)
? Allocator->useMemoryTaggingTestOnly()
: Alignment == MinAlignment &&
AllocatorT::SecondaryT::getGuardPageSize() > 0) {
- EXPECT_DEATH(
- {
- disableDebuggerdMaybe();
- reinterpret_cast<char *>(P)[Size] = 'A';
- },
- "");
+ SCUDO_EXPECT_DEATH(reinterpret_cast<char *>(P)[Size] = 'A', "");
}
}
@@ -551,12 +532,7 @@ SCUDO_TYPED_TEST(ScudoCombinedDeathTest, ReallocateDecreasingTagged) {
// Check that accessing the entire allocation after new size causes a crash.
for (size_t I = NewSize; I < Size; I++) {
- EXPECT_DEATH(
- {
- disableDebuggerdMaybe();
- reinterpret_cast<char *>(NewP)[I] = 'A';
- },
- "");
+ SCUDO_EXPECT_DEATH(reinterpret_cast<char *>(NewP)[I] = 'A', "");
}
Allocator->deallocate(P, Origin);
}
@@ -594,17 +570,15 @@ SCUDO_TYPED_TEST(ScudoCombinedDeathTest, UseAfterFree) {
const scudo::uptr Size = 1U << SizeLog;
if (!Allocator->useMemoryTaggingTestOnly())
continue;
- EXPECT_DEATH(
+ SCUDO_EXPECT_DEATH(
{
- disableDebuggerdMaybe();
void *P = Allocator->allocate(Size, Origin);
Allocator->deallocate(P, Origin);
reinterpret_cast<char *>(P)[0] = 'A';
},
"");
- EXPECT_DEATH(
+ SCUDO_EXPECT_DEATH(
{
- disableDebuggerdMaybe();
void *P = Allocator->allocate(Size, Origin);
Allocator->deallocate(P, Origin);
reinterpret_cast<char *>(P)[Size - 1] = 'A';
@@ -622,7 +596,7 @@ SCUDO_TYPED_TEST(ScudoCombinedDeathTest, DoubleFreeFromPrimary) {
break;
// Verify that a double free results in a chunk state error.
- EXPECT_DEATH(
+ SCUDO_EXPECT_DEATH(
{
// Allocate from primary
void *P = Allocator->allocate(Size, Origin);
@@ -640,7 +614,7 @@ SCUDO_TYPED_TEST(ScudoCombinedDeathTest, DisableMemoryTagging) {
if (Allocator->useMemoryTaggingTestOnly()) {
// Check that disabling memory tagging works correctly.
void *P = Allocator->allocate(2048, Origin);
- EXPECT_DEATH(reinterpret_cast<char *>(P)[2048] = 'A', "");
+ SCUDO_EXPECT_DEATH(reinterpret_cast<char *>(P)[2048] = 'A', "");
scudo::ScopedDisableMemoryTagChecks NoTagChecks;
Allocator->disableMemoryTagging();
reinterpret_cast<char *>(P)[2048] = 'A';
@@ -777,7 +751,7 @@ TEST(ScudoCombinedDeathTest, SKIP_ON_FUCHSIA(testSEGV)) {
ASSERT_TRUE(ReservedMemory.create(/*Addr=*/0U, Size, "testSEGV"));
void *P = reinterpret_cast<void *>(ReservedMemory.getBase());
ASSERT_NE(P, nullptr);
- EXPECT_DEATH(memset(P, 0xaa, Size), "");
+ SCUDO_EXPECT_DEATH(memset(P, 0xaa, Size), "");
ReservedMemory.release();
}
@@ -828,28 +802,28 @@ TEST(ScudoCombinedDeathTest, DeathCombined) {
EXPECT_NE(P, nullptr);
// Invalid sized deallocation.
- EXPECT_DEATH(Allocator->deallocateSized(P, Origin, Size + 8U), "");
+ SCUDO_EXPECT_DEATH(Allocator->deallocateSized(P, Origin, Size + 8U), "");
// Misaligned pointer. Potentially unused if EXPECT_DEATH isn't available.
UNUSED void *MisalignedP =
reinterpret_cast<void *>(reinterpret_cast<scudo::uptr>(P) | 1U);
- EXPECT_DEATH(Allocator->deallocateSized(MisalignedP, Origin, Size), "");
- EXPECT_DEATH(Allocator->reallocate(MisalignedP, Size * 2U), "");
+ SCUDO_EXPECT_DEATH(Allocator->deallocateSized(MisalignedP, Origin, Size), "");
+ SCUDO_EXPECT_DEATH(Allocator->reallocate(MisalignedP, Size * 2U), "");
// Header corruption.
scudo::u64 *H =
reinterpret_cast<scudo::u64 *>(scudo::Chunk::getAtomicHeader(P));
*H ^= 0x42U;
- EXPECT_DEATH(Allocator->deallocateSized(P, Origin, Size), "");
+ SCUDO_EXPECT_DEATH(Allocator->deallocateSized(P, Origin, Size), "");
*H ^= 0x420042U;
- EXPECT_DEATH(Allocator->deallocateSized(P, Origin, Size), "");
+ SCUDO_EXPECT_DEATH(Allocator->deallocateSized(P, Origin, Size), "");
*H ^= 0x420000U;
// Invalid chunk state.
Allocator->deallocateSized(P, Origin, Size);
- EXPECT_DEATH(Allocator->deallocateSized(P, Origin, Size), "");
- EXPECT_DEATH(Allocator->reallocate(P, Size * 2U), "");
- EXPECT_DEATH(Allocator->getUsableSize(P), "");
+ SCUDO_EXPECT_DEATH(Allocator->deallocateSized(P, Origin, Size), "");
+ SCUDO_EXPECT_DEATH(Allocator->reallocate(P, Size * 2U), "");
+ SCUDO_EXPECT_DEATH(Allocator->getUsableSize(P), "");
}
// Verify that when a region gets full, the allocator will still manage to
@@ -1836,7 +1810,7 @@ TEST(ScudoCombinedDeathTest, DeallocateAlignNotPowerOfTwo) {
auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());
void *Ptr = Allocator->allocate(10, scudo::Chunk::Origin::Memalign, 32);
- EXPECT_DEATH(
+ SCUDO_EXPECT_DEATH(
Allocator->deallocateAligned(Ptr, scudo::Chunk::Origin::Memalign, 9),
"alignment must be a power of two");
Allocator->deallocateAligned(Ptr, scudo::Chunk::Origin::Memalign, 32);
@@ -1850,26 +1824,26 @@ TEST(ScudoCombinedDeathTest, TypeMismatch) {
void *Ptr = Allocator->allocate(10, scudo::Chunk::Origin::Malloc);
EXPECT_TRUE(Ptr != nullptr);
- EXPECT_DEATH(Allocator->deallocate(Ptr, scudo::Chunk::Origin::New),
- "deallocating.*\\(malloc vs new\\)");
- EXPECT_DEATH(Allocator->deallocate(Ptr, scudo::Chunk::Origin::NewArray),
- "deallocating.*\\(malloc vs new\\[\\]\\)");
+ SCUDO_EXPECT_DEATH(Allocator->deallocate(Ptr, scudo::Chunk::Origin::New),
+ "deallocating.*\\(malloc vs new\\)");
+ SCUDO_EXPECT_DEATH(Allocator->deallocate(Ptr, scudo::Chunk::Origin::NewArray),
+ "deallocating.*\\(malloc vs new\\[\\]\\)");
Allocator->deallocate(Ptr, scudo::Chunk::Origin::Malloc);
Ptr = Allocator->allocate(10, scudo::Chunk::Origin::New);
EXPECT_TRUE(Ptr != nullptr);
- EXPECT_DEATH(Allocator->deallocate(Ptr, scudo::Chunk::Origin::Malloc),
- "deallocating.*\\(new vs malloc\\)");
- EXPECT_DEATH(Allocator->deallocate(Ptr, scudo::Chunk::Origin::NewArray),
- "deallocating.*\\(new vs new\\[\\]\\)");
+ SCUDO_EXPECT_DEATH(Allocator->deallocate(Ptr, scudo::Chunk::Origin::Malloc),
+ "deallocating.*\\(new vs malloc\\)");
+ SCUDO_EXPECT_DEATH(Allocator->deallocate(Ptr, scudo::Chunk::Origin::NewArray),
+ "deallocating.*\\(new vs new\\[\\]\\)");
Allocator->deallocate(Ptr, scudo::Chunk::Origin::New);
Ptr = Allocator->allocate(10, scudo::Chunk::Origin::NewArray);
EXPECT_TRUE(Ptr != nullptr);
- EXPECT_DEATH(Allocator->deallocate(Ptr, scudo::Chunk::Origin::Malloc),
- "deallocating.*\\(new\\[\\] vs malloc\\)");
- EXPECT_DEATH(Allocator->deallocate(Ptr, scudo::Chunk::Origin::New),
- "deallocating.*\\(new\\[\\] vs new\\)");
+ SCUDO_EXPECT_DEATH(Allocator->deallocate(Ptr, scudo::Chunk::Origin::Malloc),
+ "deallocating.*\\(new\\[\\] vs malloc\\)");
+ SCUDO_EXPECT_DEATH(Allocator->deallocate(Ptr, scudo::Chunk::Origin::New),
+ "deallocating.*\\(new\\[\\] vs new\\)");
Allocator->deallocate(Ptr, scudo::Chunk::Origin::NewArray);
}
@@ -1881,20 +1855,20 @@ TEST(ScudoCombinedDeathTest, ReallocTypeMismatch) {
void *Ptr = Allocator->allocate(10, scudo::Chunk::Origin::New);
EXPECT_TRUE(Ptr != nullptr);
- EXPECT_DEATH(Allocator->reallocate(Ptr, 1000000),
- "reallocating.*\\(new vs malloc\\)");
+ SCUDO_EXPECT_DEATH(Allocator->reallocate(Ptr, 1000000),
+ "reallocating.*\\(new vs malloc\\)");
Allocator->deallocate(Ptr, scudo::Chunk::Origin::New);
Ptr = Allocator->allocate(10, scudo::Chunk::Origin::NewArray);
EXPECT_TRUE(Ptr != nullptr);
- EXPECT_DEATH(Allocator->reallocate(Ptr, 1000000),
- "reallocating.*\\(new\\[\\] vs malloc\\)");
+ SCUDO_EXPECT_DEATH(Allocator->reallocate(Ptr, 1000000),
+ "reallocating.*\\(new\\[\\] vs malloc\\)");
Allocator->deallocate(Ptr, scudo::Chunk::Origin::NewArray);
Ptr = Allocator->allocate(10, scudo::Chunk::Origin::Memalign, 32);
EXPECT_TRUE(Ptr != nullptr);
- EXPECT_DEATH(Allocator->reallocate(Ptr, 1000000),
- "reallocating.*\\(aligned malloc vs malloc\\)");
+ SCUDO_EXPECT_DEATH(Allocator->reallocate(Ptr, 1000000),
+ "reallocating.*\\(aligned malloc vs malloc\\)");
Allocator->deallocateAligned(Ptr, scudo::Chunk::Origin::Memalign, 32);
}
@@ -1906,8 +1880,8 @@ TEST(ScudoCombinedDeathTest, AlignTypeMismatch) {
void *Ptr = Allocator->allocate(10, scudo::Chunk::Origin::Memalign, 32);
EXPECT_TRUE(Ptr != nullptr);
- EXPECT_DEATH(Allocator->reallocate(Ptr, 1000),
- "reallocating.*\\(aligned malloc vs malloc\\)");
+ SCUDO_EXPECT_DEATH(Allocator->reallocate(Ptr, 1000),
+ "reallocating.*\\(aligned malloc vs malloc\\)");
// Aligned allocate with non-aligned deallocate is okay.
Allocator->deallocate(Ptr, scudo::Chunk::Origin::Malloc);
}
@@ -1937,16 +1911,17 @@ TEST(ScudoCombinedDeathTest, SizeMismatch) {
void *Ptr = Allocator->allocate(10, scudo::Chunk::Origin::Malloc);
EXPECT_TRUE(Ptr != nullptr);
- EXPECT_DEATH(
+ SCUDO_EXPECT_DEATH(
Allocator->deallocateSized(Ptr, scudo::Chunk::Origin::Malloc, 1000000),
"invalid sized delete when deallocating.*\\(1000000 vs 10\\)");
Allocator->deallocate(Ptr, scudo::Chunk::Origin::Malloc);
Ptr = Allocator->allocate(10, scudo::Chunk::Origin::Memalign, 32);
EXPECT_TRUE(Ptr != nullptr);
- EXPECT_DEATH(Allocator->deallocateSizedAligned(
- Ptr, scudo::Chunk::Origin::Malloc, 1000000, 32),
- "invalid sized delete when deallocating.*\\(1000000 vs 10\\)");
+ SCUDO_EXPECT_DEATH(
+ Allocator->deallocateSizedAligned(Ptr, scudo::Chunk::Origin::Malloc,
+ 1000000, 32),
+ "invalid sized delete when deallocating.*\\(1000000 vs 10\\)");
Allocator->deallocateAligned(Ptr, scudo::Chunk::Origin::Malloc, 32);
}
@@ -1962,14 +1937,14 @@ TEST(ScudoCombinedDeathTest, SizeNotExactUsableMismatch) {
void *Ptr = Allocator->allocate(10, scudo::Chunk::Origin::Malloc);
EXPECT_TRUE(Ptr != nullptr);
- EXPECT_DEATH(
+ SCUDO_EXPECT_DEATH(
Allocator->deallocateSized(Ptr, scudo::Chunk::Origin::Malloc, 1000000),
"invalid sized delete when deallocating.*\\(1000000 vs 10 or .*\\)");
Allocator->deallocateSized(Ptr, scudo::Chunk::Origin::Malloc, 10);
Ptr = Allocator->allocate(10, scudo::Chunk::Origin::Memalign, 32);
EXPECT_TRUE(Ptr != nullptr);
- EXPECT_DEATH(
+ SCUDO_EXPECT_DEATH(
Allocator->deallocateSizedAligned(Ptr, scudo::Chunk::Origin::Malloc,
1000000, 32),
"invalid sized delete when deallocating.*\\(1000000 vs 10 or .*\\)");
@@ -2019,12 +1994,13 @@ TEST(ScudoCombinedDeathTest, AlignMismatch) {
}
scudo::uptr Alignment = 2 * scudo::getPageSizeCached();
- EXPECT_DEATH(Allocator->deallocateAligned(
- AlignedPtr, scudo::Chunk::Origin::Malloc, Alignment),
- "invalid aligned delete when deallocating");
- EXPECT_DEATH(Allocator->deallocateSizedAligned(
- AlignedPtr, scudo::Chunk::Origin::Malloc, 10, Alignment),
- "invalid aligned delete when deallocating");
+ SCUDO_EXPECT_DEATH(Allocator->deallocateAligned(
+ AlignedPtr, scudo::Chunk::Origin::Malloc, Alignment),
+ "invalid aligned delete when deallocating");
+ SCUDO_EXPECT_DEATH(
+ Allocator->deallocateSizedAligned(
+ AlignedPtr, scudo::Chunk::Origin::Malloc, 10, Alignment),
+ "invalid aligned delete when deallocating");
Allocator->deallocateAligned(AlignedPtr, scudo::Chunk::Origin::Malloc, 32);
}
diff --git a/compiler-rt/lib/scudo/standalone/tests/flags_test.cpp b/compiler-rt/lib/scudo/standalone/tests/flags_test.cpp
index 591611edc425a..1bac283285328 100644
--- a/compiler-rt/lib/scudo/standalone/tests/flags_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/flags_test.cpp
@@ -41,16 +41,21 @@ TEST(ScudoFlagsTest, BooleanFlags) {
}
TEST(ScudoFlagsDeathTest, BooleanFlags) {
- EXPECT_DEATH(testFlag(scudo::FlagType::FT_bool, false, "flag_name", true),
- "expected '='");
- EXPECT_DEATH(testFlag(scudo::FlagType::FT_bool, false, "flag_name=", true),
- "invalid value for bool option: ''");
- EXPECT_DEATH(testFlag(scudo::FlagType::FT_bool, false, "flag_name=2", true),
- "invalid value for bool option: '2'");
- EXPECT_DEATH(testFlag(scudo::FlagType::FT_bool, false, "flag_name=-1", true),
- "invalid value for bool option: '-1'");
- EXPECT_DEATH(testFlag(scudo::FlagType::FT_bool, false, "flag_name=on", true),
- "invalid value for bool option: 'on'");
+ SCUDO_EXPECT_DEATH(
+ testFlag(scudo::FlagType::FT_bool, false, "flag_name", true),
+ "expected '='");
+ SCUDO_EXPECT_DEATH(
+ testFlag(scudo::FlagType::FT_bool, false, "flag_name=", true),
+ "invalid value for bool option: ''");
+ SCUDO_EXPECT_DEATH(
+ testFlag(scudo::FlagType::FT_bool, false, "flag_name=2", true),
+ "invalid value for bool option: '2'");
+ SCUDO_EXPECT_DEATH(
+ testFlag(scudo::FlagType::FT_bool, false, "flag_name=-1", true),
+ "invalid value for bool option: '-1'");
+ SCUDO_EXPECT_DEATH(
+ testFlag(scudo::FlagType::FT_bool, false, "flag_name=on", true),
+ "invalid value for bool option: 'on'");
}
TEST(ScudoFlagsTest, IntFlags) {
@@ -67,10 +72,10 @@ TEST(ScudoFlagsTest, IntFlags) {
}
TEST(ScudoFlagsDeathTest, IntFlags) {
- EXPECT_DEATH(testFlag(scudo::FlagType::FT_int, -11, "flag_name", 0),
- "expected '='");
- EXPECT_DEATH(testFlag(scudo::FlagType::FT_int, -11, "flag_name=42U", 0),
- "invalid value for int option");
+ SCUDO_EXPECT_DEATH(testFlag(scudo::FlagType::FT_int, -11, "flag_name", 0),
+ "expected '='");
+ SCUDO_EXPECT_DEATH(testFlag(scudo::FlagType::FT_int, -11, "flag_name=42U", 0),
+ "invalid value for int option");
}
static void testTwoFlags(const char *Env, bool ExpectedFlag1,
diff --git a/compiler-rt/lib/scudo/standalone/tests/map_test.cpp b/compiler-rt/lib/scudo/standalone/tests/map_test.cpp
index 620dffca33b83..051a466299c6c 100644
--- a/compiler-rt/lib/scudo/standalone/tests/map_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/map_test.cpp
@@ -129,7 +129,7 @@ TEST(ScudoMapDeathTest, MapNoAccessUnmap) {
ASSERT_TRUE(ReservedMemory.create(/*Addr=*/0U, Size, MappingName));
EXPECT_NE(ReservedMemory.getBase(), 0U);
- EXPECT_DEATH(
+ SCUDO_EXPECT_DEATH(
memset(reinterpret_cast<void *>(ReservedMemory.getBase()), 0xaa, Size),
"");
@@ -138,7 +138,7 @@ TEST(ScudoMapDeathTest, MapNoAccessUnmap) {
TEST(ScudoMapDeathTest, MapUnmap) {
const scudo::uptr Size = 4 * scudo::getPageSizeCached();
- EXPECT_DEATH(
+ SCUDO_EXPECT_DEATH(
{
// Repeat few time to avoid missing crash if it's mmaped by unrelated
// code.
@@ -169,7 +169,7 @@ TEST(ScudoMapDeathTest, MapWithGuardUnmap) {
scudo::uptr Q = MemMap.getBase() + PageSize;
ASSERT_TRUE(MemMap.remap(Q, Size, MappingName));
memset(reinterpret_cast<void *>(Q), 0xaa, Size);
- EXPECT_DEATH(memset(reinterpret_cast<void *>(Q), 0xaa, Size + 1), "");
+ SCUDO_EXPECT_DEATH(memset(reinterpret_cast<void *>(Q), 0xaa, Size + 1), "");
MemMap.unmap();
}
diff --git a/compiler-rt/lib/scudo/standalone/tests/memtag_test.cpp b/compiler-rt/lib/scudo/standalone/tests/memtag_test.cpp
index d0d93316f212e..4f971bbc2e806 100644
--- a/compiler-rt/lib/scudo/standalone/tests/memtag_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/memtag_test.cpp
@@ -24,23 +24,24 @@ TEST(MemtagBasicDeathTest, Unsupported) {
if (&__hwasan_init != 0)
TEST_SKIP("Incompatible with HWASan");
- EXPECT_DEATH(archMemoryTagGranuleSize(), "not supported");
- EXPECT_DEATH(untagPointer((uptr)0), "not supported");
- EXPECT_DEATH(extractTag((uptr)0), "not supported");
-
- EXPECT_DEATH(systemDetectsMemoryTagFaultsTestOnly(), "not supported");
- EXPECT_DEATH(enableSystemMemoryTaggingTestOnly(), "not supported");
-
- EXPECT_DEATH(selectRandomTag((uptr)0, 0), "not supported");
- EXPECT_DEATH(addFixedTag((uptr)0, 1), "not supported");
- EXPECT_DEATH(storeTags((uptr)0, (uptr)0 + sizeof(0)), "not supported");
- EXPECT_DEATH(storeTag((uptr)0), "not supported");
- EXPECT_DEATH(loadTag((uptr)0), "not supported");
-
- EXPECT_DEATH(setRandomTag(nullptr, 64, 0, nullptr, nullptr), "not supported");
- EXPECT_DEATH(untagPointer(nullptr), "not supported");
- EXPECT_DEATH(loadTag(nullptr), "not supported");
- EXPECT_DEATH(addFixedTag(nullptr, 0), "not supported");
+ SCUDO_EXPECT_DEATH(archMemoryTagGranuleSize(), "not supported");
+ SCUDO_EXPECT_DEATH(untagPointer((uptr)0), "not supported");
+ SCUDO_EXPECT_DEATH(extractTag((uptr)0), "not supported");
+
+ SCUDO_EXPECT_DEATH(systemDetectsMemoryTagFaultsTestOnly(), "not supported");
+ SCUDO_EXPECT_DEATH(enableSystemMemoryTaggingTestOnly(), "not supported");
+
+ SCUDO_EXPECT_DEATH(selectRandomTag((uptr)0, 0), "not supported");
+ SCUDO_EXPECT_DEATH(addFixedTag((uptr)0, 1), "not supported");
+ SCUDO_EXPECT_DEATH(storeTags((uptr)0, (uptr)0 + sizeof(0)), "not supported");
+ SCUDO_EXPECT_DEATH(storeTag((uptr)0), "not supported");
+ SCUDO_EXPECT_DEATH(loadTag((uptr)0), "not supported");
+
+ SCUDO_EXPECT_DEATH(setRandomTag(nullptr, 64, 0, nullptr, nullptr),
+ "not supported");
+ SCUDO_EXPECT_DEATH(untagPointer(nullptr), "not supported");
+ SCUDO_EXPECT_DEATH(loadTag(nullptr), "not supported");
+ SCUDO_EXPECT_DEATH(addFixedTag(nullptr, 0), "not supported");
}
class MemtagTest : public Test {
@@ -96,8 +97,8 @@ TEST_F(MemtagDeathTest, AddFixedTag) {
for (uptr Tag = 0; Tag < 0x10; ++Tag)
EXPECT_EQ(Tag, extractTag(addFixedTag(Addr, Tag)));
if (SCUDO_DEBUG) {
- EXPECT_DEATH(addFixedTag(Addr, 16), "");
- EXPECT_DEATH(addFixedTag(~Addr, 0), "");
+ SCUDO_EXPECT_DEATH(addFixedTag(Addr, 16), "");
+ SCUDO_EXPECT_DEATH(addFixedTag(~Addr, 0), "");
}
}
@@ -113,7 +114,7 @@ TEST_F(MemtagDeathTest, ScopedDisableMemoryTagChecks) {
u8 *P = reinterpret_cast<u8 *>(addFixedTag(Addr, 1));
EXPECT_NE(P, Buffer);
- EXPECT_DEATH(*P = 20, "");
+ SCUDO_EXPECT_DEATH(*P = 20, "");
ScopedDisableMemoryTagChecks Disable;
*P = 10;
}
@@ -151,8 +152,8 @@ TEST_F(MemtagDeathTest, SKIP_NO_DEBUG(LoadStoreTagUnaligned)) {
for (uptr P = Addr; P < Addr + 4 * archMemoryTagGranuleSize(); ++P) {
if (P % archMemoryTagGranuleSize() == 0)
continue;
- EXPECT_DEATH(loadTag(P), "");
- EXPECT_DEATH(storeTag(P), "");
+ SCUDO_EXPECT_DEATH(loadTag(P), "");
+ SCUDO_EXPECT_DEATH(storeTag(P), "");
}
}
@@ -173,7 +174,7 @@ TEST_F(MemtagDeathTest, SKIP_NO_DEBUG(StoreTagsUnaligned)) {
uptr Tagged = addFixedTag(P, 5);
if (Tagged % archMemoryTagGranuleSize() == 0)
continue;
- EXPECT_DEATH(storeTags(Tagged, Tagged), "");
+ SCUDO_EXPECT_DEATH(storeTags(Tagged, Tagged), "");
}
}
diff --git a/compiler-rt/lib/scudo/standalone/tests/report_test.cpp b/compiler-rt/lib/scudo/standalone/tests/report_test.cpp
index e69f5dcc24fa7..f82c70838d731 100644
--- a/compiler-rt/lib/scudo/standalone/tests/report_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/report_test.cpp
@@ -13,82 +13,90 @@
TEST(ScudoReportDeathTest, Check) {
CHECK_LT(-1, 1);
- EXPECT_DEATH(CHECK_GT(-1, 1),
- "\\(-1\\) > \\(1\\) \\(\\(u64\\)op1=18446744073709551615, "
- "\\(u64\\)op2=1");
+ SCUDO_EXPECT_DEATH(CHECK_GT(-1, 1),
+ "\\(-1\\) > \\(1\\) \\(\\(u64\\)op1=18446744073709551615, "
+ "\\(u64\\)op2=1");
}
TEST(ScudoReportDeathTest, Generic) {
// Potentially unused if EXPECT_DEATH isn't defined.
UNUSED void *P = reinterpret_cast<void *>(0x42424242U);
UNUSED scudo::Chunk::PackedHeader Header = {};
- EXPECT_DEATH(scudo::reportError("TEST123"), "Scudo ERROR.*TEST123");
- EXPECT_DEATH(scudo::reportInvalidFlag("ABC", "DEF"), "Scudo ERROR.*ABC.*DEF");
- EXPECT_DEATH(scudo::reportHeaderCorruption(&Header, P),
- "Scudo ERROR.*42424242");
- EXPECT_DEATH(scudo::reportSanityCheckError("XYZ"), "Scudo ERROR.*XYZ");
- EXPECT_DEATH(scudo::reportAlignmentTooBig(123, 456), "Scudo ERROR.*123.*456");
- EXPECT_DEATH(scudo::reportAllocationSizeTooBig(123, 456, 789),
- "Scudo ERROR.*123.*456.*789");
- EXPECT_DEATH(scudo::reportOutOfMemory(4242), "Scudo ERROR.*4242");
- EXPECT_DEATH(
+ SCUDO_EXPECT_DEATH(scudo::reportError("TEST123"), "Scudo ERROR.*TEST123");
+ SCUDO_EXPECT_DEATH(scudo::reportInvalidFlag("ABC", "DEF"),
+ "Scudo ERROR.*ABC.*DEF");
+ SCUDO_EXPECT_DEATH(scudo::reportHeaderCorruption(&Header, P),
+ "Scudo ERROR.*42424242");
+ SCUDO_EXPECT_DEATH(scudo::reportSanityCheckError("XYZ"), "Scudo ERROR.*XYZ");
+ SCUDO_EXPECT_DEATH(scudo::reportAlignmentTooBig(123, 456),
+ "Scudo ERROR.*123.*456");
+ SCUDO_EXPECT_DEATH(scudo::reportAllocationSizeTooBig(123, 456, 789),
+ "Scudo ERROR.*123.*456.*789");
+ SCUDO_EXPECT_DEATH(scudo::reportOutOfMemory(4242), "Scudo ERROR.*4242");
+ SCUDO_EXPECT_DEATH(
scudo::reportInvalidChunkState(scudo::AllocatorAction::Recycling, P),
"Scudo ERROR.*recycling.*42424242");
- EXPECT_DEATH(
+ SCUDO_EXPECT_DEATH(
scudo::reportInvalidChunkState(scudo::AllocatorAction::Sizing, P),
"Scudo ERROR.*sizing.*42424242");
- EXPECT_DEATH(
+ SCUDO_EXPECT_DEATH(
scudo::reportMisalignedPointer(scudo::AllocatorAction::Deallocating, P),
"Scudo ERROR.*deallocating.*42424242");
- EXPECT_DEATH(
+ SCUDO_EXPECT_DEATH(
scudo::reportDeallocTypeMismatch(scudo::AllocatorAction::Reallocating, P,
scudo::Chunk::Origin::New,
scudo::Chunk::Origin::Memalign),
"Scudo ERROR.*reallocating.*42424242.*\\(new vs aligned malloc\\)");
- EXPECT_DEATH(scudo::reportDeallocTypeMismatch(
- scudo::AllocatorAction::Deallocating, P,
- scudo::Chunk::Origin::Memalign, scudo::Chunk::Origin::New),
- "Scudo ERROR.*deallocating.*\\(aligned malloc vs new\\)");
- EXPECT_DEATH(scudo::reportDeallocTypeMismatch(
- scudo::AllocatorAction::Deallocating, P,
- scudo::Chunk::Origin::Malloc | scudo::Chunk::Origin::Size,
- scudo::Chunk::Origin::NewArray | scudo::Chunk::Origin::Size |
- scudo::Chunk::Origin::Align),
- "Scudo ERROR.*deallocating.*\\(sized malloc vs sized aligned "
- "new\\[\\]\\)");
- EXPECT_DEATH(scudo::reportDeleteSizeMismatch(P, 123, 456),
- "Scudo ERROR.*42424242.*\\(123 vs 456\\)");
- EXPECT_DEATH(scudo::reportDeleteSizeMismatch(P, 123, 456, 789),
- "Scudo ERROR.*42424242.*\\(123 vs 456 or 789\\)");
- EXPECT_DEATH(
+ SCUDO_EXPECT_DEATH(
+ scudo::reportDeallocTypeMismatch(scudo::AllocatorAction::Deallocating, P,
+ scudo::Chunk::Origin::Memalign,
+ scudo::Chunk::Origin::New),
+ "Scudo ERROR.*deallocating.*\\(aligned malloc vs new\\)");
+ SCUDO_EXPECT_DEATH(
+ scudo::reportDeallocTypeMismatch(
+ scudo::AllocatorAction::Deallocating, P,
+ scudo::Chunk::Origin::Malloc | scudo::Chunk::Origin::Size,
+ scudo::Chunk::Origin::NewArray | scudo::Chunk::Origin::Size |
+ scudo::Chunk::Origin::Align),
+ "Scudo ERROR.*deallocating.*\\(sized malloc vs sized aligned "
+ "new\\[\\]\\)");
+ SCUDO_EXPECT_DEATH(scudo::reportDeleteSizeMismatch(P, 123, 456),
+ "Scudo ERROR.*42424242.*\\(123 vs 456\\)");
+ SCUDO_EXPECT_DEATH(scudo::reportDeleteSizeMismatch(P, 123, 456, 789),
+ "Scudo ERROR.*42424242.*\\(123 vs 456 or 789\\)");
+ SCUDO_EXPECT_DEATH(
scudo::reportDeleteAlignmentMismatch(reinterpret_cast<void *>(0x80),
0x100),
"Scudo ERROR.*invalid aligned delete.*\\(7 bit align vs 8 bit align\\)");
}
TEST(ScudoReportDeathTest, CSpecific) {
- EXPECT_DEATH(scudo::reportAlignmentNotPowerOfTwo(123), "Scudo ERROR.*123");
- EXPECT_DEATH(scudo::reportCallocOverflow(123, 456), "Scudo ERROR.*123.*456");
- EXPECT_DEATH(scudo::reportReallocarrayOverflow(123, 456),
- "Scudo ERROR.*reallocarray parameters.*123.*456");
- EXPECT_DEATH(scudo::reportInvalidPosixMemalignAlignment(789),
- "Scudo ERROR.*789");
- EXPECT_DEATH(scudo::reportPvallocOverflow(123), "Scudo ERROR.*123");
- EXPECT_DEATH(scudo::reportInvalidAlignedAllocAlignment(123, 456),
- "Scudo ERROR.*123.*456");
+ SCUDO_EXPECT_DEATH(scudo::reportAlignmentNotPowerOfTwo(123),
+ "Scudo ERROR.*123");
+ SCUDO_EXPECT_DEATH(scudo::reportCallocOverflow(123, 456),
+ "Scudo ERROR.*123.*456");
+ SCUDO_EXPECT_DEATH(scudo::reportReallocarrayOverflow(123, 456),
+ "Scudo ERROR.*reallocarray parameters.*123.*456");
+ SCUDO_EXPECT_DEATH(scudo::reportInvalidPosixMemalignAlignment(789),
+ "Scudo ERROR.*789");
+ SCUDO_EXPECT_DEATH(scudo::reportPvallocOverflow(123), "Scudo ERROR.*123");
+ SCUDO_EXPECT_DEATH(scudo::reportInvalidAlignedAllocAlignment(123, 456),
+ "Scudo ERROR.*123.*456");
}
TEST(ScudoReportDeathTest, HeaderCorruption) {
UNUSED void *P = reinterpret_cast<void *>(0x42424242U);
UNUSED scudo::Chunk::PackedHeader Header = {};
- EXPECT_DEATH(scudo::reportHeaderCorruption(&Header, P),
- "Scudo ERROR.*corrupted chunk header at address 0x.*42424242: "
- "chunk header is zero and might indicate memory "
- "corruption or a double free");
+ SCUDO_EXPECT_DEATH(
+ scudo::reportHeaderCorruption(&Header, P),
+ "Scudo ERROR.*corrupted chunk header at address 0x.*42424242: "
+ "chunk header is zero and might indicate memory "
+ "corruption or a double free");
Header = 10U;
- EXPECT_DEATH(scudo::reportHeaderCorruption(&Header, P),
- "Scudo ERROR.*corrupted chunk header at address 0x.*42424242: "
- "most likely due to memory corruption");
+ SCUDO_EXPECT_DEATH(
+ scudo::reportHeaderCorruption(&Header, P),
+ "Scudo ERROR.*corrupted chunk header at address 0x.*42424242: "
+ "most likely due to memory corruption");
}
#if SCUDO_LINUX || SCUDO_TRUSTY || SCUDO_ANDROID
@@ -99,22 +107,24 @@ TEST(ScudoReportDeathTest, HeaderCorruption) {
TEST(ScudoReportDeathTest, Linux) {
errno = ENOMEM;
- EXPECT_DEATH(scudo::reportMapError(),
- "Scudo ERROR:.*internal map failure \\(error desc=.*\\)");
+ SCUDO_EXPECT_DEATH(scudo::reportMapError(),
+ "Scudo ERROR:.*internal map failure \\(error desc=.*\\)");
errno = ENOMEM;
- EXPECT_DEATH(scudo::reportMapError(1024U),
- "Scudo ERROR:.*internal map failure \\(error desc=.*\\) "
- "requesting 1KB");
- EXPECT_DEATH(scudo::reportMapFixedError(0x1000U, 0x2000U),
- "Scudo ERROR:.*internal map failure using fixed address "
- "\\(expected: 0x1000 requested: 0x2000\\)");
+ SCUDO_EXPECT_DEATH(scudo::reportMapError(1024U),
+ "Scudo ERROR:.*internal map failure \\(error desc=.*\\) "
+ "requesting 1KB");
+ SCUDO_EXPECT_DEATH(scudo::reportMapFixedError(0x1000U, 0x2000U),
+ "Scudo ERROR:.*internal map failure using fixed address "
+ "\\(expected: 0x1000 requested: 0x2000\\)");
errno = ENOMEM;
- EXPECT_DEATH(scudo::reportUnmapError(0x1000U, 100U),
- "Scudo ERROR:.*internal unmap failure \\(error desc=.*\\) Addr "
- "0x1000 Size 100");
+ SCUDO_EXPECT_DEATH(
+ scudo::reportUnmapError(0x1000U, 100U),
+ "Scudo ERROR:.*internal unmap failure \\(error desc=.*\\) Addr "
+ "0x1000 Size 100");
errno = ENOMEM;
- EXPECT_DEATH(scudo::reportProtectError(0x1000U, 100U, PROT_READ),
- "Scudo ERROR:.*internal protect failure \\(error desc=.*\\) "
- "Addr 0x1000 Size 100 Prot 1");
+ SCUDO_EXPECT_DEATH(
+ scudo::reportProtectError(0x1000U, 100U, PROT_READ),
+ "Scudo ERROR:.*internal protect failure \\(error desc=.*\\) "
+ "Addr 0x1000 Size 100 Prot 1");
}
#endif
diff --git a/compiler-rt/lib/scudo/standalone/tests/scudo_unit_test.h b/compiler-rt/lib/scudo/standalone/tests/scudo_unit_test.h
index 27c0e591a2099..0bb2ac47bc1d4 100644
--- a/compiler-rt/lib/scudo/standalone/tests/scudo_unit_test.h
+++ b/compiler-rt/lib/scudo/standalone/tests/scudo_unit_test.h
@@ -58,4 +58,33 @@ using Test = ::testing::Test;
#define SCUDO_NO_TEST_MAIN
#endif
+#if SCUDO_ANDROID
+static void DisableDebuggerdMaybe() {
+ // Disable the debuggerd signal handler on Android, without this we can end
+ // up spending a significant amount of time creating tombstones.
+ signal(SIGSEGV, SIG_DFL);
+ signal(SIGABRT, SIG_DFL);
+}
+
+#define SCUDO_EXPECT_DEATH(X, Y) \
+ EXPECT_DEATH( \
+ { \
+ DisableDebuggerdMaybe(); \
+ X; \
+ }, \
+ Y);
+#define SCUDO_ASSERT_DEATH(X, Y) \
+ ASSERT_DEATH( \
+ { \
+ DisableDebuggerdMaybe(); \
+ X; \
+ }, \
+ Y);
+#else
+
+#define SCUDO_EXPECT_DEATH(X, Y) EXPECT_DEATH(X, Y);
+#define SCUDO_ASSERT_DEATH(X, Y) ASSERT_DEATH(X, Y);
+
+#endif
+
extern bool UseQuarantine;
diff --git a/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp
index 8741c8299b57c..e6ceb2d8b9317 100644
--- a/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp
@@ -145,7 +145,7 @@ template <typename Config> static void testBasic() {
// If the Secondary can't cache that pointer, it will be unmapped.
if (!Info.Allocator->canCache(Size)) {
- EXPECT_DEATH(
+ SCUDO_EXPECT_DEATH(
{
// Repeat few time to avoid missing crash if it's mmaped by unrelated
// code.
diff --git a/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp b/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp
index 12aaf8219dc2b..44654290424a5 100644
--- a/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp
@@ -188,7 +188,7 @@ TEST_F(ScudoWrappersCDeathTest, Malloc) {
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wfree-nonheap-object"
#endif
- EXPECT_DEATH(
+ SCUDO_EXPECT_DEATH(
free(reinterpret_cast<void *>(reinterpret_cast<uintptr_t>(P) | 1U)), "");
#if defined(__has_warning) && __has_warning("-Wfree-nonheap-object")
#pragma GCC diagnostic pop
@@ -202,7 +202,7 @@ TEST_F(ScudoWrappersCDeathTest, Malloc) {
// allocations before creating a new process. There is a possibility
// that the previously freed P is reused, therefore, in the new
// process doing free(P) is not a double free.
- EXPECT_DEATH(
+ SCUDO_EXPECT_DEATH(
{
// Note: volatile here prevents the calls from being optimized out.
void *volatile Ptr = malloc(Size);
@@ -374,7 +374,7 @@ TEST_F(ScudoWrappersCDeathTest, Realloc) {
verifyReallocHookPtrs(OldP, P, Size / 2U);
free(P);
- EXPECT_DEATH(P = realloc(P, Size), "");
+ SCUDO_EXPECT_DEATH(P = realloc(P, Size), "");
errno = 0;
EXPECT_EQ(realloc(nullptr, SIZE_MAX), nullptr);
@@ -425,8 +425,8 @@ TEST_F(ScudoWrappersCTest, MallocFreeSized) {
verifyAllocHookSize(Size);
if (VERIFY_FREE_SIZED_DEATH) {
- EXPECT_DEATH(free_sized(P, Size - 1), "");
- EXPECT_DEATH(free_sized(P, Size + 1), "");
+ SCUDO_EXPECT_DEATH(free_sized(P, Size - 1), "");
+ SCUDO_EXPECT_DEATH(free_sized(P, Size + 1), "");
// An update to this warning in Clang now triggers in this line, but it's ok
// because the check is expecting a bad pointer and should fail.
@@ -434,10 +434,10 @@ TEST_F(ScudoWrappersCTest, MallocFreeSized) {
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wfree-nonheap-object"
#endif
- EXPECT_DEATH(free_sized(reinterpret_cast<void *>(
- reinterpret_cast<uintptr_t>(P) | 1U),
- Size),
- "");
+ SCUDO_EXPECT_DEATH(free_sized(reinterpret_cast<void *>(
+ reinterpret_cast<uintptr_t>(P) | 1U),
+ Size),
+ "");
#if defined(__has_warning) && __has_warning("-Wfree-nonheap-object")
#pragma GCC diagnostic pop
#endif
@@ -447,7 +447,7 @@ TEST_F(ScudoWrappersCTest, MallocFreeSized) {
verifyDeallocHookPtr(P);
if (VERIFY_FREE_SIZED_DEATH)
- EXPECT_DEATH(free_sized(P, Size), "");
+ SCUDO_EXPECT_DEATH(free_sized(P, Size), "");
}
TEST_F(ScudoWrappersCTest, AlignedAllocFreeAlignedSized) {
@@ -461,9 +461,9 @@ TEST_F(ScudoWrappersCTest, AlignedAllocFreeAlignedSized) {
verifyAllocHookSize(Size);
if (VERIFY_FREE_ALIGNED_SIZED_DEATH) {
- EXPECT_DEATH(free_aligned_sized(P, Alignment, Size - 1), "");
- EXPECT_DEATH(free_aligned_sized(P, Alignment, Size + 1), "");
- EXPECT_DEATH(
+ SCUDO_EXPECT_DEATH(free_aligned_sized(P, Alignment, Size - 1), "");
+ SCUDO_EXPECT_DEATH(free_aligned_sized(P, Alignment, Size + 1), "");
+ SCUDO_EXPECT_DEATH(
free_aligned_sized(P, size_t{1} << (sizeof(size_t) * 8 - 1), Size), "");
// An update to this warning in Clang now triggers in this line, but it's ok
@@ -472,10 +472,11 @@ TEST_F(ScudoWrappersCTest, AlignedAllocFreeAlignedSized) {
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wfree-nonheap-object"
#endif
- EXPECT_DEATH(free_aligned_sized(reinterpret_cast<void *>(
- reinterpret_cast<uintptr_t>(P) | 1U),
- Alignment, Size),
- "");
+ SCUDO_EXPECT_DEATH(
+ free_aligned_sized(
+ reinterpret_cast<void *>(reinterpret_cast<uintptr_t>(P) | 1U),
+ Alignment, Size),
+ "");
#if defined(__has_warning) && __has_warning("-Wfree-nonheap-object")
#pragma GCC diagnostic pop
#endif
@@ -483,7 +484,7 @@ TEST_F(ScudoWrappersCTest, AlignedAllocFreeAlignedSized) {
free_aligned_sized(P, Alignment, Size);
verifyDeallocHookPtr(P);
- EXPECT_DEATH(free_aligned_sized(P, Alignment, Size), "");
+ SCUDO_EXPECT_DEATH(free_aligned_sized(P, Alignment, Size), "");
}
TEST_F(ScudoWrappersCDeathTest, MallocFreeAlignedSized) {
@@ -495,8 +496,9 @@ TEST_F(ScudoWrappersCDeathTest, MallocFreeAlignedSized) {
verifyAllocHookSize(Size);
if (VERIFY_FREE_ALIGNED_SIZED_DEATH) {
- EXPECT_DEATH(free_aligned_sized(P, 8, Size), "");
- EXPECT_DEATH(free_aligned_sized(P, alignof(std::max_align_t), Size), "");
+ SCUDO_EXPECT_DEATH(free_aligned_sized(P, 8, Size), "");
+ SCUDO_EXPECT_DEATH(free_aligned_sized(P, alignof(std::max_align_t), Size),
+ "");
}
free_sized(P, Size);
@@ -514,7 +516,7 @@ TEST_F(ScudoWrappersCDeathTest, AlignedAllocFreeSized) {
verifyAllocHookSize(Size);
if (VERIFY_FREE_SIZED_DEATH)
- EXPECT_DEATH(free_sized(P, Size), "");
+ SCUDO_EXPECT_DEATH(free_sized(P, Size), "");
free_aligned_sized(P, Alignment, Size);
verifyDeallocHookPtr(P);
@@ -532,7 +534,7 @@ TEST_F(ScudoWrappersCDeathTest, PosixMemalignFreeSized) {
verifyAllocHookSize(Size);
if (VERIFY_FREE_SIZED_DEATH)
- EXPECT_DEATH(free_sized(P, Size), "");
+ SCUDO_EXPECT_DEATH(free_sized(P, Size), "");
free(P);
verifyDeallocHookPtr(P);
@@ -549,7 +551,7 @@ TEST_F(ScudoWrappersCDeathTest, MemalignFreeSized) {
verifyAllocHookSize(Size);
if (VERIFY_FREE_SIZED_DEATH)
- EXPECT_DEATH(free_sized(P, Size), "");
+ SCUDO_EXPECT_DEATH(free_sized(P, Size), "");
free(P);
verifyDeallocHookPtr(P);
@@ -565,7 +567,7 @@ TEST_F(ScudoWrappersCDeathTest, PvallocFreeSized) {
verifyAllocHookSize(Size);
if (VERIFY_FREE_SIZED_DEATH)
- EXPECT_DEATH(free_sized(P, Size), "");
+ SCUDO_EXPECT_DEATH(free_sized(P, Size), "");
free(P);
verifyDeallocHookPtr(P);
@@ -582,7 +584,7 @@ TEST_F(ScudoWrappersCDeathTest, VallocFreeSized) {
verifyAllocHookSize(Size);
if (VERIFY_FREE_SIZED_DEATH)
- EXPECT_DEATH(free_sized(P, Size), "");
+ SCUDO_EXPECT_DEATH(free_sized(P, Size), "");
free(P);
verifyDeallocHookPtr(P);
@@ -756,7 +758,7 @@ TEST_F(ScudoWrappersCTest, MallocIterateBoundary) {
#if !SCUDO_FUCHSIA
TEST_F(ScudoWrappersCDeathTest, MallocDisableDeadlock) {
// We expect heap operations within a disable/enable scope to deadlock.
- EXPECT_DEATH(
+ SCUDO_EXPECT_DEATH(
{
void *P = malloc(Size);
EXPECT_NE(P, nullptr);
@@ -812,7 +814,7 @@ TEST_F(ScudoWrappersCDeathTest, Fork) {
free(P);
// fork should stall if the allocator has been disabled.
- EXPECT_DEATH(
+ SCUDO_EXPECT_DEATH(
{
malloc_disable();
alarm(1);
diff --git a/compiler-rt/lib/scudo/standalone/tests/wrappers_cpp_test.cpp b/compiler-rt/lib/scudo/standalone/tests/wrappers_cpp_test.cpp
index 5e56447ba0e3b..88ab061e229ab 100644
--- a/compiler-rt/lib/scudo/standalone/tests/wrappers_cpp_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/wrappers_cpp_test.cpp
@@ -84,10 +84,10 @@ class ScudoWrappersCppTest : public Test {
verifyAllocHookPtr(P);
verifyAllocHookSize(sizeof(T));
memset(P, 0x42, sizeof(T));
- EXPECT_DEATH(delete[] P, "");
+ SCUDO_EXPECT_DEATH(delete[] P, "");
delete P;
verifyDeallocHookPtr(P);
- EXPECT_DEATH(delete P, "");
+ SCUDO_EXPECT_DEATH(delete P, "");
P = new T;
EXPECT_NE(P, nullptr);
@@ -109,10 +109,10 @@ class ScudoWrappersCppTest : public Test {
verifyAllocHookPtr(A);
verifyAllocHookSize(sizeof(T) * N);
memset(A, 0x42, sizeof(T) * N);
- EXPECT_DEATH(delete A, "");
+ SCUDO_EXPECT_DEATH(delete A, "");
delete[] A;
verifyDeallocHookPtr(A);
- EXPECT_DEATH(delete[] A, "");
+ SCUDO_EXPECT_DEATH(delete[] A, "");
A = new T[N];
EXPECT_NE(A, nullptr);
>From 69ee64790ed69555264a200a3ae95d644a649e42 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Mon, 6 Jul 2026 19:56:53 +0200
Subject: [PATCH 30/46] InstCombine: Add baseline test for implied frexp
exponent ranges (#206926)
---
...ed-exponent-range-dominating-conditions.ll | 1277 +++++++++++++++++
1 file changed, 1277 insertions(+)
create mode 100644 llvm/test/Transforms/InstCombine/frexp-implied-exponent-range-dominating-conditions.ll
diff --git a/llvm/test/Transforms/InstCombine/frexp-implied-exponent-range-dominating-conditions.ll b/llvm/test/Transforms/InstCombine/frexp-implied-exponent-range-dominating-conditions.ll
new file mode 100644
index 0000000000000..505864cfcf35e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/frexp-implied-exponent-range-dominating-conditions.ll
@@ -0,0 +1,1277 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=instcombine < %s | FileCheck %s
+
+; Test that nofpclass(inf) can be propagated based on the contextual
+; knowledge that the exponent of a frexp must be negative
+;
+; We can infer from a not-inf source, plus the compare that the clamp
+; of infinity inside small can be deleted.
+
+define float @frexp_fcmp_ogt_1(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_ogt_1(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_LARGE:%.*]] = fcmp ogt float [[X_FABS]], 1.000000e+00
+; CHECK-NEXT: br i1 [[IS_LARGE]], label %[[LARGE:.*]], label %[[SMALL:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.large = fcmp ogt float %x.fabs, 1.0
+ br i1 %is.large, label %large, label %small
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+define float @frexp_fcmp_oge_1(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_oge_1(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_LARGE:%.*]] = fcmp ult float [[X_FABS]], 1.000000e+00
+; CHECK-NEXT: br i1 [[IS_LARGE]], label %[[SMALL:.*]], label %[[LARGE:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.large = fcmp oge float %x.fabs, 1.0
+ br i1 %is.large, label %large, label %small
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+define float @frexp_fcmp_ugt_1(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_ugt_1(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_LARGE:%.*]] = fcmp ugt float [[X_FABS]], 1.000000e+00
+; CHECK-NEXT: br i1 [[IS_LARGE]], label %[[LARGE:.*]], label %[[SMALL:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.large = fcmp ugt float %x.fabs, 1.0
+ br i1 %is.large, label %large, label %small
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+define float @frexp_fcmp_uge_1(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_uge_1(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_LARGE:%.*]] = fcmp uge float [[X_FABS]], 1.000000e+00
+; CHECK-NEXT: br i1 [[IS_LARGE]], label %[[LARGE:.*]], label %[[SMALL:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.large = fcmp uge float %x.fabs, 1.0
+ br i1 %is.large, label %large, label %small
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+define float @frexp_fcmp_olt_1(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_olt_1(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_SMALL:%.*]] = fcmp olt float [[X_FABS]], 1.000000e+00
+; CHECK-NEXT: br i1 [[IS_SMALL]], label %[[SMALL:.*]], label %[[LARGE:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.small = fcmp olt float %x.fabs, 1.0
+ br i1 %is.small, label %small, label %large
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+define float @frexp_fcmp_ole_1(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_ole_1(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_SMALL:%.*]] = fcmp ugt float [[X_FABS]], 1.000000e+00
+; CHECK-NEXT: br i1 [[IS_SMALL]], label %[[LARGE:.*]], label %[[SMALL:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.small = fcmp ole float %x.fabs, 1.0
+ br i1 %is.small, label %small, label %large
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+
+define float @frexp_fcmp_ult_1(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_ult_1(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_SMALL:%.*]] = fcmp ult float [[X_FABS]], 1.000000e+00
+; CHECK-NEXT: br i1 [[IS_SMALL]], label %[[SMALL:.*]], label %[[LARGE:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.small = fcmp ult float %x.fabs, 1.0
+ br i1 %is.small, label %small, label %large
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+define float @frexp_fcmp_ule_1(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_ule_1(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_SMALL:%.*]] = fcmp ule float [[X_FABS]], 1.000000e+00
+; CHECK-NEXT: br i1 [[IS_SMALL]], label %[[SMALL:.*]], label %[[LARGE:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.small = fcmp ule float %x.fabs, 1.0
+ br i1 %is.small, label %small, label %large
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+define float @frexp_cmp_ugt_0.5(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_cmp_ugt_0.5(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_LARGE:%.*]] = fcmp ugt float [[X_FABS]], 5.000000e-01
+; CHECK-NEXT: br i1 [[IS_LARGE]], label %[[LARGE:.*]], label %[[SMALL:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.large = fcmp ugt float %x.fabs, 0.5
+ br i1 %is.large, label %large, label %small
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+define float @frexp_cmp_uge_0.5(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_cmp_uge_0.5(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_LARGE:%.*]] = fcmp uge float [[X_FABS]], 5.000000e-01
+; CHECK-NEXT: br i1 [[IS_LARGE]], label %[[LARGE:.*]], label %[[SMALL:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.large = fcmp uge float %x.fabs, 0.5
+ br i1 %is.large, label %large, label %small
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+define float @frexp_fcmp_ult_0.5(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_ult_0.5(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_SMALL:%.*]] = fcmp ult float [[X_FABS]], 5.000000e-01
+; CHECK-NEXT: br i1 [[IS_SMALL]], label %[[SMALL:.*]], label %[[LARGE:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.small = fcmp ult float %x.fabs, 0.5
+ br i1 %is.small, label %small, label %large
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+define float @frexp_fcmp_ule_0.5(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_ule_0.5(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_SMALL:%.*]] = fcmp ule float [[X_FABS]], 5.000000e-01
+; CHECK-NEXT: br i1 [[IS_SMALL]], label %[[SMALL:.*]], label %[[LARGE:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.small = fcmp ule float %x.fabs, 0.5
+ br i1 %is.small, label %small, label %large
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+define float @frexp_fcmp_ugt_0.5(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_ugt_0.5(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_LARGE:%.*]] = fcmp ugt float [[X_FABS]], 5.000000e-01
+; CHECK-NEXT: br i1 [[IS_LARGE]], label %[[LARGE:.*]], label %[[SMALL:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.large = fcmp ugt float %x.fabs, 0.5
+ br i1 %is.large, label %large, label %small
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+define float @frexp_fcmp_uge_0.5(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_uge_0.5(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_LARGE:%.*]] = fcmp uge float [[X_FABS]], 5.000000e-01
+; CHECK-NEXT: br i1 [[IS_LARGE]], label %[[LARGE:.*]], label %[[SMALL:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.large = fcmp uge float %x.fabs, 0.5
+ br i1 %is.large, label %large, label %small
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+define float @frexp_fcmp_oeq_1(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_oeq_1(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_ONE:%.*]] = fcmp oeq float [[X_FABS]], 1.000000e+00
+; CHECK-NEXT: br i1 [[IS_ONE]], label %[[SMALL:.*]], label %[[LARGE:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.one = fcmp oeq float %x.fabs, 1.0
+ br i1 %is.one, label %small, label %large
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+define float @frexp_fcmp_ueq_1(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_ueq_1(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_ONE:%.*]] = fcmp ueq float [[X_FABS]], 1.000000e+00
+; CHECK-NEXT: br i1 [[IS_ONE]], label %[[SMALL:.*]], label %[[LARGE:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.one = fcmp ueq float %x.fabs, 1.0
+ br i1 %is.one, label %small, label %large
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+define float @frexp_fcmp_one_1(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_one_1(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_NOT_ONE:%.*]] = fcmp ueq float [[X_FABS]], 1.000000e+00
+; CHECK-NEXT: br i1 [[IS_NOT_ONE]], label %[[SMALL:.*]], label %[[LARGE:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.not.one = fcmp one float %x.fabs, 1.0
+ br i1 %is.not.one, label %large, label %small
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+define float @frexp_fcmp_une_1(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_une_1(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_NOT_ONE:%.*]] = fcmp une float [[X_FABS]], 1.000000e+00
+; CHECK-NEXT: br i1 [[IS_NOT_ONE]], label %[[LARGE:.*]], label %[[SMALL:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.not.one = fcmp une float %x.fabs, 1.0
+ br i1 %is.not.one, label %large, label %small
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+define float @frexp_fcmp_oeq_0.5(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_oeq_0.5(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_ONE:%.*]] = fcmp oeq float [[X_FABS]], 5.000000e-01
+; CHECK-NEXT: br i1 [[IS_ONE]], label %[[SMALL:.*]], label %[[LARGE:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.one = fcmp oeq float %x.fabs, 0.5
+ br i1 %is.one, label %small, label %large
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+define float @frexp_fcmp_ueq_0.5(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_ueq_0.5(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_ONE:%.*]] = fcmp ueq float [[X_FABS]], 5.000000e-01
+; CHECK-NEXT: br i1 [[IS_ONE]], label %[[SMALL:.*]], label %[[LARGE:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.one = fcmp ueq float %x.fabs, 0.5
+ br i1 %is.one, label %small, label %large
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+define float @frexp_fcmp_one_0.5(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_one_0.5(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_NOT_ONE:%.*]] = fcmp ueq float [[X_FABS]], 5.000000e-01
+; CHECK-NEXT: br i1 [[IS_NOT_ONE]], label %[[SMALL:.*]], label %[[LARGE:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.not.one = fcmp one float %x.fabs, 0.5
+ br i1 %is.not.one, label %large, label %small
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+define float @frexp_fcmp_une_0.5(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_une_0.5(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_NOT_ONE:%.*]] = fcmp une float [[X_FABS]], 5.000000e-01
+; CHECK-NEXT: br i1 [[IS_NOT_ONE]], label %[[LARGE:.*]], label %[[SMALL:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.not.one = fcmp une float %x.fabs, 0.5
+ br i1 %is.not.one, label %large, label %small
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+define float @frexp_fcmp_true_1(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_true_1(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: br i1 true, label %[[SMALL:.*]], label %[[LARGE:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.true = fcmp true float %x.fabs, 1.0
+ br i1 %is.true, label %small, label %large
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+define float @frexp_fcmp_false_1(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_false_1(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: br i1 false, label %[[SMALL:.*]], label %[[LARGE:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: ret float poison
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.false = fcmp false float %x.fabs, 1.0
+ br i1 %is.false, label %small, label %large
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+define float @frexp_fcmp_ord_1(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_ord_1(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: br i1 true, label %[[SMALL:.*]], label %[[LARGE:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.ord = fcmp ord float %x.fabs, 1.0
+ br i1 %is.ord, label %small, label %large
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+define float @frexp_fcmp_uno_1(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_uno_1(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: br i1 false, label %[[SMALL:.*]], label %[[LARGE:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: ret float poison
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.uno = fcmp uno float %x.fabs, 1.0
+ br i1 %is.uno, label %small, label %large
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+; Negative test
+define float @frexp_fcmp_ogt_nextafter1(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_ogt_nextafter1(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_LARGE:%.*]] = fcmp ogt float [[X_FABS]], f0x3F800001
+; CHECK-NEXT: br i1 [[IS_LARGE]], label %[[LARGE:.*]], label %[[SMALL:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.large = fcmp ogt float %x.fabs, f0x3f800001
+ br i1 %is.large, label %large, label %small
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+; Negative test
+define float @frexp_fcmp_ole_nextafter1(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_ole_nextafter1(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_SMALL:%.*]] = fcmp ugt float [[X_FABS]], f0x3F800001
+; CHECK-NEXT: br i1 [[IS_SMALL]], label %[[LARGE:.*]], label %[[SMALL:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.small = fcmp ole float %x.fabs, f0x3f800001
+ br i1 %is.small, label %small, label %large
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+; Negative test
+define float @frexp_fcmp_ogt_nextdown1(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_ogt_nextdown1(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_LARGE:%.*]] = fcmp ogt float [[X_FABS]], f0x3F7FFFFF
+; CHECK-NEXT: br i1 [[IS_LARGE]], label %[[LARGE:.*]], label %[[SMALL:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.large = fcmp ogt float %x.fabs, f0x3f7fffff
+ br i1 %is.large, label %large, label %small
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+; Negative test
+define float @frexp_fcmp_ole_nextdown1(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_ole_nextdown1(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_SMALL:%.*]] = fcmp ugt float [[X_FABS]], f0x3F7FFFFF
+; CHECK-NEXT: br i1 [[IS_SMALL]], label %[[LARGE:.*]], label %[[SMALL:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.small = fcmp ole float %x.fabs, f0x3f7fffff
+ br i1 %is.small, label %small, label %large
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+; Negative test
+define float @frexp_fcmp_ole_inf(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_ole_inf(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: br i1 true, label %[[SMALL:.*]], label %[[LARGE:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.small = fcmp ole float %x.fabs, +inf
+ br i1 %is.small, label %small, label %large
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+; Negative test
+define float @frexp_fcmp_ole_nan(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_ole_nan(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: br i1 false, label %[[SMALL:.*]], label %[[LARGE:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: ret float poison
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.small = fcmp ole float %x.fabs, +qnan
+ br i1 %is.small, label %small, label %large
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+; Negative test
+define float @frexp_fcmp_olt_1_not_dominated(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf, i1 %arg) {
+; CHECK-LABEL: define float @frexp_fcmp_olt_1_not_dominated(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]], i1 [[ARG:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br i1 [[ARG]], label %[[GUARD:.*]], label %[[SMALL:.*]]
+; CHECK: [[GUARD]]:
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_SMALL:%.*]] = fcmp olt float [[X_FABS]], 1.000000e+00
+; CHECK-NEXT: br i1 [[IS_SMALL]], label %[[SMALL]], label %[[EXIT:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+entry:
+ br i1 %arg, label %guard, label %small
+
+guard:
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.small = fcmp olt float %x.fabs, 1.0
+ br i1 %is.small, label %small, label %exit
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+exit:
+ ret float 0.0
+}
+
+; Make sure the inner compare against 1 is recognized and doesn't
+; abort the search after seeing the outer compare.
+define float @frexp_fcmp_olt_1_multiple_conditions_wrong_constant(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_olt_1_multiple_conditions_wrong_constant(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_SMALL:%.*]] = fcmp olt float [[X_FABS]], 2.000000e+00
+; CHECK-NEXT: br i1 [[IS_SMALL]], label %[[SMALL:.*]], label %[[EXIT:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[OTHER_COND:%.*]] = fcmp olt float [[X_FABS]], 1.000000e+00
+; CHECK-NEXT: br i1 [[OTHER_COND]], label %[[BODY:.*]], label %[[EXIT]]
+; CHECK: [[BODY]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.small = fcmp olt float %x.fabs, 2.0
+ br i1 %is.small, label %small, label %exit
+
+small:
+ %other.cond = fcmp olt float %x.fabs, 1.0
+ br i1 %other.cond, label %body, label %exit
+
+body:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+exit:
+ ret float 0.0
+}
+
+; Make sure the inner compare against 1 is recognized and doesn't
+; abort the search after seeing the outer compare.
+define float @frexp_fcmp_olt_1_multiple_conditions_nonfinite_constant(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_olt_1_multiple_conditions_nonfinite_constant(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: br i1 true, label %[[SMALL:.*]], label %[[EXIT:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[OTHER_COND:%.*]] = fcmp olt float [[X_FABS]], 1.000000e+00
+; CHECK-NEXT: br i1 [[OTHER_COND]], label %[[BODY:.*]], label %[[EXIT]]
+; CHECK: [[BODY]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.small = fcmp olt float %x.fabs, +inf
+ br i1 %is.small, label %small, label %exit
+
+small:
+ %other.cond = fcmp olt float %x.fabs, 1.0
+ br i1 %other.cond, label %body, label %exit
+
+body:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+exit:
+ ret float 0.0
+}
+
+define float @frexp_fcmp_olt_1_multiple_conditions_edge_case_compare(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_olt_1_multiple_conditions_edge_case_compare(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: br i1 true, label %[[SMALL:.*]], label %[[EXIT:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[OTHER_COND:%.*]] = fcmp olt float [[X_FABS]], 1.000000e+00
+; CHECK-NEXT: br i1 [[OTHER_COND]], label %[[BODY:.*]], label %[[EXIT]]
+; CHECK: [[BODY]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.small = fcmp ord float %x.fabs, 0.0
+ br i1 %is.small, label %small, label %exit
+
+small:
+ %other.cond = fcmp olt float %x.fabs, 1.0
+ br i1 %other.cond, label %body, label %exit
+
+body:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+exit:
+ ret float 0.0
+}
+
+
+define float @frexp_fcmp_olt_1_multiple_conditions_other(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_olt_1_multiple_conditions_other(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_SMALL:%.*]] = fcmp olt float [[X_FABS]], 1.000000e+00
+; CHECK-NEXT: br i1 [[IS_SMALL]], label %[[SMALL:.*]], label %[[EXIT:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[OTHER_COND:%.*]] = fcmp ueq float [[X]], 5.000000e-01
+; CHECK-NEXT: br i1 [[OTHER_COND]], label %[[EXIT]], label %[[BODY:.*]]
+; CHECK: [[BODY]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.small = fcmp olt float %x.fabs, 1.0
+ br i1 %is.small, label %small, label %exit
+
+small:
+ %other.cond = fcmp one float %x, 0.5
+ br i1 %other.cond, label %body, label %exit
+
+body:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+exit:
+ ret float 0.0
+}
+
+; Test no assertion on exponent type > 64
+define float @frexp_fcmp_ogt_1_large_int(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_ogt_1_large_int(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_LARGE:%.*]] = fcmp ogt float [[X_FABS]], 1.000000e+00
+; CHECK-NEXT: br i1 [[IS_LARGE]], label %[[LARGE:.*]], label %[[SMALL:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i128 } @llvm.frexp.f32.i128(float [[X]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i128 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i128(float [[NOT_INF]], i128 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.large = fcmp ogt float %x.fabs, 1.0
+ br i1 %is.large, label %large, label %small
+
+small:
+ %frexp = call { float, i128 } @llvm.frexp.f32.i128(float %x)
+ %exp = extractvalue { float, i128 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i128(float %not.inf, i128 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
+
+; Negative test
+define float @frexp_fcmp_ogt_1_fabs_wrong_value(float nofpclass(nan inf) %x, float nofpclass(inf) %not.inf) {
+; CHECK-LABEL: define float @frexp_fcmp_ogt_1_fabs_wrong_value(
+; CHECK-SAME: float nofpclass(nan inf) [[X:%.*]], float nofpclass(inf) [[NOT_INF:%.*]]) {
+; CHECK-NEXT: [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT: [[IS_LARGE:%.*]] = fcmp ogt float [[X_FABS]], 1.000000e+00
+; CHECK-NEXT: br i1 [[IS_LARGE]], label %[[LARGE:.*]], label %[[SMALL:.*]]
+; CHECK: [[SMALL]]:
+; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X_FABS]])
+; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
+; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
+; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
+; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
+; CHECK-NEXT: ret float [[SELECT]]
+; CHECK: [[LARGE]]:
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %x.fabs = call float @llvm.fabs.f32(float %x)
+ %is.large = fcmp ogt float %x.fabs, 1.0
+ br i1 %is.large, label %large, label %small
+
+small:
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x.fabs)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %scaled = call float @llvm.ldexp.f32.i32(float %not.inf, i32 %exp)
+ %is.inf = fcmp oeq float %scaled, +inf
+ %select = select i1 %is.inf, float 0.0, float %scaled
+ ret float %select
+
+large:
+ ret float 0.0
+}
>From d1eae28e79d33a4f85bf16a09cbb90aa9b6848e4 Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamleshbhalui at gmail.com>
Date: Mon, 6 Jul 2026 18:58:27 +0100
Subject: [PATCH 31/46] [InstSimplify] fold the identity interleave (#206646)
If all the extracvalue of deinterleave used in interleave in same order
then it is an identity.
---
llvm/lib/Analysis/InstructionSimplify.cpp | 31 +++
.../vector-interleave-deinterleave.ll | 211 ++++++++++++++++++
2 files changed, 242 insertions(+)
create mode 100644 llvm/test/Transforms/InstSimplify/vector-interleave-deinterleave.ll
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 1efc01a726c37..ddaa44f43e631 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -7290,6 +7290,34 @@ static Value *simplifyBinaryIntrinsic(Intrinsic::ID IID, Type *ReturnType,
return nullptr;
}
+/// interleaveN(extractvalue(deinterleaveN(x), 0), ...,
+/// extractvalue(deinterleaveN(x), N-1)) --> x
+static Value *simplifyIdentityInterleave(Intrinsic::ID IID,
+ ArrayRef<Value *> Args) {
+ unsigned Factor = getInterleaveIntrinsicFactor(IID);
+ if (!Factor || Factor != Args.size())
+ return nullptr;
+
+ Intrinsic::ID DeinterleaveID = Intrinsic::getDeinterleaveIntrinsicID(Factor);
+ IntrinsicInst *DI = nullptr;
+ for (unsigned Idx = 0; Idx != Factor; ++Idx) {
+ auto *EV = dyn_cast<ExtractValueInst>(Args[Idx]);
+ if (!EV || EV->getNumIndices() != 1 || *EV->idx_begin() != Idx)
+ return nullptr;
+
+ auto *CurDI = dyn_cast<IntrinsicInst>(EV->getAggregateOperand());
+ if (!CurDI || CurDI->getIntrinsicID() != DeinterleaveID)
+ return nullptr;
+
+ if (!DI)
+ DI = CurDI;
+ else if (DI != CurDI)
+ return nullptr;
+ }
+
+ return DI->getArgOperand(0);
+}
+
Value *llvm::simplifyIntrinsic(Intrinsic::ID IID, Type *ReturnType,
ArrayRef<Value *> Args, FastMathFlags FMF,
const SimplifyQuery &Q, Function *CxtF,
@@ -7317,6 +7345,9 @@ Value *llvm::simplifyIntrinsic(Intrinsic::ID IID, Type *ReturnType,
}
}
+ if (Value *V = simplifyIdentityInterleave(IID, Args))
+ return V;
+
if (NumOperands == 1)
return simplifyUnaryIntrinsic(IID, Args[0], FMF, Q);
diff --git a/llvm/test/Transforms/InstSimplify/vector-interleave-deinterleave.ll b/llvm/test/Transforms/InstSimplify/vector-interleave-deinterleave.ll
new file mode 100644
index 0000000000000..6e3b8beb1e52f
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/vector-interleave-deinterleave.ll
@@ -0,0 +1,211 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=instsimplify -S | FileCheck %s
+
+define <vscale x 8 x i32> @interleave_of_deinterleave_f2(<vscale x 8 x i32> %x) {
+; CHECK-LABEL: @interleave_of_deinterleave_f2(
+; CHECK-NEXT: ret <vscale x 8 x i32> [[X:%.*]]
+;
+ %d = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %x)
+ %e0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 0
+ %e1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 1
+ %r = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> %e0, <vscale x 4 x i32> %e1)
+ ret <vscale x 8 x i32> %r
+}
+
+define <vscale x 16 x i32> @interleave_of_deinterleave_f4(<vscale x 16 x i32> %x) {
+; CHECK-LABEL: @interleave_of_deinterleave_f4(
+; CHECK-NEXT: ret <vscale x 16 x i32> [[X:%.*]]
+;
+ %d = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> %x)
+ %e0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 0
+ %e1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 1
+ %e2 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 2
+ %e3 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 3
+ %r = call <vscale x 16 x i32> @llvm.vector.interleave4.nxv16i32(<vscale x 4 x i32> %e0, <vscale x 4 x i32> %e1, <vscale x 4 x i32> %e2, <vscale x 4 x i32> %e3)
+ ret <vscale x 16 x i32> %r
+}
+
+define <vscale x 20 x i32> @interleave_of_deinterleave_f5(<vscale x 20 x i32> %x) {
+; CHECK-LABEL: @interleave_of_deinterleave_f5(
+; CHECK-NEXT: ret <vscale x 20 x i32> [[X:%.*]]
+;
+ %d = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave5.nxv20i32(<vscale x 20 x i32> %x)
+ %e0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 0
+ %e1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 1
+ %e2 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 2
+ %e3 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 3
+ %e4 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 4
+ %r = call <vscale x 20 x i32> @llvm.vector.interleave5.nxv20i32(<vscale x 4 x i32> %e0, <vscale x 4 x i32> %e1, <vscale x 4 x i32> %e2, <vscale x 4 x i32> %e3, <vscale x 4 x i32> %e4)
+ ret <vscale x 20 x i32> %r
+}
+
+define void @interleave_of_deinterleave_load_store(ptr %src, ptr %dst) {
+; CHECK-LABEL: @interleave_of_deinterleave_load_store(
+; CHECK-NEXT: [[V:%.*]] = load <vscale x 8 x i32>, ptr [[SRC:%.*]], align 32
+; CHECK-NEXT: store <vscale x 8 x i32> [[V]], ptr [[DST:%.*]], align 32
+; CHECK-NEXT: ret void
+;
+ %v = load <vscale x 8 x i32>, ptr %src
+ %d = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %v)
+ %e0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 0
+ %e1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 1
+ %r = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> %e0, <vscale x 4 x i32> %e1)
+ store <vscale x 8 x i32> %r, ptr %dst
+ ret void
+}
+
+define <vscale x 8 x i32> @interleave_of_deinterleave_extra_uses(<vscale x 8 x i32> %x, ptr %p0, ptr %p1) {
+; CHECK-LABEL: @interleave_of_deinterleave_extra_uses(
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[X:%.*]])
+; CHECK-NEXT: [[E0:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[D]], 0
+; CHECK-NEXT: [[E1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[D]], 1
+; CHECK-NEXT: store <vscale x 4 x i32> [[E0]], ptr [[P0:%.*]], align 16
+; CHECK-NEXT: store <vscale x 4 x i32> [[E1]], ptr [[P1:%.*]], align 16
+; CHECK-NEXT: ret <vscale x 8 x i32> [[X]]
+;
+ %d = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %x)
+ %e0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 0
+ %e1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 1
+ store <vscale x 4 x i32> %e0, ptr %p0
+ store <vscale x 4 x i32> %e1, ptr %p1
+ %r = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> %e0, <vscale x 4 x i32> %e1)
+ ret <vscale x 8 x i32> %r
+}
+
+define <vscale x 8 x i32> @interleave_of_deinterleave_swapped(<vscale x 8 x i32> %x) {
+; CHECK-LABEL: @interleave_of_deinterleave_swapped(
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[X:%.*]])
+; CHECK-NEXT: [[E0:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[D]], 0
+; CHECK-NEXT: [[E1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[D]], 1
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[E1]], <vscale x 4 x i32> [[E0]])
+; CHECK-NEXT: ret <vscale x 8 x i32> [[R]]
+;
+ %d = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %x)
+ %e0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 0
+ %e1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 1
+ %r = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> %e1, <vscale x 4 x i32> %e0)
+ ret <vscale x 8 x i32> %r
+}
+
+define <vscale x 8 x i32> @interleave_of_two_deinterleaves(<vscale x 8 x i32> %x, <vscale x 8 x i32> %y) {
+; CHECK-LABEL: @interleave_of_two_deinterleaves(
+; CHECK-NEXT: [[DX:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[X:%.*]])
+; CHECK-NEXT: [[DY:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[Y:%.*]])
+; CHECK-NEXT: [[E0:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[DX]], 0
+; CHECK-NEXT: [[E1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[DY]], 1
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[E0]], <vscale x 4 x i32> [[E1]])
+; CHECK-NEXT: ret <vscale x 8 x i32> [[R]]
+;
+ %dx = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %x)
+ %dy = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %y)
+ %e0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %dx, 0
+ %e1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %dy, 1
+ %r = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> %e0, <vscale x 4 x i32> %e1)
+ ret <vscale x 8 x i32> %r
+}
+
+define <vscale x 8 x i32> @interleave_of_deinterleave_mismatched_factor(<vscale x 8 x i32> %x) {
+; CHECK-LABEL: @interleave_of_deinterleave_mismatched_factor(
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave4.nxv8i32(<vscale x 8 x i32> [[X:%.*]])
+; CHECK-NEXT: [[E0:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 0
+; CHECK-NEXT: [[E1:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 1
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> [[E0]], <vscale x 2 x i32> [[E1]])
+; CHECK-NEXT: [[W:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[R]], <vscale x 4 x i32> [[R]])
+; CHECK-NEXT: ret <vscale x 8 x i32> [[W]]
+;
+ %d = call { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave4.nxv8i32(<vscale x 8 x i32> %x)
+ %e0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 0
+ %e1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 1
+ %r = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> %e0, <vscale x 2 x i32> %e1)
+ %w = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> %r, <vscale x 4 x i32> %r)
+ ret <vscale x 8 x i32> %w
+}
+
+define <8 x i32> @interleave_of_deinterleave_fixed_f2(<8 x i32> %x) {
+; CHECK-LABEL: @interleave_of_deinterleave_fixed_f2(
+; CHECK-NEXT: ret <8 x i32> [[X:%.*]]
+;
+ %d = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> %x)
+ %e0 = extractvalue { <4 x i32>, <4 x i32> } %d, 0
+ %e1 = extractvalue { <4 x i32>, <4 x i32> } %d, 1
+ %r = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %e0, <4 x i32> %e1)
+ ret <8 x i32> %r
+}
+
+define <16 x i32> @interleave_of_deinterleave_fixed_f4(<16 x i32> %x) {
+; CHECK-LABEL: @interleave_of_deinterleave_fixed_f4(
+; CHECK-NEXT: ret <16 x i32> [[X:%.*]]
+;
+ %d = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave4.v16i32(<16 x i32> %x)
+ %e0 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %d, 0
+ %e1 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %d, 1
+ %e2 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %d, 2
+ %e3 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %d, 3
+ %r = call <16 x i32> @llvm.vector.interleave4.v16i32(<4 x i32> %e0, <4 x i32> %e1, <4 x i32> %e2, <4 x i32> %e3)
+ ret <16 x i32> %r
+}
+
+define void @interleave_of_deinterleave_fixed_load_store(ptr %src, ptr %dst) {
+; CHECK-LABEL: @interleave_of_deinterleave_fixed_load_store(
+; CHECK-NEXT: [[V:%.*]] = load <8 x i32>, ptr [[SRC:%.*]], align 32
+; CHECK-NEXT: store <8 x i32> [[V]], ptr [[DST:%.*]], align 32
+; CHECK-NEXT: ret void
+;
+ %v = load <8 x i32>, ptr %src
+ %d = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> %v)
+ %e0 = extractvalue { <4 x i32>, <4 x i32> } %d, 0
+ %e1 = extractvalue { <4 x i32>, <4 x i32> } %d, 1
+ %r = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %e0, <4 x i32> %e1)
+ store <8 x i32> %r, ptr %dst
+ ret void
+}
+
+define <8 x i32> @interleave_of_deinterleave_fixed_extra_uses(<8 x i32> %x, ptr %p0, ptr %p1) {
+; CHECK-LABEL: @interleave_of_deinterleave_fixed_extra_uses(
+; CHECK-NEXT: [[D:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[X:%.*]])
+; CHECK-NEXT: [[E0:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[D]], 0
+; CHECK-NEXT: [[E1:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[D]], 1
+; CHECK-NEXT: store <4 x i32> [[E0]], ptr [[P0:%.*]], align 16
+; CHECK-NEXT: store <4 x i32> [[E1]], ptr [[P1:%.*]], align 16
+; CHECK-NEXT: ret <8 x i32> [[X]]
+;
+ %d = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> %x)
+ %e0 = extractvalue { <4 x i32>, <4 x i32> } %d, 0
+ %e1 = extractvalue { <4 x i32>, <4 x i32> } %d, 1
+ store <4 x i32> %e0, ptr %p0
+ store <4 x i32> %e1, ptr %p1
+ %r = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %e0, <4 x i32> %e1)
+ ret <8 x i32> %r
+}
+
+define <8 x i32> @interleave_of_deinterleave_fixed_swapped(<8 x i32> %x) {
+; CHECK-LABEL: @interleave_of_deinterleave_fixed_swapped(
+; CHECK-NEXT: [[D:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[X:%.*]])
+; CHECK-NEXT: [[E0:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[D]], 0
+; CHECK-NEXT: [[E1:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[D]], 1
+; CHECK-NEXT: [[R:%.*]] = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> [[E1]], <4 x i32> [[E0]])
+; CHECK-NEXT: ret <8 x i32> [[R]]
+;
+ %d = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> %x)
+ %e0 = extractvalue { <4 x i32>, <4 x i32> } %d, 0
+ %e1 = extractvalue { <4 x i32>, <4 x i32> } %d, 1
+ %r = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %e1, <4 x i32> %e0)
+ ret <8 x i32> %r
+}
+
+define <8 x i32> @interleave_of_two_deinterleaves_fixed(<8 x i32> %x, <8 x i32> %y) {
+; CHECK-LABEL: @interleave_of_two_deinterleaves_fixed(
+; CHECK-NEXT: [[DX:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[X:%.*]])
+; CHECK-NEXT: [[DY:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[Y:%.*]])
+; CHECK-NEXT: [[E0:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[DX]], 0
+; CHECK-NEXT: [[E1:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[DY]], 1
+; CHECK-NEXT: [[R:%.*]] = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> [[E0]], <4 x i32> [[E1]])
+; CHECK-NEXT: ret <8 x i32> [[R]]
+;
+ %dx = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> %x)
+ %dy = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> %y)
+ %e0 = extractvalue { <4 x i32>, <4 x i32> } %dx, 0
+ %e1 = extractvalue { <4 x i32>, <4 x i32> } %dy, 1
+ %r = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %e0, <4 x i32> %e1)
+ ret <8 x i32> %r
+}
>From 5e265484030eb034a14f4e4fec5abb0209605787 Mon Sep 17 00:00:00 2001
From: Aliaksei Urbanski <aliaksei.urbanski at gmail.com>
Date: Mon, 6 Jul 2026 21:07:45 +0300
Subject: [PATCH 32/46] [NFC] Fix "is is" typos (#206288)
These changes aim to fix common "is is" typos.
---
.../docs/clang-tidy/checks/modernize/avoid-bind.rst | 2 +-
clang/include/clang/Lex/MultipleIncludeOpt.h | 2 +-
clang/lib/AST/DeclCXX.cpp | 4 ++--
clang/lib/CodeGen/README.txt | 2 +-
clang/lib/Driver/ToolChains/Flang.cpp | 2 +-
clang/lib/Headers/avx2intrin.h | 2 +-
clang/lib/Sema/SemaDeclAttr.cpp | 2 +-
clang/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp | 2 +-
clang/lib/StaticAnalyzer/Checkers/NullabilityChecker.cpp | 2 +-
compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S | 2 +-
compiler-rt/lib/hwasan/hwasan_tag_mismatch_riscv64.S | 2 +-
flang/docs/Semantics.md | 2 +-
flang/lib/Lower/CallInterface.cpp | 2 +-
flang/lib/Optimizer/Transforms/LoopInvariantCodeMotion.cpp | 2 +-
libc/src/string/memory_utils/arm/inline_memcpy.h | 2 +-
libcxx/docs/DesignDocs/FeatureTestMacros.rst | 2 +-
...ensionAllowLocationDescriptionOnTheDwarfExpressionStack.md | 2 +-
llvm/docs/LangRef.rst | 2 +-
llvm/docs/RISCVUsage.rst | 2 +-
llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp | 2 +-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 2 +-
llvm/lib/Target/ARM/ARMInstrNEON.td | 2 +-
.../Instrumentation/NumericalStabilitySanitizer.cpp | 2 +-
llvm/test/CodeGen/AArch64/regcoal-physreg.mir | 4 ++--
llvm/test/CodeGen/PowerPC/pgo-ref-directive.ll | 2 +-
mlir/include/mlir/Dialect/Async/IR/AsyncOps.td | 2 +-
mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.h | 2 +-
mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td | 4 ++--
polly/lib/External/isl/isl_union_map.c | 2 +-
29 files changed, 32 insertions(+), 32 deletions(-)
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize/avoid-bind.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize/avoid-bind.rst
index 64e7e95db8800..90e5b613560ab 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/modernize/avoid-bind.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize/avoid-bind.rst
@@ -50,7 +50,7 @@ Options
of every placeholder parameter list. Without this, it is possible for a fix-it
to perform an incorrect transformation in the case where the result of the ``bind``
is used in the context of a type erased functor such as ``std::function`` which
- allows mismatched arguments. Default is is `false`.
+ allows mismatched arguments. Default is `false`.
For example:
diff --git a/clang/include/clang/Lex/MultipleIncludeOpt.h b/clang/include/clang/Lex/MultipleIncludeOpt.h
index 8e570226c4b27..3fd750f540eff 100644
--- a/clang/include/clang/Lex/MultipleIncludeOpt.h
+++ b/clang/include/clang/Lex/MultipleIncludeOpt.h
@@ -110,7 +110,7 @@ class MultipleIncludeOpt {
/// SetReadToken - Set whether the value of 'ReadAnyTokens'. Called to
/// override when encountering tokens outside of the include guard that have
- /// no effect if the file in question is is included multiple times (e.g. the
+ /// no effect if the file in question is included multiple times (e.g. the
/// null directive).
void SetReadToken(bool Value) { ReadAnyTokens = Value; }
diff --git a/clang/lib/AST/DeclCXX.cpp b/clang/lib/AST/DeclCXX.cpp
index e7e0f75fdc84d..1e66d5f033efb 100644
--- a/clang/lib/AST/DeclCXX.cpp
+++ b/clang/lib/AST/DeclCXX.cpp
@@ -2625,7 +2625,7 @@ bool CXXMethodDecl::isUsualDeallocationFunction(
if (!PrimaryTemplate)
return true;
- // A template instance is is only a usual deallocation function if it has a
+ // A template instance is only a usual deallocation function if it has a
// type-identity parameter, the type-identity parameter is a dependent type
// (i.e. the type-identity parameter is of type std::type_identity<U> where
// U shall be a dependent type), and the type-identity parameter is the only
@@ -2645,7 +2645,7 @@ bool CXXMethodDecl::isUsualDeallocationFunction(
// A template instance is never a usual deallocation function,
// regardless of its signature.
// Post-P2719 adoption:
- // A template instance is is only a usual deallocation function if it has a
+ // A template instance is only a usual deallocation function if it has a
// type-identity parameter
if (getPrimaryTemplate())
return false;
diff --git a/clang/lib/CodeGen/README.txt b/clang/lib/CodeGen/README.txt
index e6d61095bf234..cf44e73e2efbd 100644
--- a/clang/lib/CodeGen/README.txt
+++ b/clang/lib/CodeGen/README.txt
@@ -13,7 +13,7 @@ generates an zext/sext of x which can easily be avoided.
Bitfields accesses can be shifted to simplify masking and sign
extension. For example, if the bitfield width is 8 and it is
-appropriately aligned then is is a lot shorter to just load the char
+appropriately aligned then it is a lot shorter to just load the char
directly.
//===---------------------------------------------------------------------===//
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index ee67c570fb96c..662c0c8d9bd4a 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -87,7 +87,7 @@ void Flang::addPreprocessingOptions(const ArgList &Args,
/// -Ofast
/// -O4
/// -O3
-/// For all other cases, loop versioning is is disabled.
+/// For all other cases, loop versioning is disabled.
///
/// The gfortran compiler automatically enables the option for -O3 or -Ofast.
///
diff --git a/clang/lib/Headers/avx2intrin.h b/clang/lib/Headers/avx2intrin.h
index 8eb22fe519e7d..d262a53a6a402 100644
--- a/clang/lib/Headers/avx2intrin.h
+++ b/clang/lib/Headers/avx2intrin.h
@@ -3057,7 +3057,7 @@ _mm256_broadcastsi128_si256(__m128i __X) {
/// An immediate 8-bit integer operand, with bits [7:0] specifying the
/// source for each element of the result. The position of the mask bit
/// corresponds to the index of a copied value. When a mask bit is 0, the
-/// element is copied from \a V1; otherwise, it is is copied from \a V2.
+/// element is copied from \a V1; otherwise, it is copied from \a V2.
/// \returns A 256-bit vector of [8 x i32] containing the result.
#define _mm256_blend_epi32(V1, V2, M) \
((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(V1), \
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 2159c586e5738..d3a1786bb8dbe 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -4208,7 +4208,7 @@ static void handleFormatAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
if (!S.checkUInt32Argument(AL, FirstArgExpr, FirstArg, 3))
return;
- // FirstArg == 0 is is always valid.
+ // FirstArg == 0 is always valid.
if (FirstArg != 0) {
if (Info.Kind == StrftimeFormat) {
// If the kind is strftime, FirstArg must be 0 because strftime does not
diff --git a/clang/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp
index c121052e03081..abf0bf16d4a00 100644
--- a/clang/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp
@@ -866,7 +866,7 @@ void GenericTaintChecker::checkBeginFunction(CheckerContext &C) const {
if (auto N = ArgcSVal.getAs<NonLoc>()) {
ConstraintManager &CM = C.getConstraintManager();
// The upper bound is the ARG_MAX on an arbitrary Linux
- // to model that is is typically smaller than INT_MAX.
+ // to model that is typically smaller than INT_MAX.
State = CM.assumeInclusiveRange(State, *N, llvm::APSInt::getUnsigned(1),
llvm::APSInt::getUnsigned(2097152), true);
}
diff --git a/clang/lib/StaticAnalyzer/Checkers/NullabilityChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/NullabilityChecker.cpp
index 09677a82684ce..a01143bd57949 100644
--- a/clang/lib/StaticAnalyzer/Checkers/NullabilityChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/NullabilityChecker.cpp
@@ -590,7 +590,7 @@ void NullabilityChecker::checkBeginFunction(CheckerContext &C) const {
}
// Whenever we see a load from a typed memory region that's been annotated as
-// 'nonnull', we want to trust the user on that and assume that it is is indeed
+// 'nonnull', we want to trust the user on that and assume that it is indeed
// non-null.
//
// We do so even if the value is known to have been assigned to null.
diff --git a/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S b/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S
index 1631d3257a26f..b6df86ebe8cb1 100644
--- a/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S
+++ b/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S
@@ -20,7 +20,7 @@
// particularly important as hwasan_check_* instances are duplicated in every
// translation unit where HWASan is enabled.
// This function calls HwasanTagMismatch to step back into the C++ code that
-// completes the stack unwinding and error printing. This function is is not
+// completes the stack unwinding and error printing. This function is not
// permitted to return.
diff --git a/compiler-rt/lib/hwasan/hwasan_tag_mismatch_riscv64.S b/compiler-rt/lib/hwasan/hwasan_tag_mismatch_riscv64.S
index 487a042405b60..dbbb4475bbaec 100644
--- a/compiler-rt/lib/hwasan/hwasan_tag_mismatch_riscv64.S
+++ b/compiler-rt/lib/hwasan/hwasan_tag_mismatch_riscv64.S
@@ -19,7 +19,7 @@
// This is particularly important as hwasan_check_* instances are duplicated in every
// translation unit where HWASan is enabled.
// This function calls HwasanTagMismatch to step back into the C++ code that
-// completes the stack unwinding and error printing. This function is is not
+// completes the stack unwinding and error printing. This function is not
// permitted to return.
diff --git a/flang/docs/Semantics.md b/flang/docs/Semantics.md
index 6427e2c778185..636e499ae8f64 100644
--- a/flang/docs/Semantics.md
+++ b/flang/docs/Semantics.md
@@ -15,7 +15,7 @@ local:
```
The semantic analysis pass determines if a syntactically correct Fortran
-program is is legal by enforcing the constraints of the language.
+program is legal by enforcing the constraints of the language.
The input is a parse tree with a `Program` node at the root;
and a "cooked" character stream, a contiguous stream of characters
diff --git a/flang/lib/Lower/CallInterface.cpp b/flang/lib/Lower/CallInterface.cpp
index c570e23a3078c..e317f9d760324 100644
--- a/flang/lib/Lower/CallInterface.cpp
+++ b/flang/lib/Lower/CallInterface.cpp
@@ -1578,7 +1578,7 @@ Fortran::lower::CallInterface<T>::getProcedureAttrs(
// called is recursive or not.
if (const Fortran::semantics::Symbol *sym = side().getProcedureSymbol()) {
// Note: By default procedures are RECURSIVE unless
- // -fno-automatic/-save/-Msave is set. NON_RECURSIVE is is made explicit
+ // -fno-automatic/-save/-Msave is set. NON_RECURSIVE is made explicit
// in that case in FIR.
if (sym->attrs().test(Fortran::semantics::Attr::NON_RECURSIVE) ||
(sym->owner().context().languageFeatures().IsEnabled(
diff --git a/flang/lib/Optimizer/Transforms/LoopInvariantCodeMotion.cpp b/flang/lib/Optimizer/Transforms/LoopInvariantCodeMotion.cpp
index 14fd5c6dd7fbc..ff606739cfa37 100644
--- a/flang/lib/Optimizer/Transforms/LoopInvariantCodeMotion.cpp
+++ b/flang/lib/Optimizer/Transforms/LoopInvariantCodeMotion.cpp
@@ -47,7 +47,7 @@ using namespace mlir;
/// may be added later).
/// The safety of hoisting is proven by:
/// * Proving that the loop runs at least one iteration.
-/// * Proving that is is always safe to load from this location
+/// * Proving that it is always safe to load from this location
/// (see isSafeToHoistLoad() comments below).
struct LoopInvariantCodeMotion
: fir::impl::LoopInvariantCodeMotionBase<LoopInvariantCodeMotion> {
diff --git a/libc/src/string/memory_utils/arm/inline_memcpy.h b/libc/src/string/memory_utils/arm/inline_memcpy.h
index c748048a3e586..5eddb80d25506 100644
--- a/libc/src/string/memory_utils/arm/inline_memcpy.h
+++ b/libc/src/string/memory_utils/arm/inline_memcpy.h
@@ -199,7 +199,7 @@ copy_bytes_and_bump_pointers(Ptr &dst, CPtr &src, size_t size) {
// `memcpy` prototype. This allows the compiler to merge consecutive
// load/store (LDR, STR) instructions generated in
// `copy_block_and_bump_pointers` with `BlockOp::kByWord` into load/store
- // double (LDRD, STRD) instructions, this is is undesirable so we prevent the
+ // double (LDRD, STRD) instructions, this is undesirable so we prevent the
// compiler from inferring `__restrict` with the following line.
asm volatile("" : "+r"(dst), "+r"(src));
#ifdef __ARM_FEATURE_UNALIGNED
diff --git a/libcxx/docs/DesignDocs/FeatureTestMacros.rst b/libcxx/docs/DesignDocs/FeatureTestMacros.rst
index 2c6f983ee6d01..fc3a4e355d6a0 100644
--- a/libcxx/docs/DesignDocs/FeatureTestMacros.rst
+++ b/libcxx/docs/DesignDocs/FeatureTestMacros.rst
@@ -21,7 +21,7 @@ They must be available from a list of headers, they may have different values in
different dialects, and they may or may not be implemented by libc++. In order to
track all of these conditions correctly and easily, we want a Single Source of
Truth (SSoT) that defines each feature test macro, its values, the headers it
-lives in, and whether or not is is implemented by libc++. From this SSoA we
+lives in, and whether or not it is implemented by libc++. From this SSoA we
have enough information to automatically generate the `<version>` header,
the tests, and the documentation.
diff --git a/llvm/docs/AMDGPUDwarfExtensionAllowLocationDescriptionOnTheDwarfExpressionStack/AMDGPUDwarfExtensionAllowLocationDescriptionOnTheDwarfExpressionStack.md b/llvm/docs/AMDGPUDwarfExtensionAllowLocationDescriptionOnTheDwarfExpressionStack/AMDGPUDwarfExtensionAllowLocationDescriptionOnTheDwarfExpressionStack.md
index 88d9a9f8c4b07..4a469642544b3 100644
--- a/llvm/docs/AMDGPUDwarfExtensionAllowLocationDescriptionOnTheDwarfExpressionStack/AMDGPUDwarfExtensionAllowLocationDescriptionOnTheDwarfExpressionStack.md
+++ b/llvm/docs/AMDGPUDwarfExtensionAllowLocationDescriptionOnTheDwarfExpressionStack/AMDGPUDwarfExtensionAllowLocationDescriptionOnTheDwarfExpressionStack.md
@@ -3205,7 +3205,7 @@ location list expressions.</i>
subprogram instance that immediately lexically encloses the current call
frame's subprogram or entry point.
- The DWARF is ill-formed if L is is not comprised of one memory location
+ The DWARF is ill-formed if L is not comprised of one memory location
description for one of the target architecture specific address spaces.
### A.3.4 Call Site Entries and Parameters
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 85eec0f4bee49..6e7f63ed7fb69 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -15760,7 +15760,7 @@ knows those logical layouts are lowered to the same physical layout:
- `{ i32, i32, i32, i32 }`
- `[ i32 x 4 ]`
-This means is is valid to lower the following code to either:
+This means it is valid to lower the following code to either:
.. code-block:: llvm
diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst
index 548f25ac2e596..7cfbd967eee77 100644
--- a/llvm/docs/RISCVUsage.rst
+++ b/llvm/docs/RISCVUsage.rst
@@ -627,7 +627,7 @@ To use this functionality, you need to be doing all of the following:
LLD will relax (rewrite) any code sequences that materialize an address within 2048 bytes of ``__global_pointer$`` (which will be defined if it is used and does not already exist) to instead generate the address using ``gp`` and the correct (signed) 12-bit immediate. This usually saves at least one instruction compared to materialising a full 32-bit address value.
-There can only be one ``gp`` value in a process (as ``gp`` is not changed when calling into a function in a shared library), so the symbol is is only defined and this relaxation is only done for executables, and not for shared libraries. The linker expects executable startup code to put the value of ``__global_pointer$`` (from the executable) into ``gp`` before any user code is run.
+There can only be one ``gp`` value in a process (as ``gp`` is not changed when calling into a function in a shared library), so the symbol is only defined and this relaxation is only done for executables, and not for shared libraries. The linker expects executable startup code to put the value of ``__global_pointer$`` (from the executable) into ``gp`` before any user code is run.
Arguably, the most efficient use for this addressing mode is for smaller global variables, as larger global variables likely need many more loads or stores when they are being accessed anyway, so the cost of materializing the upper bits can be shared.
diff --git a/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp b/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp
index b1e7b09306ac5..5c43412fe5b90 100644
--- a/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp
+++ b/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp
@@ -483,7 +483,7 @@ static void convertFloatingToInteger(BlockFrequencyInfoImplBase &BFI,
const Scaled64 &Min, const Scaled64 &Max) {
// Scale the Factor to a size that creates integers. If possible scale
// integers so that Max == UINT64_MAX so that they can be best differentiated.
- // Is is possible that the range between min and max cannot be accurately
+ // It is possible that the range between min and max cannot be accurately
// represented in a 64bit integer without either loosing precision for small
// values (so small unequal numbers all map to 1) or saturaturing big numbers
// loosing precision for big numbers (so unequal big numbers may map to
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index b46359ec91e8a..a29070953ee38 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -2901,7 +2901,7 @@ void SITargetLowering::allocateSpecialEntryInputVGPRs(
// Try to allocate a VGPR at the end of the argument list, or if no argument
// VGPRs are left allocating a stack slot.
-// If \p Mask is is given it indicates bitfield position in the register.
+// If \p Mask is given it indicates bitfield position in the register.
// If \p Arg is given use it with new ]p Mask instead of allocating new.
static ArgDescriptor allocateVGPR32Input(CCState &CCInfo, unsigned Mask = ~0u,
ArgDescriptor Arg = ArgDescriptor()) {
diff --git a/llvm/lib/Target/ARM/ARMInstrNEON.td b/llvm/lib/Target/ARM/ARMInstrNEON.td
index ea322e38420ff..1697a48626ca8 100644
--- a/llvm/lib/Target/ARM/ARMInstrNEON.td
+++ b/llvm/lib/Target/ARM/ARMInstrNEON.td
@@ -5564,7 +5564,7 @@ def : Pat<(v16i8 (vnotq QPR:$src)),
// The TwoAddress pass will not go looking for equivalent operations
// with different register constraints; it just inserts copies.
-// That is why pseudo VBSP implemented. Is is expanded later into
+// That is why pseudo VBSP implemented. It is expanded later into
// VBIT/VBIF/VBSL taking into account register constraints to avoid copies.
def VBSPd : PseudoNeonI<(outs DPR:$Vd), (ins DPR:$src1, DPR:$Vn, DPR:$Vm),
IIC_VBINiD, "", []>;
diff --git a/llvm/lib/Transforms/Instrumentation/NumericalStabilitySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/NumericalStabilitySanitizer.cpp
index de2854d79277e..df22d1bd5548e 100644
--- a/llvm/lib/Transforms/Instrumentation/NumericalStabilitySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/NumericalStabilitySanitizer.cpp
@@ -2089,7 +2089,7 @@ bool NumericalStabilitySanitizer::sanitizeFunction(
//
// For example, in the following example, the instrumentation in
// `instrumented_1` rejects the shadow return value from `instrumented_3`
- // because is is not tagged as expected (`&instrumented_3` instead of
+ // because it is not tagged as expected (`&instrumented_3` instead of
// `non_instrumented_2`):
//
// instrumented_1()
diff --git a/llvm/test/CodeGen/AArch64/regcoal-physreg.mir b/llvm/test/CodeGen/AArch64/regcoal-physreg.mir
index d5e4742c3ffaf..dfe7fbd959715 100644
--- a/llvm/test/CodeGen/AArch64/regcoal-physreg.mir
+++ b/llvm/test/CodeGen/AArch64/regcoal-physreg.mir
@@ -32,7 +32,7 @@ body: |
$fp = SUBXri $fp, 4, 0
STRXui %2, $x1, 0
- ; Is is fine to coalesce copies from constant physregs even when they are
+ ; It is fine to coalesce copies from constant physregs even when they are
; clobbered.
; CHECK-NOT: COPY
; CHECK: STRWui $wzr, $x1
@@ -40,7 +40,7 @@ body: |
dead $wzr = SUBSWri $w1, 0, 0, implicit-def $nzcv
STRWui %3, $x1, 0
- ; Is is fine to coalesce copies from constant physregs even when they are
+ ; It is fine to coalesce copies from constant physregs even when they are
; clobbered.
; CHECK-NOT: COPY
; CHECK: STRXui $xzr, $x1
diff --git a/llvm/test/CodeGen/PowerPC/pgo-ref-directive.ll b/llvm/test/CodeGen/PowerPC/pgo-ref-directive.ll
index 02b0a9ba4397c..e6e09f9c0f669 100644
--- a/llvm/test/CodeGen/PowerPC/pgo-ref-directive.ll
+++ b/llvm/test/CodeGen/PowerPC/pgo-ref-directive.ll
@@ -83,7 +83,7 @@ entry:
}
; There will be two __llvm_prf_cnts .csects, one to represent the actual csect
; that holds @__profc_main, and one generated to hold the .ref directives. In
-; XCOFF, a csect can be defined in pieces, so this is is legal assembly.
+; XCOFF, a csect can be defined in pieces, so this is legal assembly.
;
; NOVNDS: .csect __llvm_prf_cnts[RW],3
; NOVNDS: .csect __llvm_prf_cnts[RW],3
diff --git a/mlir/include/mlir/Dialect/Async/IR/AsyncOps.td b/mlir/include/mlir/Dialect/Async/IR/AsyncOps.td
index 058f58bda6433..722370b8f3e29 100644
--- a/mlir/include/mlir/Dialect/Async/IR/AsyncOps.td
+++ b/mlir/include/mlir/Dialect/Async/IR/AsyncOps.td
@@ -486,7 +486,7 @@ def Async_CoroEndOp : Async_Op<"coro.end"> {
let description = [{
The `async.coro.end` marks the point where a coroutine needs to return
control back to the caller if it is not an initial invocation of the
- coroutine. It the start part of the coroutine is is no-op.
+ coroutine. In the start part of the coroutine it is no-op.
}];
let arguments = (ins Async_CoroHandleType:$handle);
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.h b/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.h
index bed70816d7f1e..9ff32216ce042 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.h
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.h
@@ -154,7 +154,7 @@ isaTransposeOpInterface(GenericOp genericOp);
/// the category op.
/// A linalg.generic body could be a series of unary elementwise ops e.g.
/// `exp(neg(x))`, such as formed by linalg op fusion. Here we restrict it to
-/// detecting cases where body is is a single computation op.
+/// detecting cases where body is a single computation op.
bool isaElemwiseSingleUnaryOpInterface(GenericOp genericOp,
bool allowNonIdentityMaps = false);
diff --git a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td
index 5a1771eecd0f6..679dfdb9a4748 100644
--- a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td
+++ b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td
@@ -363,7 +363,7 @@ def NVGPU_MBarrierInitOp : NVGPU_Op<"mbarrier.init", []> {
def NVGPU_MBarrierTestWaitOp : NVGPU_Op<"mbarrier.test.wait", []> {
let summary = "Checks if the `nvgpu.mbarrier` has completed its current phase.";
let description = [{
- Checks whether the mbarrier object has completed the phase. It is is a
+ Checks whether the mbarrier object has completed the phase. It is a
non-blocking instruction which tests for the completion of the phase.
Example:
@@ -438,7 +438,7 @@ def NVGPU_MBarrierArriveExpectTxOp : NVGPU_Op<"mbarrier.arrive.expect_tx", []> {
def NVGPU_MBarrierTryWaitParityOp : NVGPU_Op<"mbarrier.try_wait.parity", []> {
let summary = "Waits for the `nvgpu.mbarrier` to complete its current phase.";
let description = [{
- Checks whether the mbarrier object has completed the phase. It is is a
+ Checks whether the mbarrier object has completed the phase. It is a
potentially blocking instruction which tests for the completion of the
phase. Suspended thread resumes execution when the specified phase completes
OR before the phase completes following a system-dependent time limit.
diff --git a/polly/lib/External/isl/isl_union_map.c b/polly/lib/External/isl/isl_union_map.c
index c8393f5e36403..bcd6f33ed0a0f 100644
--- a/polly/lib/External/isl/isl_union_map.c
+++ b/polly/lib/External/isl/isl_union_map.c
@@ -3136,7 +3136,7 @@ isl_bool isl_union_map_is_injective(__isl_keep isl_union_map *umap)
* In particular, if the domain and range spaces are the same,
* then the map is not considered to obviously not be an identity relation.
* Otherwise, the map is considered to obviously not be an identity relation
- * if it is is non-empty.
+ * if it is non-empty.
*
* If "map" is determined to obviously not be an identity relation,
* then the search is aborted.
>From 03c62ca40d19ba03c7d2b69dbedc95e498d46071 Mon Sep 17 00:00:00 2001
From: Alex Strelnikov <strel at google.com>
Date: Mon, 6 Jul 2026 14:14:59 -0400
Subject: [PATCH 33/46] [libc] Add Q length modifier to support float128
conversions in printf (#203077)
The spelling follows an existing pattern in e.g. libquadmath for format
printing `__float128` values.
A flag is added to disable float128 conversions and the "Q" length
modifier.
---
libc/docs/dev/printf_behavior.rst | 16 +-
libc/src/__support/float_to_string.h | 63 ++--
libc/src/stdio/printf_core/converter_utils.h | 4 +
libc/src/stdio/printf_core/core_structs.h | 17 +-
.../stdio/printf_core/float_dec_converter.h | 64 ++--
.../printf_core/float_dec_converter_limited.h | 29 +-
.../stdio/printf_core/float_hex_converter.h | 134 ++++----
.../printf_core/float_inf_nan_converter.h | 48 +--
libc/src/stdio/printf_core/parser.h | 66 +++-
libc/src/stdio/printf_core/printf_config.h | 5 +
.../stdio/printf_core/write_int_converter.h | 5 +
libc/test/UnitTest/CMakeLists.txt | 1 +
libc/test/UnitTest/PrintfMatcher.cpp | 9 +-
libc/test/src/stdio/CMakeLists.txt | 1 +
libc/test/src/stdio/sprintf_test.cpp | 305 ++++++++++++++++++
15 files changed, 614 insertions(+), 153 deletions(-)
diff --git a/libc/docs/dev/printf_behavior.rst b/libc/docs/dev/printf_behavior.rst
index 4b53e0b85f6d2..7d9925eed0acb 100644
--- a/libc/docs/dev/printf_behavior.rst
+++ b/libc/docs/dev/printf_behavior.rst
@@ -105,6 +105,13 @@ with hexadecimal long double conversions (%La). This will improve performance
significantly, but may cause some tests to fail. This has no effect when float
conversions are disabled.
+LIBC_COPT_PRINTF_NO_CONVERT_FLOAT128
+------------------------------------
+When set, this flag disables support for __float128 conversions using the "Q"
+length modifier (%Qa, %Qf, %Qe, %Qg). This flag has no effect on conversions
+using the "L" length modifier when long double is the same type as __float128.
+This has little to no effect on performance or binary size.
+
--------------------------------
Float Conversion Internal Flags:
--------------------------------
@@ -129,7 +136,7 @@ LIBC_COPT_FLOAT_TO_STR_USE_MEGA_LONG_DOUBLE_TABLE
-------------------------------------------------
When set, the float to string decimal conversion algorithm will use a larger
table to accelerate long double conversions. This larger table is around 5MB of
-size when compiled.
+size when compiled. This flag also affects __float128 conversions.
LIBC_COPT_FLOAT_TO_STR_USE_DYADIC_FLOAT
---------------------------------------
@@ -244,3 +251,10 @@ value of errno as an integer with the %d format, including all options. If
errno = 0 and alt form is specified, the conversion will be a string conversion
on "0" for simplicity of implementation. This matches what other libcs
implementing this feature have done.
+
+If the compiler is detected as having support for __float128, "Q" is an accepted
+length modifier for floating point conversions (%Qa, %Qf, %Qe, %Qg), unless
+disabled by LIBC_COPT_PRINTF_NO_CONVERT_FLOAT128. A conversion using the
+"Q" length modifier will be treated as invalid in any of the following
+conditions: __float128 is not supported, the "Q" length modifier is disabled, or
+the conversion does not use a floating point format specifier.
diff --git a/libc/src/__support/float_to_string.h b/libc/src/__support/float_to_string.h
index 683fb75aeda3f..6818b74c81545 100644
--- a/libc/src/__support/float_to_string.h
+++ b/libc/src/__support/float_to_string.h
@@ -385,6 +385,14 @@ LIBC_INLINE uint32_t mul_shift_mod_1e9(const FPBits::StorageType mantissa,
} // namespace internal
+#if defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64) || \
+ defined(LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE) || \
+ defined(LIBC_COPT_FLOAT_TO_STR_NO_SPECIALIZE_LD)
+#define LIBC_INTERNAL_FLOAT_TO_STR_LD_USE_RYU_IMPL
+#endif
+
+template <typename T, typename U = void> class FloatToString;
+
// Convert floating point values to their string representation.
// Because the result may not fit in a reasonably sized array, the caller must
// request blocks of digits and convert them from integers to strings themself.
@@ -401,8 +409,13 @@ LIBC_INLINE uint32_t mul_shift_mod_1e9(const FPBits::StorageType mantissa,
// be zero after the decimal point and before the non-zero digits. Additionally,
// is_lowest_block will return if the current block is the lowest non-zero
// block.
-template <typename T, cpp::enable_if_t<cpp::is_floating_point_v<T>, int> = 0>
-class FloatToString {
+template <typename T>
+class FloatToString<
+ T, cpp::enable_if_t<cpp::is_same_v<T, float> || cpp::is_same_v<T, double>
+#if defined(LIBC_INTERNAL_FLOAT_TO_STR_LD_USE_RYU_IMPL)
+ || cpp::is_same_v<T, long double>
+#endif // LIBC_INTERNAL_FLOAT_TO_STR_LD_USE_RYU_IMPL
+ >> {
fputil::FPBits<T> float_bits;
int exponent;
FPBits::StorageType mantissa;
@@ -605,36 +618,46 @@ class FloatToString {
}
};
-#if !defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64) && \
- !defined(LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE) && \
- !defined(LIBC_COPT_FLOAT_TO_STR_NO_SPECIALIZE_LD)
-// --------------------------- LONG DOUBLE FUNCTIONS ---------------------------
+#if !defined(LIBC_INTERNAL_FLOAT_TO_STR_LD_USE_RYU_IMPL) || \
+ defined(LIBC_TYPES_HAS_FLOAT128)
+// -------------------- LONG DOUBLE / FLOAT128 FUNCTIONS -----------------------
-// this algorithm will work exactly the same for 80 bit and 128 bit long
+// This algorithm will work exactly the same for 80 bit and 128 bit long
// doubles. They have the same max exponent, but even if they didn't the
// constants should be calculated to be correct for any provided floating point
// type.
-template <> class FloatToString<long double> {
- fputil::FPBits<long double> float_bits;
+#if defined(LIBC_TYPES_HAS_FLOAT128)
+template <typename T> struct IsFloat128 : cpp::is_same<T, float128> {};
+#else
+template <typename T> struct IsFloat128 : cpp::bool_constant<false> {};
+#endif
+
+template <typename T>
+class FloatToString<T, cpp::enable_if_t<IsFloat128<T>::value
+#if defined(LIBC_INTERNAL_FLOAT_TO_STR_LD_USE_RYU_IMPL)
+ && !cpp::is_same_v<T, long double>
+#else
+ || cpp::is_same_v<T, long double>
+#endif // LIBC_INTERNAL_FLOAT_TO_STR_LD_USE_RYU_IMPL
+ >> {
+ fputil::FPBits<T> float_bits;
bool is_negative = 0;
int exponent = 0;
- fputil::FPBits<long double>::StorageType mantissa = 0;
+ typename fputil::FPBits<T>::StorageType mantissa = 0;
- static constexpr int FRACTION_LEN = fputil::FPBits<long double>::FRACTION_LEN;
- static constexpr int EXP_BIAS = fputil::FPBits<long double>::EXP_BIAS;
+ static constexpr int FRACTION_LEN = fputil::FPBits<T>::FRACTION_LEN;
+ static constexpr int EXP_BIAS = fputil::FPBits<T>::EXP_BIAS;
static constexpr size_t UINT_WORD_SIZE = 64;
static constexpr size_t FLOAT_AS_INT_WIDTH =
internal::div_ceil(
- fputil::FPBits<long double>::MAX_BIASED_EXPONENT -
- fputil::FPBits<long double>::EXP_BIAS +
+ fputil::FPBits<T>::MAX_BIASED_EXPONENT - fputil::FPBits<T>::EXP_BIAS +
FRACTION_LEN, // Add fraction len to provide space for subnormals.
UINT_WORD_SIZE) *
UINT_WORD_SIZE;
static constexpr size_t EXTRA_INT_WIDTH =
- internal::div_ceil(sizeof(long double) * CHAR_BIT, UINT_WORD_SIZE) *
- UINT_WORD_SIZE;
+ internal::div_ceil(sizeof(T) * CHAR_BIT, UINT_WORD_SIZE) * UINT_WORD_SIZE;
using wide_int = UInt<FLOAT_AS_INT_WIDTH + EXTRA_INT_WIDTH>;
@@ -705,7 +728,7 @@ template <> class FloatToString<long double> {
// if the shift amount would be negative, then the shift would cause a
// loss of precision.
LIBC_ASSERT(SHIFT_AMOUNT >= 0);
- static_assert(EXTRA_INT_WIDTH >= sizeof(long double) * 8);
+ static_assert(EXTRA_INT_WIDTH >= sizeof(T) * 8);
float_as_fixed <<= SHIFT_AMOUNT;
// If there are still digits above the decimal point, handle those.
@@ -730,8 +753,7 @@ template <> class FloatToString<long double> {
}
public:
- LIBC_INLINE constexpr FloatToString(long double init_float)
- : float_bits(init_float) {
+ LIBC_INLINE constexpr FloatToString(T init_float) : float_bits(init_float) {
is_negative = float_bits.is_neg();
exponent = float_bits.get_explicit_exponent();
mantissa = float_bits.get_explicit_mantissa();
@@ -834,8 +856,7 @@ template <> class FloatToString<long double> {
}
};
-#endif // !LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64 &&
- // !LIBC_COPT_FLOAT_TO_STR_NO_SPECIALIZE_LD
+#endif // !LIBC_INTERNAL_FLOAT_TO_STR_LD_USE_RYU_IMPL || LIBC_TYPES_HAS_FLOAT128
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/stdio/printf_core/converter_utils.h b/libc/src/stdio/printf_core/converter_utils.h
index ed656ffb6e72f..b27defddc354f 100644
--- a/libc/src/stdio/printf_core/converter_utils.h
+++ b/libc/src/stdio/printf_core/converter_utils.h
@@ -57,6 +57,10 @@ LIBC_INLINE uintmax_t apply_length_modifier(uintmax_t num,
return num & mask;
}
#endif // LIBC_COPT_PRINTF_DISABLE_BITINT
+#if defined(LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128)
+ case LengthModifier::Q: // This case should never happen for integers.
+ return num;
+#endif // LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128
}
__builtin_unreachable();
}
diff --git a/libc/src/stdio/printf_core/core_structs.h b/libc/src/stdio/printf_core/core_structs.h
index 705865745db47..9b74a0a391f95 100644
--- a/libc/src/stdio/printf_core/core_structs.h
+++ b/libc/src/stdio/printf_core/core_structs.h
@@ -33,6 +33,9 @@ enum class LengthModifier {
z,
t,
L,
+#if defined(LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128)
+ Q,
+#endif // LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128
#ifndef LIBC_COPT_PRINTF_DISABLE_BITINT
w,
wf,
@@ -45,6 +48,12 @@ struct LengthSpec {
size_t bit_width;
};
+// Type large enough to store the raw bits of any floating point type.
+//
+// Does not use any specialization of FPBits, because it is unavailable on
+// PowerPC.
+using AnyFloatStorageType = UInt128;
+
enum FormatFlags : uint8_t {
LEFT_JUSTIFIED = 0x01, // -
FORCE_SIGN = 0x02, // +
@@ -69,13 +78,7 @@ struct FormatSection {
int min_width = 0;
int precision = -1;
- // Needs to be large enough to hold a long double. Special case handling for
- // the PowerPC double double type because it has no FPBits interface.
-#ifdef LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE
- UInt128 conv_val_raw;
-#else
- fputil::FPBits<long double>::StorageType conv_val_raw;
-#endif // LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE
+ AnyFloatStorageType conv_val_raw;
void *conv_val_ptr;
char conv_name;
diff --git a/libc/src/stdio/printf_core/float_dec_converter.h b/libc/src/stdio/printf_core/float_dec_converter.h
index 005481b24426c..7d86de3692115 100644
--- a/libc/src/stdio/printf_core/float_dec_converter.h
+++ b/libc/src/stdio/printf_core/float_dec_converter.h
@@ -609,16 +609,9 @@ template <typename T, WriteMode write_mode,
LIBC_INLINE int convert_float_dec_exp_typed(Writer<write_mode> *writer,
const FormatSection &to_conv,
fputil::FPBits<T> float_bits) {
-#ifdef LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE
- using StorageType = UInt128;
-#else
- using StorageType = fputil::FPBits<long double>::StorageType;
-#endif // LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE
-
// signed because later we use -FRACTION_LEN
constexpr int32_t FRACTION_LEN = fputil::FPBits<T>::FRACTION_LEN;
int exponent = float_bits.get_explicit_exponent();
- StorageType mantissa = float_bits.get_explicit_mantissa();
char sign_char = 0;
@@ -657,7 +650,7 @@ LIBC_INLINE int convert_float_dec_exp_typed(Writer<write_mode> *writer,
// If the mantissa is 0, then the number is 0, meaning that looping until a
// non-zero block is found will loop forever. The first block is just 0.
- if (mantissa != 0) {
+ if (float_bits.get_explicit_mantissa() != 0) {
// This loop finds the first block.
while (digits == 0) {
--cur_block;
@@ -777,16 +770,9 @@ template <typename T, WriteMode write_mode,
LIBC_INLINE int convert_float_dec_auto_typed(Writer<write_mode> *writer,
const FormatSection &to_conv,
fputil::FPBits<T> float_bits) {
-#ifdef LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE
- using StorageType = UInt128;
-#else
- using StorageType = fputil::FPBits<long double>::StorageType;
-#endif // LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE
-
// signed because later we use -FRACTION_LEN
constexpr int32_t FRACTION_LEN = fputil::FPBits<T>::FRACTION_LEN;
int exponent = float_bits.get_explicit_exponent();
- StorageType mantissa = float_bits.get_explicit_mantissa();
// From the standard: Let P (init_precision) equal the precision if nonzero, 6
// if the precision is omitted, or 1 if the precision is zero.
@@ -821,7 +807,7 @@ LIBC_INLINE int convert_float_dec_auto_typed(Writer<write_mode> *writer,
// If the mantissa is 0, then the number is 0, meaning that looping until a
// non-zero block is found will loop forever.
- if (mantissa != 0) {
+ if (float_bits.get_explicit_mantissa() != 0) {
// This loop finds the first non-zero block.
while (digits == 0) {
--cur_block;
@@ -1150,9 +1136,20 @@ LIBC_INLINE int convert_float_dec_auto_typed(Writer<write_mode> *writer,
template <WriteMode write_mode>
LIBC_INLINE int convert_float_decimal(Writer<write_mode> *writer,
const FormatSection &to_conv) {
+#if defined(LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128)
+ if (to_conv.length_modifier == LengthModifier::Q) {
+ fputil::FPBits<float128>::StorageType float_raw = to_conv.conv_val_raw;
+ fputil::FPBits<float128> float_bits(float_raw);
+ if (!float_bits.is_inf_or_nan()) {
+ return convert_float_decimal_typed<float128>(writer, to_conv, float_bits);
+ }
+ } else
+#endif // LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128
#ifndef LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE
- if (to_conv.length_modifier == LengthModifier::L) {
- fputil::FPBits<long double>::StorageType float_raw = to_conv.conv_val_raw;
+ if (to_conv.length_modifier == LengthModifier::L) {
+ fputil::FPBits<long double>::StorageType float_raw =
+ static_cast<fputil::FPBits<long double>::StorageType>(
+ to_conv.conv_val_raw);
fputil::FPBits<long double> float_bits(float_raw);
if (!float_bits.is_inf_or_nan()) {
return convert_float_decimal_typed<long double>(writer, to_conv,
@@ -1175,9 +1172,20 @@ LIBC_INLINE int convert_float_decimal(Writer<write_mode> *writer,
template <WriteMode write_mode>
LIBC_INLINE int convert_float_dec_exp(Writer<write_mode> *writer,
const FormatSection &to_conv) {
+#if defined(LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128)
+ if (to_conv.length_modifier == LengthModifier::Q) {
+ fputil::FPBits<float128>::StorageType float_raw = to_conv.conv_val_raw;
+ fputil::FPBits<float128> float_bits(float_raw);
+ if (!float_bits.is_inf_or_nan()) {
+ return convert_float_dec_exp_typed<float128>(writer, to_conv, float_bits);
+ }
+ } else
+#endif // LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128
#ifndef LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE
- if (to_conv.length_modifier == LengthModifier::L) {
- fputil::FPBits<long double>::StorageType float_raw = to_conv.conv_val_raw;
+ if (to_conv.length_modifier == LengthModifier::L) {
+ fputil::FPBits<long double>::StorageType float_raw =
+ static_cast<fputil::FPBits<long double>::StorageType>(
+ to_conv.conv_val_raw);
fputil::FPBits<long double> float_bits(float_raw);
if (!float_bits.is_inf_or_nan()) {
return convert_float_dec_exp_typed<long double>(writer, to_conv,
@@ -1200,9 +1208,21 @@ LIBC_INLINE int convert_float_dec_exp(Writer<write_mode> *writer,
template <WriteMode write_mode>
LIBC_INLINE int convert_float_dec_auto(Writer<write_mode> *writer,
const FormatSection &to_conv) {
+#if defined(LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128)
+ if (to_conv.length_modifier == LengthModifier::Q) {
+ fputil::FPBits<float128>::StorageType float_raw = to_conv.conv_val_raw;
+ fputil::FPBits<float128> float_bits(float_raw);
+ if (!float_bits.is_inf_or_nan()) {
+ return convert_float_dec_auto_typed<float128>(writer, to_conv,
+ float_bits);
+ }
+ } else
+#endif // LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128
#ifndef LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE
- if (to_conv.length_modifier == LengthModifier::L) {
- fputil::FPBits<long double>::StorageType float_raw = to_conv.conv_val_raw;
+ if (to_conv.length_modifier == LengthModifier::L) {
+ fputil::FPBits<long double>::StorageType float_raw =
+ static_cast<fputil::FPBits<long double>::StorageType>(
+ to_conv.conv_val_raw);
fputil::FPBits<long double> float_bits(float_raw);
if (!float_bits.is_inf_or_nan()) {
return convert_float_dec_auto_typed<long double>(writer, to_conv,
diff --git a/libc/src/stdio/printf_core/float_dec_converter_limited.h b/libc/src/stdio/printf_core/float_dec_converter_limited.h
index 637369d2f8eda..2693ebd531f96 100644
--- a/libc/src/stdio/printf_core/float_dec_converter_limited.h
+++ b/libc/src/stdio/printf_core/float_dec_converter_limited.h
@@ -78,7 +78,8 @@ struct DigitsInput {
// Constructor which accepts a mantissa direct from a floating-point format,
// and shifts it up to the top of the UInt128 so that a function consuming
// this struct afterwards doesn't have to remember which format it came from.
- DigitsInput(int32_t fraction_len, UInt128 mantissa_, int exponent_, Sign sign)
+ DigitsInput(int32_t fraction_len, AnyFloatStorageType mantissa_,
+ int exponent_, Sign sign)
: mantissa(UInt128(mantissa_) << (127 - fraction_len)),
exponent(exponent_), sign(sign) {
if (!(mantissa & (UInt128(1) << 127)) && mantissa != 0) {
@@ -374,10 +375,11 @@ DigitsOutput decimal_digits(DigitsInput input, int precision, bool e_mode) {
}
template <WriteMode write_mode>
-LIBC_INLINE int
-convert_float_inner(Writer<write_mode> *writer, const FormatSection &to_conv,
- int32_t fraction_len, int exponent, UInt128 mantissa,
- Sign sign, ConversionType ctype) {
+LIBC_INLINE int convert_float_inner(Writer<write_mode> *writer,
+ const FormatSection &to_conv,
+ int32_t fraction_len, int exponent,
+ AnyFloatStorageType mantissa, Sign sign,
+ ConversionType ctype) {
constexpr char DECIMAL_POINT = '.';
// If to_conv doesn't specify a precision, the precision defaults to 6.
unsigned precision = to_conv.precision < 0 ? 6 : to_conv.precision;
@@ -631,10 +633,21 @@ template <WriteMode write_mode>
LIBC_INLINE int convert_float_outer(Writer<write_mode> *writer,
const FormatSection &to_conv,
ConversionType ctype) {
+#if defined(LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128)
+ if (to_conv.length_modifier == LengthModifier::Q) {
+ fputil::FPBits<float128> float_bits(
+ static_cast<fputil::FPBits<float128>::StorageType>(
+ to_conv.conv_val_raw));
+ if (!float_bits.is_inf_or_nan()) {
+ return convert_float_typed<float128>(writer, to_conv, float_bits, ctype);
+ }
+ } else
+#endif // LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128
#ifndef LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE
- if (to_conv.length_modifier == LengthModifier::L) {
- fputil::FPBits<long double>::StorageType float_raw = to_conv.conv_val_raw;
- fputil::FPBits<long double> float_bits(float_raw);
+ if (to_conv.length_modifier == LengthModifier::L) {
+ fputil::FPBits<long double> float_bits(
+ static_cast<fputil::FPBits<long double>::StorageType>(
+ to_conv.conv_val_raw));
if (!float_bits.is_inf_or_nan()) {
return convert_float_typed<long double>(writer, to_conv, float_bits,
ctype);
diff --git a/libc/src/stdio/printf_core/float_hex_converter.h b/libc/src/stdio/printf_core/float_hex_converter.h
index efa01970c365f..f97c06f9dafbd 100644
--- a/libc/src/stdio/printf_core/float_hex_converter.h
+++ b/libc/src/stdio/printf_core/float_hex_converter.h
@@ -25,52 +25,72 @@
namespace LIBC_NAMESPACE_DECL {
namespace printf_core {
-template <WriteMode write_mode>
-LIBC_INLINE int convert_float_hex_exp(Writer<write_mode> *writer,
- const FormatSection &to_conv) {
-#ifdef LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE
- using LDBits = fputil::FPBits<double>;
- using StorageType = LDBits::StorageType;
-#else
- using LDBits = fputil::FPBits<long double>;
- using StorageType = LDBits::StorageType;
-#endif // LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE
-
+struct FloatHexExpFPBitsProperties {
bool is_negative;
int exponent;
- StorageType mantissa;
+ AnyFloatStorageType mantissa;
bool is_inf_or_nan;
uint32_t fraction_bits;
+};
+
+template <typename T>
+FloatHexExpFPBitsProperties
+get_float_hex_exp_fp_bits_properties(AnyFloatStorageType float_raw) {
+ fputil::FPBits<T> float_bits(
+ static_cast<typename fputil::FPBits<T>::StorageType>(float_raw));
+ return {
+ .is_negative = float_bits.is_neg(),
+ .exponent = float_bits.get_explicit_exponent(),
+ .mantissa = float_bits.get_explicit_mantissa(),
+ .is_inf_or_nan = float_bits.is_inf_or_nan(),
+ .fraction_bits = fputil::FPBits<T>::FRACTION_LEN,
+ };
+}
+template <WriteMode write_mode>
+LIBC_INLINE int convert_float_hex_exp(Writer<write_mode> *writer,
+ const FormatSection &to_conv) {
+#if defined(LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128)
+ static constexpr uint32_t MAX_POSSIBLE_FRACTION_LEN =
+ fputil::FPBits<float128>::FRACTION_LEN;
+ static constexpr uint32_t MAX_POSSIBLE_EXP_LEN =
+ fputil::FPBits<float128>::EXP_LEN;
+#elif !defined(LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE)
+ static constexpr uint32_t MAX_POSSIBLE_FRACTION_LEN =
+ fputil::FPBits<long double>::FRACTION_LEN;
+ static constexpr uint32_t MAX_POSSIBLE_EXP_LEN =
+ fputil::FPBits<long double>::EXP_LEN;
+#else
+ static constexpr uint32_t MAX_POSSIBLE_FRACTION_LEN =
+ fputil::FPBits<double>::FRACTION_LEN;
+ static constexpr uint32_t MAX_POSSIBLE_EXP_LEN =
+ fputil::FPBits<double>::EXP_LEN;
+#endif
+
+ FloatHexExpFPBitsProperties properties;
+#if defined(LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128)
+ if (to_conv.length_modifier == LengthModifier::Q) {
+ properties =
+ get_float_hex_exp_fp_bits_properties<float128>(to_conv.conv_val_raw);
+ } else
+#endif // LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128
#ifndef LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE
- if (to_conv.length_modifier == LengthModifier::L) {
- fraction_bits = LDBits::FRACTION_LEN;
- LDBits::StorageType float_raw = to_conv.conv_val_raw;
- LDBits float_bits(float_raw);
- is_negative = float_bits.is_neg();
- exponent = float_bits.get_explicit_exponent();
- mantissa = float_bits.get_explicit_mantissa();
- is_inf_or_nan = float_bits.is_inf_or_nan();
+ if (to_conv.length_modifier == LengthModifier::L) {
+ properties =
+ get_float_hex_exp_fp_bits_properties<long double>(to_conv.conv_val_raw);
} else
#endif // !LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE
{
- using LBits = fputil::FPBits<double>;
- fraction_bits = LBits::FRACTION_LEN;
- LBits::StorageType float_raw =
- static_cast<LBits::StorageType>(to_conv.conv_val_raw);
- LBits float_bits(float_raw);
- is_negative = float_bits.is_neg();
- exponent = float_bits.get_explicit_exponent();
- mantissa = float_bits.get_explicit_mantissa();
- is_inf_or_nan = float_bits.is_inf_or_nan();
+ properties =
+ get_float_hex_exp_fp_bits_properties<double>(to_conv.conv_val_raw);
}
- if (is_inf_or_nan)
+ if (properties.is_inf_or_nan)
return convert_inf_nan(writer, to_conv);
char sign_char = 0;
- if (is_negative)
+ if (properties.is_negative)
sign_char = '-';
else if ((to_conv.flags & FormatFlags::FORCE_SIGN) == FormatFlags::FORCE_SIGN)
sign_char = '+'; // FORCE_SIGN has precedence over SPACE_PREFIX
@@ -84,8 +104,9 @@ LIBC_INLINE int convert_float_hex_exp(Writer<write_mode> *writer,
// digits. This is primarily relevant for x86 80 bit long doubles, which have
// 63 bit mantissas. In the case where the mantissa is 0, however, the
// exponent should stay as 0.
- if (fraction_bits % BITS_IN_HEX_DIGIT != 0 && mantissa > 0) {
- exponent -= fraction_bits % BITS_IN_HEX_DIGIT;
+ if (properties.fraction_bits % BITS_IN_HEX_DIGIT != 0 &&
+ properties.mantissa > 0) {
+ properties.exponent -= properties.fraction_bits % BITS_IN_HEX_DIGIT;
}
// This is the max number of digits it can take to represent the mantissa.
@@ -93,10 +114,10 @@ LIBC_INLINE int convert_float_hex_exp(Writer<write_mode> *writer,
// for the extra implicit bit. We use the larger of the two possible values
// since the size must be constant.
constexpr size_t MANT_BUFF_LEN =
- (LDBits::FRACTION_LEN / BITS_IN_HEX_DIGIT) + 1;
+ (MAX_POSSIBLE_FRACTION_LEN / BITS_IN_HEX_DIGIT) + 1;
char mant_buffer[MANT_BUFF_LEN];
- size_t mant_len = (fraction_bits / BITS_IN_HEX_DIGIT) + 1;
+ size_t mant_len = (properties.fraction_bits / BITS_IN_HEX_DIGIT) + 1;
// Precision only tracks the number of digits after the hexadecimal point, so
// we have to add one to account for the digit before the hexadecimal point.
@@ -106,11 +127,12 @@ LIBC_INLINE int convert_float_hex_exp(Writer<write_mode> *writer,
const size_t shift_amount =
(mant_len - intended_digits) * BITS_IN_HEX_DIGIT;
- const StorageType truncated_bits =
- mantissa & ((StorageType(1) << shift_amount) - 1);
- const StorageType halfway_const = StorageType(1) << (shift_amount - 1);
+ const AnyFloatStorageType truncated_bits =
+ properties.mantissa & ((AnyFloatStorageType(1) << shift_amount) - 1);
+ const AnyFloatStorageType halfway_const = AnyFloatStorageType(1)
+ << (shift_amount - 1);
- mantissa >>= shift_amount;
+ properties.mantissa >>= shift_amount;
#ifdef LIBC_MATH_HAS_ASSUME_ROUND_NEAREST_ONLY
// Round to nearest, if it's exactly halfway then round to even.
@@ -123,17 +145,17 @@ LIBC_INLINE int convert_float_hex_exp(Writer<write_mode> *writer,
case FE_TONEAREST:
// Round to nearest, if it's exactly halfway then round to even.
if (truncated_bits > halfway_const)
- ++mantissa;
+ ++properties.mantissa;
else if (truncated_bits == halfway_const)
- mantissa = mantissa + (mantissa & 1);
+ properties.mantissa = properties.mantissa + (properties.mantissa & 1);
break;
case FE_DOWNWARD:
- if (truncated_bits > 0 && is_negative)
- ++mantissa;
+ if (truncated_bits > 0 && properties.is_negative)
+ ++properties.mantissa;
break;
case FE_UPWARD:
- if (truncated_bits > 0 && !is_negative)
- ++mantissa;
+ if (truncated_bits > 0 && !properties.is_negative)
+ ++properties.mantissa;
break;
case FE_TOWARDZERO:
break;
@@ -142,9 +164,10 @@ LIBC_INLINE int convert_float_hex_exp(Writer<write_mode> *writer,
// If the rounding caused an overflow, shift the mantissa and adjust the
// exponent to match.
- if (mantissa >= (StorageType(1) << (intended_digits * BITS_IN_HEX_DIGIT))) {
- mantissa >>= BITS_IN_HEX_DIGIT;
- exponent += BITS_IN_HEX_DIGIT;
+ if (properties.mantissa >=
+ (AnyFloatStorageType(1) << (intended_digits * BITS_IN_HEX_DIGIT))) {
+ properties.mantissa >>= BITS_IN_HEX_DIGIT;
+ properties.exponent += BITS_IN_HEX_DIGIT;
}
mant_len = intended_digits;
@@ -152,8 +175,8 @@ LIBC_INLINE int convert_float_hex_exp(Writer<write_mode> *writer,
size_t mant_cur = mant_len;
size_t first_non_zero = 1;
- for (; mant_cur > 0; --mant_cur, mantissa >>= 4) {
- char mant_mod_16 = static_cast<char>(mantissa % 16);
+ for (; mant_cur > 0; --mant_cur, properties.mantissa >>= 4) {
+ char mant_mod_16 = static_cast<char>(properties.mantissa % 16);
char new_digit = internal::int_to_b36_char(mant_mod_16);
if (internal::isupper(to_conv.conv_name))
new_digit = internal::toupper(new_digit);
@@ -173,18 +196,19 @@ LIBC_INLINE int convert_float_hex_exp(Writer<write_mode> *writer,
// 15 -> 5
// 11 -> 4
// 8 -> 3
- constexpr size_t EXP_LEN = (((LDBits::EXP_LEN * 5) + 15) / 16) + 1;
+ constexpr size_t EXP_LEN = (((MAX_POSSIBLE_EXP_LEN * 5) + 15) / 16) + 1;
char exp_buffer[EXP_LEN];
bool exp_is_negative = false;
- if (exponent < 0) {
+ if (properties.exponent < 0) {
exp_is_negative = true;
- exponent = -exponent;
+ properties.exponent = -properties.exponent;
}
size_t exp_cur = EXP_LEN;
- for (; exponent > 0; --exp_cur, exponent /= 10) {
- exp_buffer[exp_cur - 1] = internal::int_to_b36_char(exponent % 10);
+ for (; properties.exponent > 0; --exp_cur, properties.exponent /= 10) {
+ exp_buffer[exp_cur - 1] =
+ internal::int_to_b36_char(properties.exponent % 10);
}
if (exp_cur == EXP_LEN) { // if nothing else was written, write a 0.
exp_buffer[EXP_LEN - 1] = '0';
diff --git a/libc/src/stdio/printf_core/float_inf_nan_converter.h b/libc/src/stdio/printf_core/float_inf_nan_converter.h
index 973199d55094b..2af2c7feee336 100644
--- a/libc/src/stdio/printf_core/float_inf_nan_converter.h
+++ b/libc/src/stdio/printf_core/float_inf_nan_converter.h
@@ -22,38 +22,46 @@
namespace LIBC_NAMESPACE_DECL {
namespace printf_core {
+struct InfNanFPBitsProperties {
+ bool is_negative;
+ bool mantissa_is_zero;
+};
+
+template <typename T>
+InfNanFPBitsProperties
+get_inf_nan_fp_bits_properties(AnyFloatStorageType float_raw) {
+ fputil::FPBits<T> float_bits(
+ static_cast<typename fputil::FPBits<T>::StorageType>(float_raw));
+ return {
+ .is_negative = float_bits.is_neg(),
+ .mantissa_is_zero = float_bits.get_mantissa() == 0,
+ };
+}
+
template <WriteMode write_mode>
LIBC_INLINE int convert_inf_nan(Writer<write_mode> *writer,
const FormatSection &to_conv) {
-#ifdef LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE
- using StorageType = UInt128;
-#else
- using StorageType = fputil::FPBits<long double>::StorageType;
-#endif // LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE
-
// All of the letters will be defined relative to variable a, which will be
// the appropriate case based on the case of the conversion.
- bool is_negative;
- StorageType mantissa;
+ InfNanFPBitsProperties properties;
+#if defined(LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128)
+ if (to_conv.length_modifier == LengthModifier::Q) {
+ properties = get_inf_nan_fp_bits_properties<float128>(to_conv.conv_val_raw);
+ } else
+#endif // LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128
#ifndef LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE
- if (to_conv.length_modifier == LengthModifier::L) {
- fputil::FPBits<long double>::StorageType float_raw = to_conv.conv_val_raw;
- fputil::FPBits<long double> float_bits(float_raw);
- is_negative = float_bits.is_neg();
- mantissa = float_bits.get_mantissa();
+ if (to_conv.length_modifier == LengthModifier::L) {
+ properties =
+ get_inf_nan_fp_bits_properties<long double>(to_conv.conv_val_raw);
} else
#endif // !LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE
{
- fputil::FPBits<double>::StorageType float_raw =
- static_cast<fputil::FPBits<double>::StorageType>(to_conv.conv_val_raw);
- fputil::FPBits<double> float_bits(float_raw);
- is_negative = float_bits.is_neg();
- mantissa = float_bits.get_mantissa();
+ properties = get_inf_nan_fp_bits_properties<double>(to_conv.conv_val_raw);
}
char sign_char = 0;
- if (is_negative)
+ if (properties.is_negative)
sign_char = '-';
else if ((to_conv.flags & FormatFlags::FORCE_SIGN) == FormatFlags::FORCE_SIGN)
sign_char = '+'; // FORCE_SIGN has precedence over SPACE_PREFIX
@@ -73,7 +81,7 @@ LIBC_INLINE int convert_inf_nan(Writer<write_mode> *writer,
if (sign_char)
RET_IF_RESULT_NEGATIVE(writer->write(sign_char));
- if (mantissa == 0) { // inf
+ if (properties.mantissa_is_zero) { // inf
RET_IF_RESULT_NEGATIVE(
writer->write(internal::islower(to_conv.conv_name) ? "inf" : "INF"));
} else { // nan
diff --git a/libc/src/stdio/printf_core/parser.h b/libc/src/stdio/printf_core/parser.h
index ed24250491517..369358882b621 100644
--- a/libc/src/stdio/printf_core/parser.h
+++ b/libc/src/stdio/printf_core/parser.h
@@ -35,21 +35,25 @@
namespace LIBC_NAMESPACE_DECL {
namespace printf_core {
-template <typename T> struct int_type_of {
+template <typename T, typename U = void> struct int_type_of {
using type = T;
};
-template <> struct int_type_of<double> {
- using type = fputil::FPBits<double>::StorageType;
-};
-#ifndef LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE
-template <> struct int_type_of<long double> {
- using type = fputil::FPBits<long double>::StorageType;
-};
+
+template <typename T>
+struct int_type_of<T, cpp::enable_if_t<cpp::is_same_v<T, double>
+#if !defined(LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE)
+ || cpp::is_same_v<T, long double>
#endif // LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE
+#if defined(LIBC_TYPES_HAS_FLOAT128)
+ || cpp::is_same_v<T, float128>
+#endif // LIBC_TYPES_HAS_FLOAT128
+ >> {
+ using type = typename fputil::FPBits<T>::StorageType;
+};
#ifdef LIBC_INTERNAL_PRINTF_HAS_FIXED_POINT
template <typename T>
-struct int_type_of<cpp::enable_if<cpp::is_fixed_point_v<T>, T>> {
+struct int_type_of<T, cpp::enable_if<cpp::is_fixed_point_v<T>>> {
using type = typename fixed_point::FXRep<T>::StorageType;
};
#endif // LIBC_INTERNAL_PRINTF_HAS_FIXED_POINT
@@ -243,6 +247,11 @@ template <typename ArgProvider> class Parser {
}
break;
#endif // LIBC_COPT_PRINTF_DISABLE_BITINT
+#if defined(LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128)
+ case (LengthModifier::Q):
+ section.has_conv = false;
+ break;
+#endif // LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128
}
break;
#ifndef LIBC_COPT_PRINTF_DISABLE_FLOAT
@@ -254,12 +263,20 @@ template <typename ArgProvider> class Parser {
case ('A'):
case ('g'):
case ('G'):
- if (lm != LengthModifier::L) {
- WRITE_ARG_VAL_SIMPLEST(section.conv_val_raw, double, conv_index);
- } else {
+ switch (lm) {
+#if defined(LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128)
+ case (LengthModifier::Q):
+ WRITE_ARG_VAL_SIMPLEST(section.conv_val_raw, float128, conv_index);
+ break;
+#endif // LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128
#ifndef LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE
+ case (LengthModifier::L):
WRITE_ARG_VAL_SIMPLEST(section.conv_val_raw, long double, conv_index);
+ break;
#endif // !LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE
+ default:
+ WRITE_ARG_VAL_SIMPLEST(section.conv_val_raw, double, conv_index);
+ break;
}
break;
#endif // LIBC_COPT_PRINTF_DISABLE_FLOAT
@@ -396,6 +413,11 @@ template <typename ArgProvider> class Parser {
case ('L'):
++*local_pos;
return {LengthModifier::L, 0};
+#if defined(LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128)
+ case ('Q'):
+ ++*local_pos;
+ return {LengthModifier::Q, 0};
+#endif // LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128
case ('j'):
++*local_pos;
return {LengthModifier::j, 0};
@@ -654,6 +676,11 @@ template <typename ArgProvider> class Parser {
}
break;
#endif // LIBC_COPT_PRINTF_DISABLE_BITINT
+#if defined(LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128)
+ case (LengthModifier::Q):
+ conv_size = type_desc_from_type<void>();
+ break;
+#endif // LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128
}
break;
#ifndef LIBC_COPT_PRINTF_DISABLE_FLOAT
@@ -665,12 +692,21 @@ template <typename ArgProvider> class Parser {
case ('A'):
case ('g'):
case ('G'):
- if (lm != LengthModifier::L)
- conv_size = type_desc_from_type<double>();
+ switch (lm) {
+#if defined(LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128)
+ case LengthModifier::Q:
+ conv_size = type_desc_from_type<float128>();
+ break;
+#endif // LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128
#ifndef LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE
- else
+ case LengthModifier::L:
conv_size = type_desc_from_type<long double>();
+ break;
#endif // !LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE
+ default:
+ conv_size = type_desc_from_type<double>();
+ break;
+ }
break;
#endif // LIBC_COPT_PRINTF_DISABLE_FLOAT
#ifdef LIBC_INTERNAL_PRINTF_HAS_FIXED_POINT
diff --git a/libc/src/stdio/printf_core/printf_config.h b/libc/src/stdio/printf_core/printf_config.h
index 5be382a34e619..929b1c5b3931f 100644
--- a/libc/src/stdio/printf_core/printf_config.h
+++ b/libc/src/stdio/printf_core/printf_config.h
@@ -36,6 +36,11 @@
#define LIBC_INTERNAL_PRINTF_HAS_FIXED_POINT
#endif
+#if defined(LIBC_TYPES_HAS_FLOAT128) && \
+ !defined(LIBC_COPT_PRINTF_NO_CONVERT_FLOAT128)
+#define LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128
+#endif
+
// TODO(michaelrj): Provide a proper interface for these options.
// LIBC_COPT_FLOAT_TO_STR_USE_MEGA_LONG_DOUBLE_TABLE
// LIBC_COPT_FLOAT_TO_STR_USE_DYADIC_FLOAT
diff --git a/libc/src/stdio/printf_core/write_int_converter.h b/libc/src/stdio/printf_core/write_int_converter.h
index 0c7efedb5f3bf..2844a56dc9922 100644
--- a/libc/src/stdio/printf_core/write_int_converter.h
+++ b/libc/src/stdio/printf_core/write_int_converter.h
@@ -63,6 +63,11 @@ LIBC_INLINE int convert_write_int(Writer<write_mode> *writer,
#endif // LIBC_COPT_PRINTF_DISABLE_BITINT
*reinterpret_cast<uintmax_t *>(to_conv.conv_val_ptr) = written;
break;
+#if defined(LIBC_TYPES_HAS_FLOAT128)
+ case (LengthModifier::Q): // 'Q' is not valid for integer format; this case
+ // should not be reachable.
+ break;
+#endif // LIBC_TYPES_HAS_FLOAT128
}
return WRITE_OK;
}
diff --git a/libc/test/UnitTest/CMakeLists.txt b/libc/test/UnitTest/CMakeLists.txt
index aa245b2234793..02489107efc06 100644
--- a/libc/test/UnitTest/CMakeLists.txt
+++ b/libc/test/UnitTest/CMakeLists.txt
@@ -184,6 +184,7 @@ add_unittest_framework_library(
libc.hdr.stdint_proxy
libc.src.__support.FPUtil.fp_bits
libc.src.stdio.printf_core.core_structs
+ libc.src.stdio.printf_core.printf_config
libc.test.UnitTest.string_utils
)
diff --git a/libc/test/UnitTest/PrintfMatcher.cpp b/libc/test/UnitTest/PrintfMatcher.cpp
index 3a940aad8e435..886737f0dcb06 100644
--- a/libc/test/UnitTest/PrintfMatcher.cpp
+++ b/libc/test/UnitTest/PrintfMatcher.cpp
@@ -12,6 +12,7 @@
#include "src/__support/FPUtil/FPBits.h"
#include "src/__support/macros/config.h"
#include "src/stdio/printf_core/core_structs.h"
+#include "src/stdio/printf_core/printf_config.h"
#include "test/UnitTest/StringUtils.h"
#include "test/UnitTest/Test.h"
@@ -71,6 +72,9 @@ static void display(FormatSection form) {
CASE_LM(z);
CASE_LM(t);
CASE_LM(L);
+#if defined(LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128)
+ CASE_LM(Q);
+#endif // LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128
#ifndef LIBC_COPT_PRINTF_DISABLE_BITINT
CASE_LM_BIT_WIDTH(w, form.bit_width);
CASE_LM_BIT_WIDTH(wf, form.bit_width);
@@ -84,10 +88,7 @@ static void display(FormatSection form) {
reinterpret_cast<uintptr_t>(form.conv_val_ptr))
<< "\n";
else if (form.conv_name != '%')
- tlog << "\tvalue: "
- << int_to_hex<fputil::FPBits<long double>::StorageType>(
- form.conv_val_raw)
- << "\n";
+ tlog << "\tvalue: " << int_to_hex(form.conv_val_raw) << "\n";
}
}
} // anonymous namespace
diff --git a/libc/test/src/stdio/CMakeLists.txt b/libc/test/src/stdio/CMakeLists.txt
index 9737dcba7f904..5f586238dc1e9 100644
--- a/libc/test/src/stdio/CMakeLists.txt
+++ b/libc/test/src/stdio/CMakeLists.txt
@@ -173,6 +173,7 @@ add_fp_unittest(
SRCS
sprintf_test.cpp
DEPENDS
+ libc.src.stdio.printf_core.printf_config
libc.src.stdio.sprintf
libc.src.__support.FPUtil.fp_bits
libc.include.inttypes
diff --git a/libc/test/src/stdio/sprintf_test.cpp b/libc/test/src/stdio/sprintf_test.cpp
index 259bd53e2fd42..3c0e9c74d0c06 100644
--- a/libc/test/src/stdio/sprintf_test.cpp
+++ b/libc/test/src/stdio/sprintf_test.cpp
@@ -7,6 +7,7 @@
//===----------------------------------------------------------------------===//
#include "src/__support/macros/config.h"
+#include "src/stdio/printf_core/printf_config.h"
#include "src/stdio/sprintf.h"
#ifndef LIBC_COPT_PRINTF_DISABLE_WIDE
@@ -55,6 +56,44 @@ TEST(LlvmLibcSPrintfTest, Macros) {
macro_test(PRIo64, 0123456712345671234567ll, "123456712345671234567");
}
+#if defined(LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128)
+
+// Returns a non-negative float128 value:
+//
+// Result = (explicit_mantissa_bits_high << 64) | explicit_mantissa_bits_low) *
+// 2^(exponent - 112)
+//
+// This is a convenience so test values can be expressed with hex literals that
+// map directly to how they could appear as a combined native floating-point
+// literal. The high 15 bits of explicit_mantissa_bits_high should be zero.
+float128 make_f128(uint64_t explicit_mantissa_bits_high,
+ uint64_t explicit_mantissa_bits_low, int exponent) {
+ if (explicit_mantissa_bits_high == 0 && explicit_mantissa_bits_low == 0)
+ return float128(0);
+
+ using BitsType = LIBC_NAMESPACE::fputil::FPBits<float128>;
+ UInt128 mantissa = (UInt128(explicit_mantissa_bits_high) << 64) |
+ UInt128(explicit_mantissa_bits_low);
+ // exponent += LIBC_NAMESPACE::cpp::countl_zero(mantissa) -
+ // (BitsType::STORAGE_LEN - BitsType::FRACTION_LEN);
+ int biased_exponent = exponent - 1 + BitsType::EXP_BIAS;
+ return BitsType::make_value(mantissa, biased_exponent).get_val();
+}
+
+TEST(LlvmLibcSPrintfTest, MakeF128Helper) {
+ EXPECT_TRUE(make_f128(0, 0, 0) == float128(0));
+ EXPECT_TRUE(make_f128(0, 1, 0) == float128(0x1p-112));
+ EXPECT_TRUE(make_f128(0x1'0000'0000'0000, 0, 0) == float128(1));
+ EXPECT_TRUE(make_f128(0x1'FF00'0000'0000, 0, 3) ==
+ float128((2.0 - 1.0 / 256)) * 8.0);
+#if defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128)
+ EXPECT_TRUE(make_f128(0x1'2345'6789'0abc, 0x0123'4567'89ab'cdef, -1234) ==
+ 0x1.2345'6789'0abc'0123'4567'89ab'cdefp-1234L);
+#endif // LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128
+}
+
+#endif // LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128
+
TEST(LlvmLibcSPrintfTest, SimpleNoConv) {
char buff[64];
int written;
@@ -725,6 +764,10 @@ TEST(LlvmLibcSPrintfTest, FloatHexExpConv) {
#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128)
ASSERT_STREQ_LEN(written, buff, "0x1.999999999999999999999999999ap-4");
#endif
+#if defined(LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128)
+ written = LIBC_NAMESPACE::sprintf(buff, "%Qa", float128(1) / float128(10));
+ ASSERT_STREQ_LEN(written, buff, "0x1.999999999999999999999999999ap-4");
+#endif // LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128
written = LIBC_NAMESPACE::sprintf(buff, "%La", 1.0e1000L);
#if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80)
@@ -734,6 +777,12 @@ TEST(LlvmLibcSPrintfTest, FloatHexExpConv) {
#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128)
ASSERT_STREQ_LEN(written, buff, "0x1.e71b63f3ba7b580af1a52d2a7379p+3321");
#endif
+#if defined(LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128) && \
+ defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128)
+ written = LIBC_NAMESPACE::sprintf(buff, "%Qa", 1.0e1000L);
+ ASSERT_STREQ_LEN(written, buff, "0x1.e71b63f3ba7b580af1a52d2a7379p+3321");
+#endif // LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128 &&
+ // LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128
written = LIBC_NAMESPACE::sprintf(buff, "%La", 1.0e-1000L);
#if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80)
@@ -743,6 +792,12 @@ TEST(LlvmLibcSPrintfTest, FloatHexExpConv) {
#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128)
ASSERT_STREQ_LEN(written, buff, "0x1.0d152311513c28ce202627c06ec2p-3322");
#endif
+#if defined(LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128) && \
+ defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128)
+ written = LIBC_NAMESPACE::sprintf(buff, "%Qa", 1.0e-1000L);
+ ASSERT_STREQ_LEN(written, buff, "0x1.0d152311513c28ce202627c06ec2p-3322");
+#endif // LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128 &&
+ // LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128
// Min Width Tests.
@@ -849,6 +904,10 @@ TEST(LlvmLibcSPrintfTest, FloatHexExpConv) {
#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128)
ASSERT_STREQ_LEN(written, buff, "0x1.ap-4");
#endif
+#if defined(LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128)
+ written = LIBC_NAMESPACE::sprintf(buff, "%.1Qa", float128(0.1));
+ ASSERT_STREQ_LEN(written, buff, "0x1.ap-4");
+#endif // LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128
written = LIBC_NAMESPACE::sprintf(buff, "%.1La", 0xf.fffffffffffffffp16380L);
#if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80)
@@ -858,6 +917,11 @@ TEST(LlvmLibcSPrintfTest, FloatHexExpConv) {
#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128)
ASSERT_STREQ_LEN(written, buff, "0x2.0p+16383");
#endif
+#if defined(LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128)
+ written = LIBC_NAMESPACE::sprintf(buff, "%.1Qa",
+ float128(0xf.fffffffffffffffp16380L));
+ ASSERT_STREQ_LEN(written, buff, "0x2.0p+16383");
+#endif // LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128
// Rounding Mode Tests.
@@ -1034,6 +1098,11 @@ TEST(LlvmLibcSPrintfTest, FloatDecimalConv) {
LIBC_NAMESPACE::fputil::FPBits<long double>::inf().get_val();
long double ld_nan =
LIBC_NAMESPACE::fputil::FPBits<long double>::quiet_nan().get_val();
+#if defined(LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128)
+ float128 f128_inf = LIBC_NAMESPACE::fputil::FPBits<float128>::inf().get_val();
+ float128 f128_nan =
+ LIBC_NAMESPACE::fputil::FPBits<float128>::quiet_nan().get_val();
+#endif // LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128
written = LIBC_NAMESPACE::sprintf(buff, "%f", 1.0);
ASSERT_STREQ_LEN(written, buff, "1.000000");
@@ -1109,6 +1178,26 @@ TEST(LlvmLibcSPrintfTest, FloatDecimalConv) {
ASSERT_STREQ_LEN(written, buff, "-NAN");
#endif
+#if defined(LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128)
+ written = LIBC_NAMESPACE::sprintf(buff, "%Qf", f128_inf);
+ ASSERT_STREQ_LEN(written, buff, "inf");
+
+ written = LIBC_NAMESPACE::sprintf(buff, "%Qf", -f128_inf);
+ ASSERT_STREQ_LEN(written, buff, "-inf");
+
+ written = LIBC_NAMESPACE::sprintf(buff, "%QF", f128_inf);
+ ASSERT_STREQ_LEN(written, buff, "INF");
+
+ written = LIBC_NAMESPACE::sprintf(buff, "%QF", -f128_inf);
+ ASSERT_STREQ_LEN(written, buff, "-INF");
+
+ written = LIBC_NAMESPACE::sprintf(buff, "%Qf", f128_nan);
+ ASSERT_STREQ_LEN(written, buff, "nan");
+
+ written = LIBC_NAMESPACE::sprintf(buff, "%QF", f128_nan);
+ ASSERT_STREQ_LEN(written, buff, "NAN");
+#endif // LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128
+
// Min Width Tests.
written = LIBC_NAMESPACE::sprintf(buff, "%15f", 1.0);
@@ -1986,6 +2075,145 @@ TEST(LlvmLibcSPrintfTest, FloatDecimalLongDoubleConv) {
#endif // LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128
}
+#if defined(LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128)
+TEST(LlvmLibcSPrintfTest, FloatDecimalFloat128Conv) {
+ char buff[1000];
+ int written;
+
+ ForceRoundingMode r(RoundingMode::Nearest);
+
+ written = LIBC_NAMESPACE::sprintf(buff, "%Qf", float128(1.0L));
+ ASSERT_STREQ_LEN(written, buff, "1.000000");
+
+ written = LIBC_NAMESPACE::sprintf(buff, "%.Qf", float128(-2.5L));
+ ASSERT_STREQ_LEN(written, buff, "-2");
+
+ // Some exceptionally difficult cases for 39-digit precision. (That's the
+ // number of digits supported by the float320 algorithm, and should still be
+ // correct under other algorithms. So these are still enabled even under
+ // LIBC_COPT_FLOAT_TO_STR_REDUCED_PRECISION.)
+ //
+ // These were found by number-theoretic search to be the worst cases in terms
+ // of being extremely close to the rounding boundary between two possible
+ // decimal outputs. For example, the first of these cases has a true value
+ // beginning with
+ //
+ // 2.245786964418815522831613614422112838795000000000000000000
+ // 0000000000000000010767969...
+ //
+ // so you need to compute a _long_ way past the 39th digit to find out
+ // whether to round the ...8795 up to 880 or not!
+ //
+ // The first half of these cases all rounded up; the second half all rounded
+ // down. You can see that in both sections the final decimal digit is
+ // sometimes odd and sometimes even, ruling out the possibility that we're
+ // getting these right by mistakenly assuming them to be _exactly_ on the
+ // boundary.
+
+ written = LIBC_NAMESPACE::sprintf(
+ buff, "%#.39Qg",
+ make_f128(0x1'bde5'716b'ba8d, 0x7025'5b4b'e10e'0a0a, -3388));
+ ASSERT_STREQ_LEN(written, buff,
+ "2.24578696441881552283161361442211283880e-1020");
+ written = LIBC_NAMESPACE::sprintf(
+ buff, "%#.39Qg",
+ make_f128(0x1'64b7'8def'c871, 0x2684'4909'80d8'0808, -3391));
+ ASSERT_STREQ_LEN(written, buff,
+ "2.24578696441881552283161361442211283880e-1021");
+ written = LIBC_NAMESPACE::sprintf(
+ buff, "%#.39Qg",
+ make_f128(0x1'c0ee'd9d1'ea4b, 0x6f21'5acc'b15c'b42c, -4714));
+ ASSERT_STREQ_LEN(written, buff,
+ "1.54362575487963943466308346767014523161e-1419");
+ written = LIBC_NAMESPACE::sprintf(
+ buff, "%#.39Qg",
+ make_f128(0x1'a393'eaaf'c3db, 0xabfc'd304'42ce'f525, -12600));
+ ASSERT_STREQ_LEN(written, buff,
+ "1.72435694193924008931441874634575361189e-3793");
+ written = LIBC_NAMESPACE::sprintf(
+ buff, "%#.39Qg",
+ make_f128(0x1'b7a5'248b'af85, 0x133c'9b7b'f324'1f75, 11050));
+ ASSERT_STREQ_LEN(written, buff,
+ "4.13346579244549095104252956440178208514e+3326");
+ written = LIBC_NAMESPACE::sprintf(
+ buff, "%#.39Qg",
+ make_f128(0x1'81bf'6d97'7f99, 0xff7b'd9de'bdd5'2815, 1359));
+ ASSERT_STREQ_LEN(written, buff,
+ "1.89595297593127274811683259716608232064e+409");
+ written = LIBC_NAMESPACE::sprintf(
+ buff, "%#.39Qg",
+ make_f128(0x1'fcb9'cfd6'5f06, 0x8b75'8d30'ba19'a494, -4451));
+ ASSERT_STREQ_LEN(written, buff,
+ "2.59258570015527007681686041122929728653e-1340");
+ written = LIBC_NAMESPACE::sprintf(
+ buff, "%#.39Qg",
+ make_f128(0x1'7d8b'5be0'c744, 0xe898'29e4'8b93'3b6f, -4448));
+ ASSERT_STREQ_LEN(written, buff,
+ "1.55555142009316204609011624673757837192e-1339");
+ written = LIBC_NAMESPACE::sprintf(
+ buff, "%#.39Qg",
+ make_f128(0x1'dbb1'f01a'ef7b, 0x93c3'7ca0'0217'888c, -12205));
+ ASSERT_STREQ_LEN(written, buff,
+ "1.57758077908296078543773740016012372216e-3674");
+ written = LIBC_NAMESPACE::sprintf(
+ buff, "%#.39Qg",
+ make_f128(0x1'fb03'7d72'fdf1, 0xa8aa'8bdf'c258'6fe3, 2580));
+ ASSERT_STREQ_LEN(written, buff,
+ "8.99846610004600287527755301065037553046e+776");
+
+ written = LIBC_NAMESPACE::sprintf(
+ buff, "%#.39Qg",
+ make_f128(0x1'9af8'fe5f'ebf7, 0x51a7'9529'2e30'335d, 2052));
+ ASSERT_STREQ_LEN(written, buff,
+ "8.30087814106622071390265113980150545241e+617");
+ written = LIBC_NAMESPACE::sprintf(
+ buff, "%#.39Qg",
+ make_f128(0x1'48c7'31e6'565f, 0x7486'10ed'be8c'f5e4, 2049));
+ ASSERT_STREQ_LEN(written, buff,
+ "8.30087814106622071390265113980150545241e+616");
+ written = LIBC_NAMESPACE::sprintf(
+ buff, "%#.39Qg",
+ make_f128(0x1'8e78'bc76'e989, 0x98b7'bbf6'c8f8'0f2a, -5671));
+ ASSERT_STREQ_LEN(written, buff,
+ "1.12473970318904704063044350553302771341e-1707");
+ written = LIBC_NAMESPACE::sprintf(
+ buff, "%#.39Qg",
+ make_f128(0x1'59a8'32c1'6a77, 0x97ae'c701'96dd'f9db, -16181));
+ ASSERT_STREQ_LEN(written, buff,
+ "1.45896738321434823250358135932839533040e-4871");
+ written = LIBC_NAMESPACE::sprintf(
+ buff, "%#.39Qg",
+ make_f128(0x1'1486'8f01'21f9, 0x4625'6c01'457e'617c, -16184));
+ ASSERT_STREQ_LEN(written, buff,
+ "1.45896738321434823250358135932839533040e-4872");
+ written = LIBC_NAMESPACE::sprintf(
+ buff, "%#.39Qg",
+ make_f128(0x1'0ca8'c4b5'25b1, 0x1285'06cd'c668'df43, 11017));
+ ASSERT_STREQ_LEN(written, buff,
+ "2.94051951004764266903705588743096762647e+3316");
+ written = LIBC_NAMESPACE::sprintf(
+ buff, "%#.39Qg",
+ make_f128(0x1'fcd4'0bb4'9b18, 0xaa19'd66a'3c55'72de, 5547));
+ ASSERT_STREQ_LEN(written, buff,
+ "1.29335350323078956384272678475060580129e+1670");
+ written = LIBC_NAMESPACE::sprintf(
+ buff, "%#.39Qg",
+ make_f128(0x1'0e5e'5151'0a65, 0x3e74'4d6b'84ef'ed86, -13613));
+ ASSERT_STREQ_LEN(written, buff,
+ "1.26585813611514592337442314391432904094e-4098");
+ written = LIBC_NAMESPACE::sprintf(
+ buff, "%#.39Qg",
+ make_f128(0x1'ff41'f8aa'97e6, 0x76e9'5b1a'6a77'51fa, 16366));
+ ASSERT_STREQ_LEN(written, buff,
+ "9.06377119787295161934827646572467920486e+4926");
+ written = LIBC_NAMESPACE::sprintf(
+ buff, "%#.39Qg",
+ make_f128(0x1'4770'bf66'ccaf, 0xd7d8'0ac9'bb3d'ded7, -1843));
+ ASSERT_STREQ_LEN(written, buff,
+ "2.03521509642091239545213340619368190283e-555");
+}
+#endif // LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128
+
TEST(LlvmLibcSPrintfTest, FloatExponentConv) {
char buff[1000];
int written;
@@ -2507,6 +2735,56 @@ TEST(LlvmLibcSPrintfTest, FloatExponentLongDoubleConv) {
#endif
}
+#if defined(LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128)
+TEST(LlvmLibcSPrintfTest, FloatExponentFloat128Conv) {
+ char buff[1000];
+ int written;
+
+ ForceRoundingMode r(RoundingMode::Nearest);
+
+ written =
+ LIBC_NAMESPACE::sprintf(buff, "%.9Qe", float128(1000000000500000000.1L));
+ ASSERT_STREQ_LEN(written, buff, "1.000000001e+18");
+
+ written =
+ LIBC_NAMESPACE::sprintf(buff, "%.9Qe", float128(1000000000500000000.0L));
+ ASSERT_STREQ_LEN(written, buff, "1.000000000e+18");
+
+ written = LIBC_NAMESPACE::sprintf(buff, "%Qe", float128(1e100L));
+ ASSERT_STREQ_LEN(written, buff, "1.000000e+100");
+
+ written = LIBC_NAMESPACE::sprintf(buff, "%Qe", float128(1.0L));
+ ASSERT_STREQ_LEN(written, buff, "1.000000e+00");
+
+#if !defined(LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE)
+ written = LIBC_NAMESPACE::sprintf(buff, "%Qe",
+ float128(0xf.fffffffffffffffp+16380L));
+ ASSERT_STREQ_LEN(written, buff, "1.189731e+4932");
+
+ written = LIBC_NAMESPACE::sprintf(buff, "%Qe", float128(1e1000L));
+ ASSERT_STREQ_LEN(written, buff, "1.000000e+1000");
+
+ written = LIBC_NAMESPACE::sprintf(buff, "%Qe", float128(1e4900L));
+ ASSERT_STREQ_LEN(written, buff, "1.000000e+4900");
+
+ written = LIBC_NAMESPACE::sprintf(buff, "%Qe", float128(1.2345678e4900L));
+ ASSERT_STREQ_LEN(written, buff, "1.234568e+4900");
+
+#ifndef LIBC_COPT_FLOAT_TO_STR_REDUCED_PRECISION
+ // Minimum normal + epsilon
+ written = LIBC_NAMESPACE::sprintf(buff, "%.20Qe",
+ float128(3.36210314311209350663E-4932L));
+ ASSERT_STREQ_LEN(written, buff, "3.36210314311209350663e-4932");
+
+ // Minimum subnormal
+ written = LIBC_NAMESPACE::sprintf(buff, "%.20Qe",
+ float128(3.64519953188247460253E-4951L));
+ ASSERT_STREQ_LEN(written, buff, "3.64519953188247460253e-4951");
+#endif // LIBC_COPT_FLOAT_TO_STR_REDUCED_PRECISION
+#endif // !LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE
+}
+#endif // LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128
+
TEST(LlvmLibcSPrintfTest, FloatAutoConv) {
char buff[1000];
int written;
@@ -3040,6 +3318,33 @@ TEST(LlvmLibcSPrintfTest, FloatAutoLongDoubleConv) {
ASSERT_STREQ_LEN(written, big_buff, "1e+4900");
}
+#if defined(LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128)
+TEST(LlvmLibcSPrintfTest, FloatAutoFloat128Conv) {
+ char buff[1000];
+ int written;
+
+ ForceRoundingMode r(RoundingMode::Nearest);
+
+ written = LIBC_NAMESPACE::sprintf(buff, "%Qg", float128(9.99999999999e-100L));
+ ASSERT_STREQ_LEN(written, buff, "1e-99");
+
+ written = LIBC_NAMESPACE::sprintf(buff, "%Qg", float128(1e100L));
+ ASSERT_STREQ_LEN(written, buff, "1e+100");
+
+ written = LIBC_NAMESPACE::sprintf(buff, "%Qg", float128(1.0L));
+ ASSERT_STREQ_LEN(written, buff, "1");
+
+ written = LIBC_NAMESPACE::sprintf(buff, "%Qg", float128(0.1L));
+ ASSERT_STREQ_LEN(written, buff, "0.1");
+
+ written = LIBC_NAMESPACE::sprintf(buff, "%Qg", float128(1e1000L));
+ ASSERT_STREQ_LEN(written, big_buff, "1e+1000");
+
+ written = LIBC_NAMESPACE::sprintf(buff, "%Qg", float128(1e4900L));
+ ASSERT_STREQ_LEN(written, big_buff, "1e+4900");
+}
+#endif // LIBC_INTERNAL_PRINTF_CONVERT_FLOAT128
+
#endif // LIBC_COPT_PRINTF_DISABLE_FLOAT
#if defined(LIBC_COMPILER_HAS_FIXED_POINT) && \
>From a4107d67f91946a97d0acd0a70e2ae16513858b9 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2 at gmail.com>
Date: Mon, 6 Jul 2026 14:19:53 -0400
Subject: [PATCH 34/46] [libc++] Don't install custom Python on self-hosted
macOS runners (#207794)
That shouldn't be necessary, since they already come with Python 3, and
it doesn't work anyway because it requires special setup that the macOS
runners don't currently perform.
---
.github/workflows/libcxx-run-benchmarks.yml | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/libcxx-run-benchmarks.yml b/.github/workflows/libcxx-run-benchmarks.yml
index 8529c0937af37..edb825367891a 100644
--- a/.github/workflows/libcxx-run-benchmarks.yml
+++ b/.github/workflows/libcxx-run-benchmarks.yml
@@ -98,10 +98,12 @@ jobs:
runner: ["self-hosted", "macOS", "ARM64", "26", "26.5"]
cc: clang
cxx: clang++
+ install-python: false
- platform: Linux x86_64
runner: llvm-premerge-libcxx-next-runners
cc: clang-22
cxx: clang++-22
+ install-python: true
fail-fast: false
permissions:
pull-requests: write
@@ -119,7 +121,9 @@ jobs:
fetch-depth: 0
fetch-tags: true # This job requires access to all the Git branches so it can diff against (usually) main
- - uses: actions/setup-python at a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+ - name: Install Python
+ if: ${{ matrix.install-python }}
+ uses: actions/setup-python at a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: '3.14'
>From 7b9a58e6cfa0dab49ef417f731ee7a8dc5e3aab5 Mon Sep 17 00:00:00 2001
From: Michael Jones <michaelrj at google.com>
Date: Mon, 6 Jul 2026 11:22:42 -0700
Subject: [PATCH 35/46] [libc] Implement fdopendir (#206590)
The fdopendir function takes a file descriptor opened on a directory,
then opens a DIR* on it.
Needed for #191075
Assisted-by: Automated tooling, human reviewed.
---
libc/config/linux/aarch64/entrypoints.txt | 1 +
libc/config/linux/arm/entrypoints.txt | 7 ++
libc/config/linux/riscv/entrypoints.txt | 1 +
libc/config/linux/x86_64/entrypoints.txt | 1 +
libc/hdr/CMakeLists.txt | 1 +
libc/hdr/dirent_overlay.h | 23 ++++
libc/hdr/types/CMakeLists.txt | 20 ++++
libc/hdr/types/DIR.h | 27 +++++
libc/hdr/types/struct_dirent.h | 27 +++++
libc/src/__support/File/dir.cpp | 18 +--
libc/src/__support/File/dir.h | 8 +-
libc/src/__support/File/linux/CMakeLists.txt | 7 +-
libc/src/__support/File/linux/dir.cpp | 45 +++++--
.../OSUtil/linux/stat/kernel_statx_types.h | 3 +
.../linux/syscall_wrappers/CMakeLists.txt | 12 ++
.../OSUtil/linux/syscall_wrappers/fcntl.h | 43 +++++++
libc/src/dirent/CMakeLists.txt | 15 +++
libc/src/dirent/fdopendir.cpp | 37 ++++++
libc/src/dirent/fdopendir.h | 26 ++++
libc/test/src/dirent/CMakeLists.txt | 22 ++++
libc/test/src/dirent/fdopendir_test.cpp | 113 ++++++++++++++++++
21 files changed, 439 insertions(+), 18 deletions(-)
create mode 100644 libc/hdr/dirent_overlay.h
create mode 100644 libc/hdr/types/DIR.h
create mode 100644 libc/hdr/types/struct_dirent.h
create mode 100644 libc/src/__support/OSUtil/linux/syscall_wrappers/fcntl.h
create mode 100644 libc/src/dirent/fdopendir.cpp
create mode 100644 libc/src/dirent/fdopendir.h
create mode 100644 libc/test/src/dirent/fdopendir_test.cpp
diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index c20067edca55a..c4db45ad3c50e 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -1022,6 +1022,7 @@ if(LLVM_LIBC_FULL_BUILD)
libc.src.dirent.dirfd
libc.src.dirent.opendir
libc.src.dirent.readdir
+ libc.src.dirent.fdopendir
# arpa/inet.h entrypoints
libc.src.arpa.inet.htonl
diff --git a/libc/config/linux/arm/entrypoints.txt b/libc/config/linux/arm/entrypoints.txt
index bf0897868989c..20763eec40e8b 100644
--- a/libc/config/linux/arm/entrypoints.txt
+++ b/libc/config/linux/arm/entrypoints.txt
@@ -228,6 +228,13 @@ if(LLVM_LIBC_FULL_BUILD)
libc.src.setjmp.setjmp
libc.src.setjmp.siglongjmp
libc.src.setjmp.sigsetjmp
+
+ # dirent.h entrypoints
+ libc.src.dirent.closedir
+ libc.src.dirent.dirfd
+ libc.src.dirent.opendir
+ libc.src.dirent.readdir
+ libc.src.dirent.fdopendir
)
endif()
diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt
index e71eefca04ace..a7506ddbc2705 100644
--- a/libc/config/linux/riscv/entrypoints.txt
+++ b/libc/config/linux/riscv/entrypoints.txt
@@ -1155,6 +1155,7 @@ if(LLVM_LIBC_FULL_BUILD)
libc.src.dirent.dirfd
libc.src.dirent.opendir
libc.src.dirent.readdir
+ libc.src.dirent.fdopendir
# arpa/inet.h entrypoints
libc.src.arpa.inet.htonl
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index d12f8d8c73a36..e0bf4ff52d13a 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -1226,6 +1226,7 @@ if(LLVM_LIBC_FULL_BUILD)
libc.src.dirent.dirfd
libc.src.dirent.opendir
libc.src.dirent.readdir
+ libc.src.dirent.fdopendir
# pthread.h entrypoints
libc.src.pthread.pthread_atfork
diff --git a/libc/hdr/CMakeLists.txt b/libc/hdr/CMakeLists.txt
index cf3e76d5c005b..21d88624a06da 100644
--- a/libc/hdr/CMakeLists.txt
+++ b/libc/hdr/CMakeLists.txt
@@ -126,6 +126,7 @@ add_proxy_header_library(
libc.include.llvm-libc-macros.stdlib_macros
)
+add_header_library(dirent_overlay HDRS dirent_overlay.h)
add_header_library(stdio_overlay HDRS stdio_overlay.h)
add_proxy_header_library(
diff --git a/libc/hdr/dirent_overlay.h b/libc/hdr/dirent_overlay.h
new file mode 100644
index 0000000000000..26dcace618a83
--- /dev/null
+++ b/libc/hdr/dirent_overlay.h
@@ -0,0 +1,23 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Overlay header for dirent.h in overlay build mode.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_DIRENT_OVERLAY_H
+#define LLVM_LIBC_HDR_DIRENT_OVERLAY_H
+
+#ifdef LIBC_FULL_BUILD
+#error "This header should only be included in overlay mode"
+#endif
+
+#include <dirent.h>
+
+#endif // LLVM_LIBC_HDR_DIRENT_OVERLAY_H
diff --git a/libc/hdr/types/CMakeLists.txt b/libc/hdr/types/CMakeLists.txt
index ab16836dbab92..cc985933df5ea 100644
--- a/libc/hdr/types/CMakeLists.txt
+++ b/libc/hdr/types/CMakeLists.txt
@@ -87,6 +87,16 @@ add_proxy_header_library(
libc.include.llvm-libc-types.siginfo_t
)
+add_proxy_header_library(
+ struct_dirent
+ HDRS
+ struct_dirent.h
+ DEPENDS
+ libc.hdr.dirent_overlay
+ FULL_BUILD_DEPENDS
+ libc.include.llvm-libc-types.struct_dirent
+)
+
add_proxy_header_library(
struct_epoll_event
HDRS
@@ -343,6 +353,16 @@ add_proxy_header_library(
libc.include.stdio
)
+add_proxy_header_library(
+ DIR
+ HDRS
+ DIR.h
+ DEPENDS
+ libc.hdr.dirent_overlay
+ FULL_BUILD_DEPENDS
+ libc.include.llvm-libc-types.DIR
+)
+
add_proxy_header_library(
off_t
HDRS
diff --git a/libc/hdr/types/DIR.h b/libc/hdr/types/DIR.h
new file mode 100644
index 0000000000000..27a65e0ceff05
--- /dev/null
+++ b/libc/hdr/types/DIR.h
@@ -0,0 +1,27 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Proxy for DIR.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_TYPES_DIR_H
+#define LLVM_LIBC_HDR_TYPES_DIR_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/DIR.h"
+
+#else // Overlay mode
+
+#include "hdr/dirent_overlay.h"
+
+#endif // LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_TYPES_DIR_H
diff --git a/libc/hdr/types/struct_dirent.h b/libc/hdr/types/struct_dirent.h
new file mode 100644
index 0000000000000..5e9e0bf974811
--- /dev/null
+++ b/libc/hdr/types/struct_dirent.h
@@ -0,0 +1,27 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Proxy for struct dirent.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_TYPES_STRUCT_DIRENT_H
+#define LLVM_LIBC_HDR_TYPES_STRUCT_DIRENT_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/struct_dirent.h"
+
+#else // Overlay mode
+
+#include "hdr/dirent_overlay.h"
+
+#endif // LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_TYPES_STRUCT_DIRENT_H
diff --git a/libc/src/__support/File/dir.cpp b/libc/src/__support/File/dir.cpp
index eb33656808414..76a45b3df8c0a 100644
--- a/libc/src/__support/File/dir.cpp
+++ b/libc/src/__support/File/dir.cpp
@@ -17,19 +17,23 @@
namespace LIBC_NAMESPACE_DECL {
+ErrorOr<Dir *> Dir::fdopen(int fd) {
+ LIBC_NAMESPACE::AllocChecker ac;
+ Dir *dir = new (ac) Dir(fd);
+ if (!ac)
+ return LIBC_NAMESPACE::Error(ENOMEM);
+ return dir;
+}
+
ErrorOr<Dir *> Dir::open(const char *path) {
auto fd = platform_opendir(path);
if (!fd)
return LIBC_NAMESPACE::Error(fd.error());
- LIBC_NAMESPACE::AllocChecker ac;
- Dir *dir = new (ac) Dir(fd.value());
- if (!ac)
- return LIBC_NAMESPACE::Error(ENOMEM);
- return dir;
+ return Dir::fdopen(fd.value());
}
-ErrorOr<struct ::dirent *> Dir::read() {
+ErrorOr<struct dirent *> Dir::read() {
cpp::lock_guard lock(mutex);
if (readptr >= fillsize) {
auto readsize = platform_fetch_dirents(fd, buffer);
@@ -41,7 +45,7 @@ ErrorOr<struct ::dirent *> Dir::read() {
if (fillsize == 0)
return nullptr;
- struct ::dirent *d = reinterpret_cast<struct ::dirent *>(buffer + readptr);
+ struct dirent *d = reinterpret_cast<struct dirent *>(buffer + readptr);
#ifdef __linux__
// The d_reclen field is available on Linux but not required by POSIX.
readptr += d->d_reclen;
diff --git a/libc/src/__support/File/dir.h b/libc/src/__support/File/dir.h
index 247ac1225ceea..89f67ede0ecf5 100644
--- a/libc/src/__support/File/dir.h
+++ b/libc/src/__support/File/dir.h
@@ -31,6 +31,11 @@ int platform_closedir(int fd);
// failure.
ErrorOr<size_t> platform_fetch_dirents(int fd, cpp::span<uint8_t> buffer);
+// Platform specific function which checks if |fd| is a valid file descriptor
+// open for reading, and refers to a directory. Returns 0 on success, or the
+// error number on failure.
+int platform_check_dir(int fd);
+
// This class is designed to allow implementation of the POSIX dirent.h API.
// By itself, it is platform independent but calls platform specific
// functions to perform OS operations.
@@ -63,8 +68,9 @@ class Dir {
public:
static ErrorOr<Dir *> open(const char *path);
+ static ErrorOr<Dir *> fdopen(int fd);
- ErrorOr<struct ::dirent *> read();
+ ErrorOr<struct dirent *> read();
// Returns 0 on success or the error number on failure. If an error number
// was returned, then the resources associated with the directory are not
diff --git a/libc/src/__support/File/linux/CMakeLists.txt b/libc/src/__support/File/linux/CMakeLists.txt
index 7b770f916a251..ba64ecf41a76e 100644
--- a/libc/src/__support/File/linux/CMakeLists.txt
+++ b/libc/src/__support/File/linux/CMakeLists.txt
@@ -25,10 +25,15 @@ add_object_library(
SRCS
dir.cpp
DEPENDS
+ libc.src.__support.common
+ libc.src.__support.macros.config
libc.hdr.fcntl_macros
- libc.include.sys_syscall
+ libc.hdr.sys_stat_macros
+ libc.src.__support.OSUtil.linux.stat.kernel_statx_types
libc.src.__support.OSUtil.osutil
libc.src.__support.OSUtil.linux.syscall_wrappers.open
+ libc.src.__support.OSUtil.linux.syscall_wrappers.statx
+ libc.src.__support.OSUtil.linux.syscall_wrappers.fcntl
libc.src.__support.error_or
libc.src.errno.errno
libc.src.__support.File.dir
diff --git a/libc/src/__support/File/linux/dir.cpp b/libc/src/__support/File/linux/dir.cpp
index 8412b42348559..77f9a51952c3b 100644
--- a/libc/src/__support/File/linux/dir.cpp
+++ b/libc/src/__support/File/linux/dir.cpp
@@ -1,21 +1,27 @@
-//===--- Linux implementation of the Dir helpers --------------------------===//
+//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Linux implementation of the Dir helpers.
+///
+//===----------------------------------------------------------------------===//
#include "src/__support/File/dir.h"
-
+#include "hdr/fcntl_macros.h" // For open flags
+#include "hdr/sys_stat_macros.h" // For S_ISDIR
+#include "src/__support/OSUtil/linux/stat/kernel_statx_types.h"
+#include "src/__support/OSUtil/linux/syscall_wrappers/fcntl.h"
#include "src/__support/OSUtil/linux/syscall_wrappers/open.h"
+#include "src/__support/OSUtil/linux/syscall_wrappers/statx.h"
#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
#include "src/__support/error_or.h"
#include "src/__support/macros/config.h"
-#include "hdr/fcntl_macros.h" // For open flags
-#include <sys/syscall.h> // For syscall numbers
-
namespace LIBC_NAMESPACE_DECL {
ErrorOr<int> platform_opendir(const char *name) {
@@ -30,17 +36,38 @@ ErrorOr<size_t> platform_fetch_dirents(int fd, cpp::span<uint8_t> buffer) {
#error "getdents64 syscalls not available to perform a fetch dirents operation."
#endif
- if (size < 0) {
+ if (size < 0)
return LIBC_NAMESPACE::Error(static_cast<int>(-size));
- }
return size;
}
int platform_closedir(int fd) {
int ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_close, fd);
- if (ret < 0) {
+ if (ret < 0)
return static_cast<int>(-ret);
- }
+ return 0;
+}
+
+int platform_check_dir(int fd) {
+ // 1. Verify fd is valid and open for reading.
+ auto flags = linux_syscalls::fcntl(fd, F_GETFL);
+ if (!flags)
+ return flags.error();
+
+ // Check if open for reading.
+ if ((flags.value() & O_PATH) || ((flags.value() & O_ACCMODE) == O_WRONLY))
+ return EBADF;
+
+ // 2. Verify fd refers to a directory.
+ internal::kernel_statx_buf xbuf;
+ auto result = linux_syscalls::statx(fd, "", AT_EMPTY_PATH,
+ internal::KERNEL_STATX_TYPE_MASK, &xbuf);
+ if (!result)
+ return result.error();
+
+ if (!S_ISDIR(xbuf.stx_mode))
+ return ENOTDIR;
+
return 0;
}
diff --git a/libc/src/__support/OSUtil/linux/stat/kernel_statx_types.h b/libc/src/__support/OSUtil/linux/stat/kernel_statx_types.h
index 080d48bb176ef..bbaf6d82a7d73 100644
--- a/libc/src/__support/OSUtil/linux/stat/kernel_statx_types.h
+++ b/libc/src/__support/OSUtil/linux/stat/kernel_statx_types.h
@@ -65,6 +65,9 @@ struct kernel_statx_buf {
// stx_btime field will be filled in.
constexpr unsigned int KERNEL_STATX_BASIC_STATS_MASK = 0x7FF;
+// This mask is used to check the type of the file descriptor.
+constexpr unsigned int KERNEL_STATX_TYPE_MASK = 0x008;
+
} // namespace internal
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/OSUtil/linux/syscall_wrappers/CMakeLists.txt b/libc/src/__support/OSUtil/linux/syscall_wrappers/CMakeLists.txt
index f3282c315d9a9..7f416bb5a2407 100644
--- a/libc/src/__support/OSUtil/linux/syscall_wrappers/CMakeLists.txt
+++ b/libc/src/__support/OSUtil/linux/syscall_wrappers/CMakeLists.txt
@@ -631,6 +631,18 @@ add_header_library(
libc.include.sys_syscall
)
+add_header_library(
+ fcntl
+ HDRS
+ fcntl.h
+ DEPENDS
+ libc.src.__support.OSUtil.osutil
+ libc.src.__support.common
+ libc.src.__support.error_or
+ libc.src.__support.macros.config
+ libc.include.sys_syscall
+)
+
add_header_library(
fsync
HDRS
diff --git a/libc/src/__support/OSUtil/linux/syscall_wrappers/fcntl.h b/libc/src/__support/OSUtil/linux/syscall_wrappers/fcntl.h
new file mode 100644
index 0000000000000..ce73fe71f18cb
--- /dev/null
+++ b/libc/src/__support/OSUtil/linux/syscall_wrappers/fcntl.h
@@ -0,0 +1,43 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// ErrorOr-returning syscall wrapper for fcntl.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_SYSCALL_WRAPPERS_FCNTL_H
+#define LLVM_LIBC_SRC___SUPPORT_OSUTIL_SYSCALL_WRAPPERS_FCNTL_H
+
+#include "src/__support/OSUtil/linux/syscall.h" // syscall_impl
+#include "src/__support/common.h"
+#include "src/__support/error_or.h"
+#include "src/__support/macros/config.h"
+#include <sys/syscall.h> // For syscall numbers
+
+namespace LIBC_NAMESPACE_DECL {
+namespace linux_syscalls {
+
+LIBC_INLINE ErrorOr<int> fcntl(int fd, int cmd, void *arg) {
+ int ret = syscall_impl<int>(SYS_fcntl, fd, cmd, arg);
+ if (ret < 0)
+ return Error(-ret);
+ return ret;
+}
+
+LIBC_INLINE ErrorOr<int> fcntl(int fd, int cmd, long arg = 0) {
+ int ret = syscall_impl<int>(SYS_fcntl, fd, cmd, arg);
+ if (ret < 0)
+ return Error(-ret);
+ return ret;
+}
+
+} // namespace linux_syscalls
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_SYSCALL_WRAPPERS_FCNTL_H
diff --git a/libc/src/dirent/CMakeLists.txt b/libc/src/dirent/CMakeLists.txt
index eb4184693a43c..ea17bd4cfbba3 100644
--- a/libc/src/dirent/CMakeLists.txt
+++ b/libc/src/dirent/CMakeLists.txt
@@ -48,3 +48,18 @@ add_entrypoint_object(
libc.src.__support.File.platform_dir
libc.src.errno.errno
)
+
+add_entrypoint_object(
+ fdopendir
+ SRCS
+ fdopendir.cpp
+ HDRS
+ fdopendir.h
+ DEPENDS
+ libc.src.__support.common
+ libc.src.__support.macros.config
+ libc.hdr.types.DIR
+ libc.src.__support.File.dir
+ libc.src.errno.errno
+)
+
diff --git a/libc/src/dirent/fdopendir.cpp b/libc/src/dirent/fdopendir.cpp
new file mode 100644
index 0000000000000..b92709c4a54e8
--- /dev/null
+++ b/libc/src/dirent/fdopendir.cpp
@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Implementation of the POSIX fdopendir function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "fdopendir.h"
+
+#include "src/__support/File/dir.h"
+#include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(DIR *, fdopendir, (int fd)) {
+ int check = platform_check_dir(fd);
+ if (check != 0) {
+ libc_errno = check;
+ return nullptr;
+ }
+
+ auto dir = Dir::fdopen(fd);
+ if (!dir) {
+ libc_errno = dir.error();
+ return nullptr;
+ }
+ return reinterpret_cast<DIR *>(dir.value());
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/dirent/fdopendir.h b/libc/src/dirent/fdopendir.h
new file mode 100644
index 0000000000000..d70f2f7cddecf
--- /dev/null
+++ b/libc/src/dirent/fdopendir.h
@@ -0,0 +1,26 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Declaration of the POSIX fdopendir function.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_DIRENT_FDOPENDIR_H
+#define LLVM_LIBC_SRC_DIRENT_FDOPENDIR_H
+
+#include "hdr/types/DIR.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+DIR *fdopendir(int fd);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_DIRENT_FDOPENDIR_H
diff --git a/libc/test/src/dirent/CMakeLists.txt b/libc/test/src/dirent/CMakeLists.txt
index 8db512129f893..af18006c5d122 100644
--- a/libc/test/src/dirent/CMakeLists.txt
+++ b/libc/test/src/dirent/CMakeLists.txt
@@ -17,3 +17,25 @@ add_libc_unittest(
libc.test.UnitTest.ErrnoCheckingTest
)
+add_libc_unittest(
+ fdopendir_test
+ SUITE
+ libc_dirent_unittests
+ SRCS
+ fdopendir_test.cpp
+ DEPENDS
+ libc.src.__support.CPP.string_view
+ libc.src.dirent.fdopendir
+ libc.src.dirent.closedir
+ libc.src.dirent.readdir
+ libc.src.fcntl.open
+ libc.src.unistd.close
+ libc.src.fcntl.fcntl
+ libc.src.errno.errno
+ libc.test.UnitTest.ErrnoCheckingTest
+ libc.test.UnitTest.ErrnoSetterMatcher
+ libc.hdr.types.DIR
+ libc.hdr.types.struct_dirent
+)
+
+
diff --git a/libc/test/src/dirent/fdopendir_test.cpp b/libc/test/src/dirent/fdopendir_test.cpp
new file mode 100644
index 0000000000000..29db66bcc1636
--- /dev/null
+++ b/libc/test/src/dirent/fdopendir_test.cpp
@@ -0,0 +1,113 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Unit tests for the POSIX fdopendir function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "hdr/types/DIR.h"
+#include "hdr/types/struct_dirent.h"
+#include "src/dirent/closedir.h"
+#include "src/dirent/dirfd.h"
+#include "src/dirent/fdopendir.h"
+#include "src/dirent/readdir.h"
+#include "src/fcntl/fcntl.h"
+#include "src/fcntl/open.h"
+#include "src/unistd/close.h"
+
+#include "src/__support/CPP/string_view.h"
+#include "test/UnitTest/ErrnoCheckingTest.h"
+#include "test/UnitTest/ErrnoSetterMatcher.h"
+#include "test/UnitTest/Test.h"
+
+#include "hdr/fcntl_macros.h"
+
+using LlvmLibcFdopendirTest = LIBC_NAMESPACE::testing::ErrnoCheckingTest;
+using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
+
+TEST_F(LlvmLibcFdopendirTest, SuccessCase) {
+ int fd = LIBC_NAMESPACE::open("testdata", O_RDONLY | O_DIRECTORY);
+ ASSERT_GT(fd, 0);
+ ASSERT_ERRNO_SUCCESS();
+
+ DIR *dir = LIBC_NAMESPACE::fdopendir(fd);
+ ASSERT_TRUE(dir != nullptr);
+ ASSERT_ERRNO_SUCCESS();
+
+ struct dirent *file1 = nullptr, *file2 = nullptr;
+ while (true) {
+ struct dirent *d = LIBC_NAMESPACE::readdir(dir);
+ if (d == nullptr)
+ break;
+ if (LIBC_NAMESPACE::cpp::string_view(d->d_name) == "file1.txt")
+ file1 = d;
+ if (LIBC_NAMESPACE::cpp::string_view(d->d_name) == "file2.txt")
+ file2 = d;
+ }
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_TRUE(file1 != nullptr);
+ ASSERT_TRUE(file2 != nullptr);
+
+ ASSERT_EQ(LIBC_NAMESPACE::closedir(dir), 0);
+
+ // Verify fd is closed
+ int fcntl_res = LIBC_NAMESPACE::fcntl(fd, F_GETFD);
+ ASSERT_EQ(fcntl_res, -1);
+ ASSERT_ERRNO_EQ(EBADF);
+}
+
+TEST_F(LlvmLibcFdopendirTest, InvalidFd) {
+ DIR *dir = LIBC_NAMESPACE::fdopendir(-1);
+ ASSERT_TRUE(dir == nullptr);
+ ASSERT_ERRNO_EQ(EBADF);
+}
+
+TEST_F(LlvmLibcFdopendirTest, ClosedFd) {
+ int fd = LIBC_NAMESPACE::open("testdata", O_RDONLY | O_DIRECTORY);
+ ASSERT_GT(fd, 0);
+ ASSERT_THAT(LIBC_NAMESPACE::close(fd), Succeeds(0));
+
+ DIR *dir = LIBC_NAMESPACE::fdopendir(fd);
+ ASSERT_TRUE(dir == nullptr);
+ ASSERT_ERRNO_EQ(EBADF);
+}
+
+TEST_F(LlvmLibcFdopendirTest, NotADirectory) {
+ int fd = LIBC_NAMESPACE::open("testdata/file1.txt", O_RDONLY);
+ ASSERT_GT(fd, 0);
+
+ DIR *dir = LIBC_NAMESPACE::fdopendir(fd);
+ ASSERT_TRUE(dir == nullptr);
+ ASSERT_ERRNO_EQ(ENOTDIR);
+
+ ASSERT_THAT(LIBC_NAMESPACE::close(fd), Succeeds(0));
+}
+
+TEST_F(LlvmLibcFdopendirTest, OPathFd) {
+ int fd = LIBC_NAMESPACE::open("testdata", O_PATH);
+ ASSERT_GT(fd, 0);
+
+ DIR *dir = LIBC_NAMESPACE::fdopendir(fd);
+ ASSERT_TRUE(dir == nullptr);
+ ASSERT_ERRNO_EQ(EBADF);
+
+ ASSERT_THAT(LIBC_NAMESPACE::close(fd), Succeeds(0));
+}
+
+TEST_F(LlvmLibcFdopendirTest, WriteOnlyFd) {
+ int fd = LIBC_NAMESPACE::open("testdata/file1.txt", O_WRONLY);
+ ASSERT_GT(fd, 0);
+
+ DIR *dir = LIBC_NAMESPACE::fdopendir(fd);
+ ASSERT_TRUE(dir == nullptr);
+ ASSERT_TRUE(libc_errno == EBADF || libc_errno == ENOTDIR);
+ libc_errno = 0;
+
+ ASSERT_THAT(LIBC_NAMESPACE::close(fd), Succeeds(0));
+}
>From 2590272e5674c300c9438ef5c10a65358b0636e9 Mon Sep 17 00:00:00 2001
From: Alex Langford <alangford at apple.com>
Date: Mon, 6 Jul 2026 11:24:41 -0700
Subject: [PATCH 36/46] [lldb] Move RegisterInfo and RegisterSet into their own
header (#207286)
RegisterInfo holds onto a pointer to RegisterFlags which is in
lldbUtility. My understanding of lldb-private-types is that it contains
types used throughout LLDB but should generally stand on their own (i.e.
without dependencies on specific lldb types and headers). Thus I am
moving it to lldbUtility.
RegisterSet _does_ stand on its own but it is pretty strongly coupled to
RegisterInfo.
---
lldb/include/lldb/Core/EmulateInstruction.h | 1 +
lldb/include/lldb/Core/Value.h | 1 +
lldb/include/lldb/Expression/Materializer.h | 1 +
.../lldb/Host/common/NativeRegisterContext.h | 1 +
lldb/include/lldb/Symbol/UnwindPlan.h | 1 +
.../include/lldb/Target/DynamicRegisterInfo.h | 1 +
lldb/include/lldb/Target/RegisterContext.h | 1 +
lldb/include/lldb/Utility/RegisterInfo.h | 90 +++++++++++++++++++
lldb/include/lldb/Utility/RegisterValue.h | 2 +-
lldb/include/lldb/lldb-private-types.h | 68 --------------
.../Utility/RegisterFlagsDetector_arm64.h | 1 +
.../Process/Utility/RegisterInfoInterface.h | 2 +-
12 files changed, 100 insertions(+), 70 deletions(-)
create mode 100644 lldb/include/lldb/Utility/RegisterInfo.h
diff --git a/lldb/include/lldb/Core/EmulateInstruction.h b/lldb/include/lldb/Core/EmulateInstruction.h
index 6f2d272895dce..f661bb7ab6483 100644
--- a/lldb/include/lldb/Core/EmulateInstruction.h
+++ b/lldb/include/lldb/Core/EmulateInstruction.h
@@ -16,6 +16,7 @@
#include "lldb/Core/Opcode.h"
#include "lldb/Core/PluginInterface.h"
#include "lldb/Utility/ArchSpec.h"
+#include "lldb/Utility/RegisterInfo.h"
#include "lldb/lldb-defines.h"
#include "lldb/lldb-enumerations.h"
#include "lldb/lldb-private-enumerations.h"
diff --git a/lldb/include/lldb/Core/Value.h b/lldb/include/lldb/Core/Value.h
index 3714621b469ec..6335d31e84db8 100644
--- a/lldb/include/lldb/Core/Value.h
+++ b/lldb/include/lldb/Core/Value.h
@@ -11,6 +11,7 @@
#include "lldb/Symbol/CompilerType.h"
#include "lldb/Utility/DataBufferHeap.h"
+#include "lldb/Utility/RegisterInfo.h"
#include "lldb/Utility/Scalar.h"
#include "lldb/Utility/Status.h"
#include "lldb/lldb-enumerations.h"
diff --git a/lldb/include/lldb/Expression/Materializer.h b/lldb/include/lldb/Expression/Materializer.h
index 8f850c3f46439..64ed545febf15 100644
--- a/lldb/include/lldb/Expression/Materializer.h
+++ b/lldb/include/lldb/Expression/Materializer.h
@@ -15,6 +15,7 @@
#include "lldb/Expression/IRMemoryMap.h"
#include "lldb/Symbol/TaggedASTType.h"
#include "lldb/Target/StackFrame.h"
+#include "lldb/Utility/RegisterInfo.h"
#include "lldb/Utility/Status.h"
#include "lldb/lldb-private-types.h"
diff --git a/lldb/include/lldb/Host/common/NativeRegisterContext.h b/lldb/include/lldb/Host/common/NativeRegisterContext.h
index 0a7647d780803..02792d0e09294 100644
--- a/lldb/include/lldb/Host/common/NativeRegisterContext.h
+++ b/lldb/include/lldb/Host/common/NativeRegisterContext.h
@@ -10,6 +10,7 @@
#define LLDB_HOST_COMMON_NATIVEREGISTERCONTEXT_H
#include "lldb/Host/common/NativeWatchpointList.h"
+#include "lldb/Utility/RegisterInfo.h"
#include "lldb/lldb-private.h"
namespace lldb_private {
diff --git a/lldb/include/lldb/Symbol/UnwindPlan.h b/lldb/include/lldb/Symbol/UnwindPlan.h
index 99d948f8b8a8c..957eb6afc6573 100644
--- a/lldb/include/lldb/Symbol/UnwindPlan.h
+++ b/lldb/include/lldb/Symbol/UnwindPlan.h
@@ -15,6 +15,7 @@
#include "lldb/Core/AddressRange.h"
#include "lldb/Utility/ConstString.h"
+#include "lldb/Utility/RegisterInfo.h"
#include "lldb/Utility/Stream.h"
#include "lldb/lldb-private.h"
diff --git a/lldb/include/lldb/Target/DynamicRegisterInfo.h b/lldb/include/lldb/Target/DynamicRegisterInfo.h
index ab476bffde45a..fda9f407d7ce8 100644
--- a/lldb/include/lldb/Target/DynamicRegisterInfo.h
+++ b/lldb/include/lldb/Target/DynamicRegisterInfo.h
@@ -14,6 +14,7 @@
#include "lldb/Utility/ConstString.h"
#include "lldb/Utility/RegisterFlags.h"
+#include "lldb/Utility/RegisterInfo.h"
#include "lldb/Utility/StructuredData.h"
#include "lldb/lldb-private.h"
diff --git a/lldb/include/lldb/Target/RegisterContext.h b/lldb/include/lldb/Target/RegisterContext.h
index 309e231b2321e..2b81006d01911 100644
--- a/lldb/include/lldb/Target/RegisterContext.h
+++ b/lldb/include/lldb/Target/RegisterContext.h
@@ -10,6 +10,7 @@
#define LLDB_TARGET_REGISTERCONTEXT_H
#include "lldb/Target/ExecutionContextScope.h"
+#include "lldb/Utility/RegisterInfo.h"
#include "lldb/lldb-private.h"
namespace lldb_private {
diff --git a/lldb/include/lldb/Utility/RegisterInfo.h b/lldb/include/lldb/Utility/RegisterInfo.h
new file mode 100644
index 0000000000000..8c903138e2b4b
--- /dev/null
+++ b/lldb/include/lldb/Utility/RegisterInfo.h
@@ -0,0 +1,90 @@
+//===------------------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_UTILITY_REGISTERINFO_H
+#define LLDB_UTILITY_REGISTERINFO_H
+
+#include "lldb/Utility/RegisterFlags.h"
+#include "lldb/lldb-enumerations.h"
+
+#include "llvm/ADT/ArrayRef.h"
+
+#include <cstdint>
+
+namespace lldb_private {
+
+/// Every register is described in detail including its name, alternate name
+/// (optional), encoding, size in bytes and the default display format.
+struct RegisterInfo {
+ /// Name of this register, can't be NULL.
+ const char *name;
+ /// Alternate name of this register, can be NULL.
+ const char *alt_name;
+ /// Size in bytes of the register.
+ uint32_t byte_size;
+ /// The byte offset in the register context data where this register's value
+ /// is found.
+ /// This is optional, and can be 0 if a particular RegisterContext does not
+ /// need to address its registers by byte offset.
+ uint32_t byte_offset;
+ /// Encoding of the register bits.
+ lldb::Encoding encoding;
+ /// Default display format.
+ lldb::Format format;
+ /// Holds all of the various register numbers for all register kinds.
+ uint32_t kinds[lldb::kNumRegisterKinds];
+ /// List of registers (terminated with LLDB_INVALID_REGNUM). If this value is
+ /// not null, all registers in this list will be read first, at which point
+ /// the value for this register will be valid. FOr example, the value list for
+ /// ah would be eax (x86) or rax (x64). Register numbers are of
+ /// eRegisterKindLLDB. If multiple registers are listed, the final value will
+ /// be the concatenation of them.
+ uint32_t *value_regs;
+ /// List of registers (terminated with LLDB_INVALID_REGNUM). If this value is
+ /// not null, all registers in this list will be invalidated when the value of
+ /// this register changes. For example, the invalidate list for eax would be
+ /// rax ax, ah, and al
+ uint32_t *invalidate_regs;
+ /// If not nullptr, a type defined by XML descriptions.
+ /// Register info tables are constructed as const, but this field may need to
+ /// be updated if a specific target OS has a different layout. To enable that,
+ /// this is mutable. The data pointed to is still const, so you must swap a
+ /// whole set of flags for another.
+ mutable const RegisterFlags *flags_type;
+
+ llvm::ArrayRef<uint8_t> data(const uint8_t *context_base) const {
+ return llvm::ArrayRef<uint8_t>(context_base + byte_offset, byte_size);
+ }
+
+ llvm::MutableArrayRef<uint8_t> mutable_data(uint8_t *context_base) const {
+ return llvm::MutableArrayRef<uint8_t>(context_base + byte_offset,
+ byte_size);
+ }
+};
+static_assert(std::is_trivial<RegisterInfo>::value,
+ "RegisterInfo must be trivial.");
+
+/// Registers are grouped into register sets
+struct RegisterSet {
+ /// Name of this register set.
+ const char *name;
+ /// A short name for this register set.
+ const char *short_name;
+ /// The number of registers in REGISTERS array below.
+ size_t num_registers;
+ /// An array of register indicies in this set. The values in this array are
+ /// *indices* (not register numbers) into a particular RegisterContext's
+ /// register array. For example, if eax is defined at index 4 for a particular
+ /// RegisterContext, eax would be included in this RegisterSet by adding the
+ /// value 4. Not by adding the value lldb_eax_i386.
+ const uint32_t *registers;
+};
+
+} // namespace lldb_private
+
+#endif // LLDB_UTILITY_REGISTERINFO_H
diff --git a/lldb/include/lldb/Utility/RegisterValue.h b/lldb/include/lldb/Utility/RegisterValue.h
index baf984cbcb052..e4a2de35559fa 100644
--- a/lldb/include/lldb/Utility/RegisterValue.h
+++ b/lldb/include/lldb/Utility/RegisterValue.h
@@ -10,6 +10,7 @@
#define LLDB_UTILITY_REGISTERVALUE_H
#include "lldb/Utility/Endian.h"
+#include "lldb/Utility/RegisterInfo.h"
#include "lldb/Utility/Scalar.h"
#include "lldb/Utility/Status.h"
#include "lldb/lldb-enumerations.h"
@@ -24,7 +25,6 @@
namespace lldb_private {
class DataExtractor;
class Stream;
-struct RegisterInfo;
class RegisterValue {
public:
diff --git a/lldb/include/lldb/lldb-private-types.h b/lldb/include/lldb/lldb-private-types.h
index a60034314b77e..d21f3cedc0419 100644
--- a/lldb/include/lldb/lldb-private-types.h
+++ b/lldb/include/lldb/lldb-private-types.h
@@ -23,80 +23,12 @@ class DynamicLibrary;
namespace lldb_private {
class Platform;
class ExecutionContext;
-class RegisterFlags;
typedef llvm::SmallString<256> PathSmallString;
typedef llvm::sys::DynamicLibrary (*LoadPluginCallbackType)(
const lldb::DebuggerSP &debugger_sp, const FileSpec &spec, Status &error);
-/// Every register is described in detail including its name, alternate name
-/// (optional), encoding, size in bytes and the default display format.
-struct RegisterInfo {
- /// Name of this register, can't be NULL.
- const char *name;
- /// Alternate name of this register, can be NULL.
- const char *alt_name;
- /// Size in bytes of the register.
- uint32_t byte_size;
- /// The byte offset in the register context data where this register's
- /// value is found.
- /// This is optional, and can be 0 if a particular RegisterContext does not
- /// need to address its registers by byte offset.
- uint32_t byte_offset;
- /// Encoding of the register bits.
- lldb::Encoding encoding;
- /// Default display format.
- lldb::Format format;
- /// Holds all of the various register numbers for all register kinds.
- uint32_t kinds[lldb::kNumRegisterKinds]; //
- /// List of registers (terminated with LLDB_INVALID_REGNUM). If this value is
- /// not null, all registers in this list will be read first, at which point
- /// the value for this register will be valid. For example, the value list
- /// for ah would be eax (x86) or rax (x64). Register numbers are
- /// of eRegisterKindLLDB. If multiple registers are listed, the final
- /// value will be the concatenation of them.
- uint32_t *value_regs;
- /// List of registers (terminated with LLDB_INVALID_REGNUM). If this value is
- /// not null, all registers in this list will be invalidated when the value of
- /// this register changes. For example, the invalidate list for eax would be
- /// rax ax, ah, and al.
- uint32_t *invalidate_regs;
- /// If not nullptr, a type defined by XML descriptions.
- /// Register info tables are constructed as const, but this field may need to
- /// be updated if a specific target OS has a different layout. To enable that,
- /// this is mutable. The data pointed to is still const, so you must swap a
- /// whole set of flags for another.
- mutable const RegisterFlags *flags_type;
-
- llvm::ArrayRef<uint8_t> data(const uint8_t *context_base) const {
- return llvm::ArrayRef<uint8_t>(context_base + byte_offset, byte_size);
- }
-
- llvm::MutableArrayRef<uint8_t> mutable_data(uint8_t *context_base) const {
- return llvm::MutableArrayRef<uint8_t>(context_base + byte_offset,
- byte_size);
- }
-};
-static_assert(std::is_trivial<RegisterInfo>::value,
- "RegisterInfo must be trivial.");
-
-/// Registers are grouped into register sets
-struct RegisterSet {
- /// Name of this register set.
- const char *name;
- /// A short name for this register set.
- const char *short_name;
- /// The number of registers in REGISTERS array below.
- size_t num_registers;
- /// An array of register indices in this set. The values in this array are
- /// *indices* (not register numbers) into a particular RegisterContext's
- /// register array. For example, if eax is defined at index 4 for a
- /// particular RegisterContext, eax would be included in this RegisterSet by
- /// adding the value 4. Not by adding the value lldb_eax_i386.
- const uint32_t *registers;
-};
-
/// A type-erased pair of llvm::dwarf::SourceLanguageName and version.
struct SourceLanguage {
SourceLanguage() = default;
diff --git a/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.h b/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.h
index ff69c94e0a7af..178b79cc53f28 100644
--- a/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.h
+++ b/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.h
@@ -10,6 +10,7 @@
#define LLDB_SOURCE_PLUGINS_PROCESS_UTILITY_REGISTERFLAGSDETECTOR_ARM64_H
#include "lldb/Utility/RegisterFlags.h"
+#include "lldb/Utility/RegisterInfo.h"
#include "llvm/ADT/StringRef.h"
#include <functional>
diff --git a/lldb/source/Plugins/Process/Utility/RegisterInfoInterface.h b/lldb/source/Plugins/Process/Utility/RegisterInfoInterface.h
index a79c5cc22b24f..38d086de22bee 100644
--- a/lldb/source/Plugins/Process/Utility/RegisterInfoInterface.h
+++ b/lldb/source/Plugins/Process/Utility/RegisterInfoInterface.h
@@ -10,7 +10,7 @@
#define LLDB_SOURCE_PLUGINS_PROCESS_UTILITY_REGISTERINFOINTERFACE_H
#include "lldb/Utility/ArchSpec.h"
-#include "lldb/lldb-private-types.h"
+#include "lldb/Utility/RegisterInfo.h"
#include <vector>
namespace lldb_private {
>From b74512899707cfcf962ebfa21f9209378372e78b Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Mon, 6 Jul 2026 11:30:44 -0700
Subject: [PATCH 37/46] [NFC][PatternMatch] Simplify m_Intrinsic impl using
variadic templates (#207211)
Updates both IR/VPlan pattern matchers.
---
llvm/include/llvm/IR/PatternMatch.h | 177 ++++--------------
.../Transforms/Vectorize/VPlanPatternMatch.h | 65 ++-----
2 files changed, 53 insertions(+), 189 deletions(-)
diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h
index 95f5a9c5bba80..8ff9565244964 100644
--- a/llvm/include/llvm/IR/PatternMatch.h
+++ b/llvm/include/llvm/IR/PatternMatch.h
@@ -44,6 +44,7 @@
#include "llvm/Support/Casting.h"
#include "llvm/Support/PatternMatchHelpers.h"
#include <cstdint>
+#include <utility>
using namespace llvm::PatternMatchHelpers;
@@ -2752,50 +2753,19 @@ template <Intrinsic::ID... IntrIDs> struct IntrinsicIDs_match {
}
};
-/// Intrinsic matches are combinations of ID matchers, and argument
-/// matchers. Higher arity matcher are defined recursively in terms of and-ing
-/// them with lower arity matchers. Here's some convenient typedefs for up to
-/// several arguments, and more can be added as needed
-template <typename T0 = void, typename T1 = void, typename T2 = void,
- typename T3 = void, typename T4 = void, typename T5 = void,
- typename T6 = void, typename T7 = void, typename T8 = void,
- typename T9 = void, typename T10 = void>
-struct m_Intrinsic_Ty;
-template <typename T0> struct m_Intrinsic_Ty<T0> {
- using Ty = match_combine_and<IntrinsicID_match, Argument_match<T0>>;
-};
-template <typename T0, typename T1> struct m_Intrinsic_Ty<T0, T1> {
- using Ty =
- match_combine_and<typename m_Intrinsic_Ty<T0>::Ty, Argument_match<T1>>;
-};
-template <typename T0, typename T1, typename T2>
-struct m_Intrinsic_Ty<T0, T1, T2> {
- using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1>::Ty,
- Argument_match<T2>>;
-};
-template <typename T0, typename T1, typename T2, typename T3>
-struct m_Intrinsic_Ty<T0, T1, T2, T3> {
- using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2>::Ty,
- Argument_match<T3>>;
-};
-
-template <typename T0, typename T1, typename T2, typename T3, typename T4>
-struct m_Intrinsic_Ty<T0, T1, T2, T3, T4> {
- using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty,
- Argument_match<T4>>;
-};
-
-template <typename T0, typename T1, typename T2, typename T3, typename T4,
- typename T5>
-struct m_Intrinsic_Ty<T0, T1, T2, T3, T4, T5> {
- using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2, T3, T4>::Ty,
- Argument_match<T5>>;
+struct IntrinsicMatchImpl {
+ template <Intrinsic::ID IntrID, typename... Ts, size_t... Is>
+ static auto impl(std::index_sequence<Is...>, const Ts &...Ops) {
+ return m_CombineAnd(IntrinsicID_match(IntrID), m_Argument<Is>(Ops)...);
+ }
};
/// Match intrinsic calls like this:
/// m_Intrinsic<Intrinsic::fabs>(m_Value(X))
-template <Intrinsic::ID IntrID> inline IntrinsicID_match m_Intrinsic() {
- return IntrinsicID_match(IntrID);
+template <Intrinsic::ID IntrID, typename... Ts>
+inline auto m_Intrinsic(const Ts &...Ops) {
+ return IntrinsicMatchImpl::impl<IntrID>(
+ std::make_index_sequence<sizeof...(Ts)>{}, Ops...);
}
/// Match intrinsic calls with any of the given IDs like this:
@@ -2809,101 +2779,51 @@ inline IntrinsicIDs_match<IntrIDs...> m_AnyIntrinsic() {
/// Matches MaskedLoad Intrinsic.
template <typename Opnd0, typename Opnd1, typename Opnd2>
-inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2>::Ty
-m_MaskedLoad(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) {
+inline auto m_MaskedLoad(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) {
return m_Intrinsic<Intrinsic::masked_load>(Op0, Op1, Op2);
}
/// Matches MaskedStore Intrinsic.
template <typename Opnd0, typename Opnd1, typename Opnd2>
-inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2>::Ty
-m_MaskedStore(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) {
+inline auto m_MaskedStore(const Opnd0 &Op0, const Opnd1 &Op1,
+ const Opnd2 &Op2) {
return m_Intrinsic<Intrinsic::masked_store>(Op0, Op1, Op2);
}
/// Matches MaskedGather Intrinsic.
template <typename Opnd0, typename Opnd1, typename Opnd2>
-inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2>::Ty
-m_MaskedGather(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) {
+inline auto m_MaskedGather(const Opnd0 &Op0, const Opnd1 &Op1,
+ const Opnd2 &Op2) {
return m_Intrinsic<Intrinsic::masked_gather>(Op0, Op1, Op2);
}
-template <Intrinsic::ID IntrID, typename T0>
-inline typename m_Intrinsic_Ty<T0>::Ty m_Intrinsic(const T0 &Op0) {
- return m_CombineAnd(m_Intrinsic<IntrID>(), m_Argument<0>(Op0));
-}
-
-template <Intrinsic::ID IntrID, typename T0, typename T1>
-inline typename m_Intrinsic_Ty<T0, T1>::Ty m_Intrinsic(const T0 &Op0,
- const T1 &Op1) {
- return m_CombineAnd(m_Intrinsic<IntrID>(Op0), m_Argument<1>(Op1));
-}
-
-template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2>
-inline typename m_Intrinsic_Ty<T0, T1, T2>::Ty
-m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2) {
- return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1), m_Argument<2>(Op2));
-}
-
-template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2,
- typename T3>
-inline typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty
-m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3) {
- return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2), m_Argument<3>(Op3));
-}
-
-template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2,
- typename T3, typename T4>
-inline typename m_Intrinsic_Ty<T0, T1, T2, T3, T4>::Ty
-m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3,
- const T4 &Op4) {
- return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2, Op3),
- m_Argument<4>(Op4));
-}
-
-template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2,
- typename T3, typename T4, typename T5>
-inline typename m_Intrinsic_Ty<T0, T1, T2, T3, T4, T5>::Ty
-m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3,
- const T4 &Op4, const T5 &Op5) {
- return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2, Op3, Op4),
- m_Argument<5>(Op5));
-}
-
// Helper intrinsic matching specializations.
-template <typename Opnd0>
-inline typename m_Intrinsic_Ty<Opnd0>::Ty m_BitReverse(const Opnd0 &Op0) {
+template <typename Opnd0> inline auto m_BitReverse(const Opnd0 &Op0) {
return m_Intrinsic<Intrinsic::bitreverse>(Op0);
}
-template <typename Opnd0>
-inline typename m_Intrinsic_Ty<Opnd0>::Ty m_BSwap(const Opnd0 &Op0) {
+template <typename Opnd0> inline auto m_BSwap(const Opnd0 &Op0) {
return m_Intrinsic<Intrinsic::bswap>(Op0);
}
-template <typename Opnd0>
-inline typename m_Intrinsic_Ty<Opnd0>::Ty m_Ctpop(const Opnd0 &Op0) {
+template <typename Opnd0> inline auto m_Ctpop(const Opnd0 &Op0) {
return m_Intrinsic<Intrinsic::ctpop>(Op0);
}
-template <typename Opnd0>
-inline typename m_Intrinsic_Ty<Opnd0>::Ty m_FAbs(const Opnd0 &Op0) {
+template <typename Opnd0> inline auto m_FAbs(const Opnd0 &Op0) {
return m_Intrinsic<Intrinsic::fabs>(Op0);
}
-template <typename Opnd0>
-inline typename m_Intrinsic_Ty<Opnd0>::Ty m_FCanonicalize(const Opnd0 &Op0) {
+template <typename Opnd0> inline auto m_FCanonicalize(const Opnd0 &Op0) {
return m_Intrinsic<Intrinsic::canonicalize>(Op0);
}
template <typename Opnd0, typename Opnd1>
-inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_Ctlz(const Opnd0 &Op0,
- const Opnd1 &Op1) {
+inline auto m_Ctlz(const Opnd0 &Op0, const Opnd1 &Op1) {
return m_Intrinsic<Intrinsic::ctlz>(Op0, Op1);
}
template <typename Opnd0, typename Opnd1>
-inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_Cttz(const Opnd0 &Op0,
- const Opnd1 &Op1) {
+inline auto m_Cttz(const Opnd0 &Op0, const Opnd1 &Op1) {
return m_Intrinsic<Intrinsic::cttz>(Op0, Op1);
}
@@ -2934,86 +2854,71 @@ inline auto m_MaxOrMin(const Opnd0 &Op0, const Opnd1 &Op1) {
}
template <typename Opnd0, typename Opnd1>
-inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_FMinNum(const Opnd0 &Op0,
- const Opnd1 &Op1) {
+inline auto m_FMinNum(const Opnd0 &Op0, const Opnd1 &Op1) {
return m_Intrinsic<Intrinsic::minnum>(Op0, Op1);
}
template <typename Opnd0, typename Opnd1>
-inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_FMinimum(const Opnd0 &Op0,
- const Opnd1 &Op1) {
+inline auto m_FMinimum(const Opnd0 &Op0, const Opnd1 &Op1) {
return m_Intrinsic<Intrinsic::minimum>(Op0, Op1);
}
template <typename Opnd0, typename Opnd1>
-inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty
-m_FMinimumNum(const Opnd0 &Op0, const Opnd1 &Op1) {
+inline auto m_FMinimumNum(const Opnd0 &Op0, const Opnd1 &Op1) {
return m_Intrinsic<Intrinsic::minimumnum>(Op0, Op1);
}
template <typename Opnd0, typename Opnd1>
-inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_FMaxNum(const Opnd0 &Op0,
- const Opnd1 &Op1) {
+inline auto m_FMaxNum(const Opnd0 &Op0, const Opnd1 &Op1) {
return m_Intrinsic<Intrinsic::maxnum>(Op0, Op1);
}
template <typename Opnd0, typename Opnd1>
-inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_FMaximum(const Opnd0 &Op0,
- const Opnd1 &Op1) {
+inline auto m_FMaximum(const Opnd0 &Op0, const Opnd1 &Op1) {
return m_Intrinsic<Intrinsic::maximum>(Op0, Op1);
}
template <typename Opnd0, typename Opnd1>
-inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty
-m_FMaximumNum(const Opnd0 &Op0, const Opnd1 &Op1) {
+inline auto m_FMaximumNum(const Opnd0 &Op0, const Opnd1 &Op1) {
return m_Intrinsic<Intrinsic::maximumnum>(Op0, Op1);
}
template <typename Opnd0, typename Opnd1>
-inline match_combine_or<typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty,
- typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty>
-m_FMaxNum_or_FMaximumNum(const Opnd0 &Op0, const Opnd1 &Op1) {
+inline auto m_FMaxNum_or_FMaximumNum(const Opnd0 &Op0, const Opnd1 &Op1) {
return m_CombineOr(m_FMaxNum(Op0, Op1), m_FMaximumNum(Op0, Op1));
}
template <typename Opnd0, typename Opnd1>
-inline match_combine_or<typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty,
- typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty>
-m_FMinNum_or_FMinimumNum(const Opnd0 &Op0, const Opnd1 &Op1) {
+inline auto m_FMinNum_or_FMinimumNum(const Opnd0 &Op0, const Opnd1 &Op1) {
return m_CombineOr(m_FMinNum(Op0, Op1), m_FMinimumNum(Op0, Op1));
}
template <typename Opnd0, typename Opnd1, typename Opnd2>
-inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2>::Ty
-m_FShl(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) {
+inline auto m_FShl(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) {
return m_Intrinsic<Intrinsic::fshl>(Op0, Op1, Op2);
}
template <typename Opnd0, typename Opnd1, typename Opnd2>
-inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2>::Ty
-m_FShr(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) {
+inline auto m_FShr(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) {
return m_Intrinsic<Intrinsic::fshr>(Op0, Op1, Op2);
}
-template <typename Opnd0>
-inline typename m_Intrinsic_Ty<Opnd0>::Ty m_Sqrt(const Opnd0 &Op0) {
+template <typename Opnd0> inline auto m_Sqrt(const Opnd0 &Op0) {
return m_Intrinsic<Intrinsic::sqrt>(Op0);
}
template <typename Opnd0, typename Opnd1>
-inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_CopySign(const Opnd0 &Op0,
- const Opnd1 &Op1) {
+inline auto m_CopySign(const Opnd0 &Op0, const Opnd1 &Op1) {
return m_Intrinsic<Intrinsic::copysign>(Op0, Op1);
}
-template <typename Opnd0>
-inline typename m_Intrinsic_Ty<Opnd0>::Ty m_VecReverse(const Opnd0 &Op0) {
+template <typename Opnd0> inline auto m_VecReverse(const Opnd0 &Op0) {
return m_Intrinsic<Intrinsic::vector_reverse>(Op0);
}
template <typename Opnd0, typename Opnd1, typename Opnd2>
-inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2>::Ty
-m_VectorInsert(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) {
+inline auto m_VectorInsert(const Opnd0 &Op0, const Opnd1 &Op1,
+ const Opnd2 &Op2) {
return m_Intrinsic<Intrinsic::vector_insert>(Op0, Op1, Op2);
}
@@ -3256,16 +3161,14 @@ inline InsertValue_match<Ind, Val_t, Elt_t> m_InsertValue(const Val_t &Val,
}
/// Matches a call to `llvm.vscale()`.
-inline IntrinsicID_match m_VScale() { return m_Intrinsic<Intrinsic::vscale>(); }
+inline auto m_VScale() { return m_Intrinsic<Intrinsic::vscale>(); }
template <typename Opnd0, typename Opnd1>
-inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty
-m_Interleave2(const Opnd0 &Op0, const Opnd1 &Op1) {
+inline auto m_Interleave2(const Opnd0 &Op0, const Opnd1 &Op1) {
return m_Intrinsic<Intrinsic::vector_interleave2>(Op0, Op1);
}
-template <typename Opnd>
-inline typename m_Intrinsic_Ty<Opnd>::Ty m_Deinterleave2(const Opnd &Op) {
+template <typename Opnd> inline auto m_Deinterleave2(const Opnd &Op) {
return m_Intrinsic<Intrinsic::vector_deinterleave2>(Op);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index c57e106021637..4a4e0ee270760 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -18,6 +18,7 @@
#include "VPlan.h"
#include "VPlanUtils.h"
#include "llvm/Support/PatternMatchHelpers.h"
+#include <utility>
using namespace llvm::PatternMatchHelpers;
@@ -1016,64 +1017,24 @@ struct IntrinsicID_match {
}
};
-/// Intrinsic matches are combinations of ID matchers, and argument
-/// matchers. Higher arity matcher are defined recursively in terms of and-ing
-/// them with lower arity matchers. Here's some convenient typedefs for up to
-/// several arguments, and more can be added as needed
-template <typename T0 = void, typename T1 = void, typename T2 = void,
- typename T3 = void>
-struct m_Intrinsic_Ty;
-template <typename T0> struct m_Intrinsic_Ty<T0> {
- using Ty = match_combine_and<IntrinsicID_match, Argument_match<T0>>;
-};
-template <typename T0, typename T1> struct m_Intrinsic_Ty<T0, T1> {
- using Ty =
- match_combine_and<typename m_Intrinsic_Ty<T0>::Ty, Argument_match<T1>>;
-};
-template <typename T0, typename T1, typename T2>
-struct m_Intrinsic_Ty<T0, T1, T2> {
- using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1>::Ty,
- Argument_match<T2>>;
-};
-template <typename T0, typename T1, typename T2, typename T3>
-struct m_Intrinsic_Ty {
- using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2>::Ty,
- Argument_match<T3>>;
-};
-
-/// Match intrinsic calls like this:
-/// m_Intrinsic<Intrinsic::fabs>(m_VPValue(X), ...)
-template <Intrinsic::ID IntrID> inline IntrinsicID_match m_Intrinsic() {
- return IntrinsicID_match(IntrID);
-}
-
/// Match intrinsic calls with a runtime intrinsic ID.
inline IntrinsicID_match m_Intrinsic(Intrinsic::ID IntrID) {
return IntrinsicID_match(IntrID);
}
-template <Intrinsic::ID IntrID, typename T0>
-inline typename m_Intrinsic_Ty<T0>::Ty m_Intrinsic(const T0 &Op0) {
- return m_CombineAnd(m_Intrinsic<IntrID>(), m_Argument<0>(Op0));
-}
-
-template <Intrinsic::ID IntrID, typename T0, typename T1>
-inline typename m_Intrinsic_Ty<T0, T1>::Ty m_Intrinsic(const T0 &Op0,
- const T1 &Op1) {
- return m_CombineAnd(m_Intrinsic<IntrID>(Op0), m_Argument<1>(Op1));
-}
-
-template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2>
-inline typename m_Intrinsic_Ty<T0, T1, T2>::Ty
-m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2) {
- return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1), m_Argument<2>(Op2));
-}
+struct IntrinsicMatchImpl {
+ template <Intrinsic::ID IntrID, typename... Ts, size_t... Is>
+ static auto impl(std::index_sequence<Is...>, const Ts &...Ops) {
+ return m_CombineAnd(IntrinsicID_match(IntrID), m_Argument<Is>(Ops)...);
+ }
+};
-template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2,
- typename T3>
-inline typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty
-m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3) {
- return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2), m_Argument<3>(Op3));
+/// Match intrinsic calls like this:
+/// m_Intrinsic<Intrinsic::fabs>(m_VPValue(X), ...)
+template <Intrinsic::ID IntrID, typename... Ts>
+inline auto m_Intrinsic(const Ts &...Ops) {
+ return IntrinsicMatchImpl::impl<IntrID>(
+ std::make_index_sequence<sizeof...(Ts)>{}, Ops...);
}
template <Intrinsic::ID IntrID, typename... T>
>From 76d253e2736c7da493a836c9c150772056dafd73 Mon Sep 17 00:00:00 2001
From: Krzysztof Drewniak <Krzysztof.Drewniak at amd.com>
Date: Mon, 6 Jul 2026 11:36:40 -0700
Subject: [PATCH 38/46] [docs][AMDGPU] Document amdgpu.buffer.oob.mode more,
especially on fat pointers (#134734)
This commit expands the documentation of the amdgpu.buffer.oob.mode so
that fat buffer pointer users are more aware of its effects.
---
llvm/docs/AMDGPUUsage.rst | 52 +++++++++++++++++++++++++++++++++++++--
1 file changed, 50 insertions(+), 2 deletions(-)
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 0129c5e4f8994..2b3d396965f7a 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -876,11 +876,14 @@ consumed by the AMDGPU backend during code generation.
semantics. This is an alias of **strict** that is allowed to link
with any other module. Code generation is identical to **strict**.
- ``1``: **relaxed**. The backend may merge misaligned buffer
- accesses for performance, even if that changes OOB behaviour.
+ accesses for performance, even if that changes OOB behaviour, and will
+ not split vector accesses that may produce unexpected OOB accesses.
- ``2``: **strict**. The backend preserves per-byte OOB guarantees
by preventing merging of misaligned buffer accesses that could
straddle an OOB boundary (e.g. as required by Vulkan
- ``robustBufferAccess2``).
+ ``robustBufferAccess2``) and by splitting vector accesses to buffer
+ fat pointers that may produce incorrect results under a robust buffer
+ access scheme. See :ref:`amdgpu-fat-buffer-oob-handling` for details.
* - ``amdgpu.tbuffer.oob.mode``
- ``i32``
@@ -1356,6 +1359,51 @@ instruction with ``one-as`` sync scope is specified via
other operations within the same address space.
======================= ===================================================
+.. _amdgpu-fat-buffer-oob-handling:
+
+Buffer Fat Pointer out of bounds (OOB) handling
+-----------------------------------------------
+
+Instructions that load from or store to buffer resources (and thus, by extension
+buffer fat pointers and buffer strided pointers) generally implement handling for
+out of bounds (OOB) memory accesses, including those that are partially OOB,
+if the buffer resource resource has the required flags set. How ordinary
+``load`` and ``store`` instructions are lowered to buffer operations is partly
+controlled by the ``amdgpu.buffer.oob.mode`` (see :ref:`amdgpu-module-flags`). If
+that flag is set to ``1`` (relaxed), no handling to improve the expected behavior
+of OOB accesses is performed, while if it is set to ``0`` (any) or ``2`` (strict),
+operations may be split to ensure correctness under stronger models of how
+out-of-bounds accesses should behave.
+
+When operating on more than 32 bits of data, the ``voffset`` used for the access
+will be range-checked for each 32-bit word independently. This check uses saturating
+arithmetic and interprets the offset as an unsigned value.
+
+The behavior described above conflicts with the ABI requirements of certain graphics
+APIs that require out of bounds accesses to be handled strictly so that accessed
+that begin out of bounds but then access in-bounds elements (such as loading a
+``<4 x i32>`` beginning at offset ``-4``) still load the three in-bounds integers
+(producing ``<0, v0, v1, v2>`` and not ``<0, 0, 0, 0>``. So, under strict OOB
+handling, such an access will be split into four ``i32`` accesses. Note this this
+can only happen for underaligned loads - such wraparound isn't possible for
+loads that are alligned to their natural size.
+
+Similarly, buffer fat pointers permit operating on types such as ``<8 x i8>`` which
+must be accessed (and bounds-checked) 4 bytes at a time. Non-word-aligned
+accesses to such types from near the end of a buffer resource (such as starting
+a load of an ``<8 x i8>`` from an offset of ``6`` on an 8-byte buffer) will treat
+the initial two bytes to be loaded/stored as out of bounds, even though, under
+a strict interpretation of the bounds-checking semantics, they would be in bounds.
+Under strict OOB handling, such a load will be split into a sequence of ``<2 x i16>``
+loads.
+
+.. note::
+
+ No attempt will be made to shrink buffer intrinsic calls (calling
+ ``llvm.amdgcn.raw.ptr.buffer.*`` directly) in order to cause them to meet the
+ requirements of strict OOB mode. However, in strict OOB mode, those intrinsics
+ will not be combined in a way that would violate the guarantees of that mode.
+
Target Types
------------
>From c8c20662e036099b00469bf26324b93151acb00c Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Mon, 6 Jul 2026 20:37:15 +0200
Subject: [PATCH 39/46] ValueTracking: Improve frexp known range from
dominating conditions (#206927)
Try to restrict the known range of the exponent result of llvm.frexp
based on dominating conditions. Identify comparisons that imply the
incoming value cannot introduce an overflow in a downstream ldexp
use. This pattern appears in the implementation of some complex math
functions and allows finite only math to prune out more edge case
paths.
One attributor test for ldexp regresses due to the switch from
computeKnownBits to computeConstantRange. computeConstantRange
does not try to handle non-splat vector constants for the binary
operators.
As a side effect, this also improves knowing that ldexp can't
introduce overflow for the 0 case.
---
llvm/include/llvm/Support/KnownFPClass.h | 12 +-
llvm/lib/Analysis/ValueTracking.cpp | 70 +++++++++--
llvm/lib/Support/KnownFPClass.cpp | 19 ++-
.../Transforms/Attributor/nofpclass-ldexp.ll | 29 ++++-
...ed-exponent-range-dominating-conditions.ll | 116 +++++-------------
.../simplify-demanded-fpclass-ldexp.ll | 14 +++
6 files changed, 154 insertions(+), 106 deletions(-)
diff --git a/llvm/include/llvm/Support/KnownFPClass.h b/llvm/include/llvm/Support/KnownFPClass.h
index 7e018d25d2d1d..956141b9449a2 100644
--- a/llvm/include/llvm/Support/KnownFPClass.h
+++ b/llvm/include/llvm/Support/KnownFPClass.h
@@ -20,6 +20,7 @@
namespace llvm {
class APFloat;
+class APInt;
struct fltSemantics;
struct KnownBits;
@@ -448,10 +449,17 @@ struct KnownFPClass {
static LLVM_ABI KnownFPClass frexp_mant(
const KnownFPClass &Src, DenormalMode Mode = DenormalMode::getDynamic());
- /// Propagate known class for ldexp
+ /// Propagate known class for ldexp, assuming the exponent is known to be
+ /// within [\p ConstantRangeMin, \p ConstantRangeMax]
+ ///
+ // TODO: This really ought to use ConstantRange, but it's in IR not Support.
static LLVM_ABI KnownFPClass
- ldexp(const KnownFPClass &Src, const KnownBits &N, const fltSemantics &Flt,
+ ldexp(const KnownFPClass &Src, const APInt &ConstantRangeMin,
+ const APInt &ConstantRangeMax, const fltSemantics &Flt,
DenormalMode Mode = DenormalMode::getDynamic());
+ static LLVM_ABI KnownFPClass ldexp(
+ const KnownFPClass &Src, const KnownBits &ExpBits,
+ const fltSemantics &Flt, DenormalMode Mode = DenormalMode::getDynamic());
/// Propagate known class for powi
static LLVM_ABI KnownFPClass powi(const KnownFPClass &Src,
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 7dd23f24dfcc7..5869ab28c6226 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -4867,6 +4867,49 @@ static void computeKnownFPClassFromCond(const Value *V, Value *Cond,
}
}
+/// Compute the minimum and maximum values (inclusive) for the exponent of \p V,
+/// assuming it is not nan.
+static std::pair<int, int>
+computeKnownExponentRangeFromContext(const Value *V, const SimplifyQuery &Q) {
+ if (!Q.CxtI || !Q.DC || !Q.DT)
+ return {APFloat::IEK_NaN, APFloat::IEK_Inf};
+
+ for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
+ CmpPredicate Pred;
+ const APFloat *LimitC;
+ if (!match(BI->getCondition(),
+ m_FCmp(Pred, m_FAbs(m_Specific(V)), m_Finite(LimitC))))
+ continue;
+
+ if (Pred == FCmpInst::FCMP_ORD || Pred == FCmpInst::FCMP_UNO ||
+ Pred == FCmpInst::FCMP_TRUE || Pred == FCmpInst::FCMP_FALSE)
+ continue;
+
+ APFloat::cmpResult CmpOne =
+ LimitC->compare(APFloat::getOne(LimitC->getSemantics()));
+ if (CmpOne > APFloat::cmpEqual)
+ continue;
+
+ // If fabs(x) <= K, K <= 1.0 => exponent min exp range
+ // if fabs(x) >= K, K <= 1.0 swap the successor
+ bool IsLessEqual =
+ Pred == FCmpInst::FCMP_OLT || Pred == FCmpInst::FCMP_OLE ||
+ Pred == FCmpInst::FCMP_ULT || Pred == FCmpInst::FCMP_ULE ||
+ Pred == FCmpInst::FCMP_OEQ || Pred == FCmpInst::FCMP_UEQ;
+
+ BasicBlockEdge Edge1(BI->getParent(),
+ BI->getSuccessor(IsLessEqual ? 0 : 1));
+ if (Q.DT->dominates(Edge1, Q.CxtI->getParent())) {
+ int Exp = ilogb(*LimitC);
+
+ // TODO: Figure out lower bound to detect no-underflow.
+ return {APFloat::IEK_NaN, Exp};
+ }
+ }
+
+ return {APFloat::IEK_NaN, APFloat::IEK_Inf};
+}
+
static KnownFPClass computeKnownFPClassFromContext(const Value *V,
const SimplifyQuery &Q) {
KnownFPClass KnownFromContext;
@@ -4990,6 +5033,7 @@ static constexpr KnownFPClass::MinMaxKind getMinMaxKind(Intrinsic::ID IID) {
static bool isAbsoluteValueULEOne(const Value *V) {
// TODO: Handle frexp
// TODO: Other rounding intrinsics?
+ // TODO: Try computeKnownExponentRangeFromContext
// fabs(x - floor(x)) <= 1
const Value *SubFloorX;
@@ -5507,11 +5551,12 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
// Can refine inf/zero handling based on the exponent operand.
const FPClassTest ExpInfoMask = fcZero | fcSubnormal | fcInf;
- KnownBits ExpBits;
- if ((KnownSrc.KnownFPClasses & ExpInfoMask) != fcNone) {
- const Value *ExpArg = II->getArgOperand(1);
- ExpBits = computeKnownBits(ExpArg, DemandedElts, Q, Depth + 1);
- }
+ const Value *ExpArg = II->getArgOperand(1);
+ ConstantRange ExpKnownRange =
+ ((KnownSrc.KnownFPClasses & ExpInfoMask) != fcNone)
+ ? computeConstantRange(ExpArg, /*ForSigned=*/true, Q, Depth + 1)
+ : ConstantRange::getFull(
+ ExpArg->getType()->getScalarSizeInBits());
const fltSemantics &Flt =
II->getType()->getScalarType()->getFltSemantics();
@@ -5520,7 +5565,8 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
DenormalMode Mode =
F ? F->getDenormalMode(Flt) : DenormalMode::getDynamic();
- Known = KnownFPClass::ldexp(KnownSrc, ExpBits, Flt, Mode);
+ Known = KnownFPClass::ldexp(KnownSrc, ExpKnownRange.getSignedMin(),
+ ExpKnownRange.getSignedMax(), Flt, Mode);
break;
}
case Intrinsic::arithmetic_fence: {
@@ -10474,9 +10520,17 @@ ConstantRange llvm::computeConstantRange(const Value *V, bool ForSigned,
MinExp -= (APFloat::semanticsPrecision(FltSem) - 1);
int MaxExp = APFloat::semanticsMaxExponent(FltSem) + 1;
+
+ auto [AdjustedMin, AdjustedMax] =
+ computeKnownExponentRangeFromContext(FrexpSrc, SQ);
+
+ MinExp = std::max(AdjustedMin, MinExp);
+ MaxExp = std::min(AdjustedMax, MaxExp);
+
CR = ConstantRange::getNonEmpty(
- APInt(BitWidth, MinExp, /*isSigned=*/true),
- APInt(BitWidth, MaxExp + 1, /*isSigned=*/true));
+ APInt(BitWidth, static_cast<int64_t>(MinExp), /*isSigned=*/true),
+ APInt(BitWidth, static_cast<int64_t>(MaxExp) + 1,
+ /*isSigned=*/true));
}
}
}
diff --git a/llvm/lib/Support/KnownFPClass.cpp b/llvm/lib/Support/KnownFPClass.cpp
index 38762f05d74ac..4fb1d4cd4723e 100644
--- a/llvm/lib/Support/KnownFPClass.cpp
+++ b/llvm/lib/Support/KnownFPClass.cpp
@@ -789,7 +789,8 @@ KnownFPClass KnownFPClass::frexp_mant(const KnownFPClass &KnownSrc,
}
KnownFPClass KnownFPClass::ldexp(const KnownFPClass &KnownSrc,
- const KnownBits &ExpBits,
+ const APInt &ConstantRangeExpMin,
+ const APInt &ConstantRangeExpMax,
const fltSemantics &Flt, DenormalMode Mode) {
KnownFPClass Known;
Known.propagateNaN(KnownSrc, /*PropagateSign=*/true);
@@ -807,20 +808,19 @@ KnownFPClass KnownFPClass::ldexp(const KnownFPClass &KnownSrc,
unsigned Precision = APFloat::semanticsPrecision(Flt);
const int MantissaBits = Precision - 1;
-
- if (ExpBits.getSignedMinValue().sge(static_cast<int64_t>(MantissaBits)))
+ if (ConstantRangeExpMin.sge(MantissaBits))
Known.knownNot(fcSubnormal);
- if (ExpBits.isConstant() && ExpBits.getConstant().isZero()) {
+ if (ConstantRangeExpMin.isZero() && ConstantRangeExpMax.isZero()) {
// ldexp(x, 0) -> x, so propagate everything.
Known.propagateCanonicalizingSrc(KnownSrc, Mode);
- } else if (ExpBits.isNegative()) {
+ } else if (ConstantRangeExpMax.isNonPositive()) {
// If we know the power is <= 0, can't introduce inf
if (KnownSrc.isKnownNeverPosInfinity())
Known.knownNot(fcPosInf);
if (KnownSrc.isKnownNeverNegInfinity())
Known.knownNot(fcNegInf);
- } else if (ExpBits.isNonNegative()) {
+ } else if (ConstantRangeExpMin.isNonNegative()) {
// If we know the power is >= 0, can't introduce subnormal or zero
if (KnownSrc.isKnownNeverPosSubnormal())
Known.knownNot(fcPosSubnormal);
@@ -835,6 +835,13 @@ KnownFPClass KnownFPClass::ldexp(const KnownFPClass &KnownSrc,
return Known;
}
+KnownFPClass KnownFPClass::ldexp(const KnownFPClass &KnownSrc,
+ const KnownBits &ExpBits,
+ const fltSemantics &Flt, DenormalMode Mode) {
+ return ldexp(KnownSrc, ExpBits.getSignedMinValue(),
+ ExpBits.getSignedMaxValue(), Flt, Mode);
+}
+
KnownFPClass KnownFPClass::powi(const KnownFPClass &KnownSrc,
const KnownBits &ExponentKnownBits) {
KnownFPClass Known;
diff --git a/llvm/test/Transforms/Attributor/nofpclass-ldexp.ll b/llvm/test/Transforms/Attributor/nofpclass-ldexp.ll
index 1d8c7af63c4fe..630e510a12d72 100644
--- a/llvm/test/Transforms/Attributor/nofpclass-ldexp.ll
+++ b/llvm/test/Transforms/Attributor/nofpclass-ldexp.ll
@@ -701,12 +701,25 @@ define <2 x float> @ret_ldexp_v2f32_known_pos_exp_noinf(<2 x float> nofpclass(in
ret <2 x float> %call
}
-define <2 x float> @ret_ldexp_v2f32_known_neg_exp_noinf(<2 x float> nofpclass(inf) %arg0, <2 x i32> %arg1) #0 {
-; CHECK-LABEL: define nofpclass(inf) <2 x float> @ret_ldexp_v2f32_known_neg_exp_noinf
+define <2 x float> @ret_ldexp_v2f32_known_neg_exp_noinf_splat(<2 x float> nofpclass(inf) %arg0, <2 x i32> %arg1) #0 {
+; CHECK-LABEL: define nofpclass(inf) <2 x float> @ret_ldexp_v2f32_known_neg_exp_noinf_splat
; CHECK-SAME: (<2 x float> nofpclass(inf) [[ARG0:%.*]], <2 x i32> [[ARG1:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT: [[OR_ARG1:%.*]] = or <2 x i32> [[ARG1]], <i32 -16, i32 -32>
+; CHECK-NEXT: [[OR_ARG1:%.*]] = or <2 x i32> [[ARG1]], splat (i32 -32)
; CHECK-NEXT: [[CALL:%.*]] = call nofpclass(inf) <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> nofpclass(inf) [[ARG0]], <2 x i32> [[OR_ARG1]]) #[[ATTR10]]
; CHECK-NEXT: ret <2 x float> [[CALL]]
+;
+ %or.arg1 = or <2 x i32> %arg1, splat (i32 -32)
+ %call = call <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> %arg0, <2 x i32> %or.arg1)
+ ret <2 x float> %call
+}
+
+; TODO: computeConstantRange does not handle non-splat operator constants.
+define <2 x float> @ret_ldexp_v2f32_known_neg_exp_noinf_nonsplat(<2 x float> nofpclass(inf) %arg0, <2 x i32> %arg1) #0 {
+; CHECK-LABEL: define <2 x float> @ret_ldexp_v2f32_known_neg_exp_noinf_nonsplat
+; CHECK-SAME: (<2 x float> nofpclass(inf) [[ARG0:%.*]], <2 x i32> [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[OR_ARG1:%.*]] = or <2 x i32> [[ARG1]], <i32 -16, i32 -32>
+; CHECK-NEXT: [[CALL:%.*]] = call <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> nofpclass(inf) [[ARG0]], <2 x i32> [[OR_ARG1]]) #[[ATTR10]]
+; CHECK-NEXT: ret <2 x float> [[CALL]]
;
%or.arg1 = or <2 x i32> %arg1, <i32 -16, i32 -32>
%call = call <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> %arg0, <2 x i32> %or.arg1)
@@ -1035,6 +1048,16 @@ define float @ret_ldexp_f32_neg127(float %arg0) #0 {
ret float %call
}
+define float @ret_ldexp_f32_noinf_exp_known_neg_or_0(float nofpclass(inf) %arg0, i32 range(i32 -256, 1) %arg1) #0 {
+; CHECK-LABEL: define nofpclass(inf) float @ret_ldexp_f32_noinf_exp_known_neg_or_0
+; CHECK-SAME: (float nofpclass(inf) [[ARG0:%.*]], i32 range(i32 -256, 1) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[CALL:%.*]] = call nofpclass(inf) float @llvm.ldexp.f32.i32(float nofpclass(inf) [[ARG0]], i32 [[ARG1]]) #[[ATTR10]]
+; CHECK-NEXT: ret float [[CALL]]
+;
+ %call = call float @llvm.ldexp.f32.i32(float %arg0, i32 %arg1)
+ ret float %call
+}
+
attributes #0 = { denormal_fpenv(ieee|ieee) }
attributes #1 = { denormal_fpenv(preservesign) }
attributes #2 = { denormal_fpenv(positivezero|positivezero) }
diff --git a/llvm/test/Transforms/InstCombine/frexp-implied-exponent-range-dominating-conditions.ll b/llvm/test/Transforms/InstCombine/frexp-implied-exponent-range-dominating-conditions.ll
index 505864cfcf35e..706e5da46879c 100644
--- a/llvm/test/Transforms/InstCombine/frexp-implied-exponent-range-dominating-conditions.ll
+++ b/llvm/test/Transforms/InstCombine/frexp-implied-exponent-range-dominating-conditions.ll
@@ -17,9 +17,7 @@ define float @frexp_fcmp_ogt_1(float nofpclass(nan inf) %x, float nofpclass(inf)
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[LARGE]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -49,9 +47,7 @@ define float @frexp_fcmp_oge_1(float nofpclass(nan inf) %x, float nofpclass(inf)
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[LARGE]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -81,9 +77,7 @@ define float @frexp_fcmp_ugt_1(float nofpclass(nan inf) %x, float nofpclass(inf)
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[LARGE]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -113,9 +107,7 @@ define float @frexp_fcmp_uge_1(float nofpclass(nan inf) %x, float nofpclass(inf)
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[LARGE]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -145,9 +137,7 @@ define float @frexp_fcmp_olt_1(float nofpclass(nan inf) %x, float nofpclass(inf)
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[LARGE]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -177,9 +167,7 @@ define float @frexp_fcmp_ole_1(float nofpclass(nan inf) %x, float nofpclass(inf)
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[LARGE]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -210,9 +198,7 @@ define float @frexp_fcmp_ult_1(float nofpclass(nan inf) %x, float nofpclass(inf)
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[LARGE]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -242,9 +228,7 @@ define float @frexp_fcmp_ule_1(float nofpclass(nan inf) %x, float nofpclass(inf)
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[LARGE]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -274,9 +258,7 @@ define float @frexp_cmp_ugt_0.5(float nofpclass(nan inf) %x, float nofpclass(inf
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[LARGE]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -306,9 +288,7 @@ define float @frexp_cmp_uge_0.5(float nofpclass(nan inf) %x, float nofpclass(inf
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[LARGE]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -338,9 +318,7 @@ define float @frexp_fcmp_ult_0.5(float nofpclass(nan inf) %x, float nofpclass(in
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[LARGE]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -370,9 +348,7 @@ define float @frexp_fcmp_ule_0.5(float nofpclass(nan inf) %x, float nofpclass(in
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[LARGE]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -402,9 +378,7 @@ define float @frexp_fcmp_ugt_0.5(float nofpclass(nan inf) %x, float nofpclass(in
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[LARGE]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -434,9 +408,7 @@ define float @frexp_fcmp_uge_0.5(float nofpclass(nan inf) %x, float nofpclass(in
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[LARGE]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -466,9 +438,7 @@ define float @frexp_fcmp_oeq_1(float nofpclass(nan inf) %x, float nofpclass(inf)
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[LARGE]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -498,9 +468,7 @@ define float @frexp_fcmp_ueq_1(float nofpclass(nan inf) %x, float nofpclass(inf)
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[LARGE]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -530,9 +498,7 @@ define float @frexp_fcmp_one_1(float nofpclass(nan inf) %x, float nofpclass(inf)
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[LARGE]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -562,9 +528,7 @@ define float @frexp_fcmp_une_1(float nofpclass(nan inf) %x, float nofpclass(inf)
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[LARGE]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -594,9 +558,7 @@ define float @frexp_fcmp_oeq_0.5(float nofpclass(nan inf) %x, float nofpclass(in
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[LARGE]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -626,9 +588,7 @@ define float @frexp_fcmp_ueq_0.5(float nofpclass(nan inf) %x, float nofpclass(in
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[LARGE]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -658,9 +618,7 @@ define float @frexp_fcmp_one_0.5(float nofpclass(nan inf) %x, float nofpclass(in
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[LARGE]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -690,9 +648,7 @@ define float @frexp_fcmp_une_0.5(float nofpclass(nan inf) %x, float nofpclass(in
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[LARGE]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -899,9 +855,7 @@ define float @frexp_fcmp_ogt_nextdown1(float nofpclass(nan inf) %x, float nofpcl
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[LARGE]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -932,9 +886,7 @@ define float @frexp_fcmp_ole_nextdown1(float nofpclass(nan inf) %x, float nofpcl
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[LARGE]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -1066,9 +1018,7 @@ define float @frexp_fcmp_olt_1_multiple_conditions_wrong_constant(float nofpclas
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -1106,9 +1056,7 @@ define float @frexp_fcmp_olt_1_multiple_conditions_nonfinite_constant(float nofp
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -1144,9 +1092,7 @@ define float @frexp_fcmp_olt_1_multiple_conditions_edge_case_compare(float nofpc
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -1184,9 +1130,7 @@ define float @frexp_fcmp_olt_1_multiple_conditions_other(float nofpclass(nan inf
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i32 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i32(float [[NOT_INF]], i32 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret float 0.000000e+00
;
@@ -1221,9 +1165,7 @@ define float @frexp_fcmp_ogt_1_large_int(float nofpclass(nan inf) %x, float nofp
; CHECK-NEXT: [[FREXP:%.*]] = call { float, i128 } @llvm.frexp.f32.i128(float [[X]])
; CHECK-NEXT: [[EXP:%.*]] = extractvalue { float, i128 } [[FREXP]], 1
; CHECK-NEXT: [[SCALED:%.*]] = call float @llvm.ldexp.f32.i128(float [[NOT_INF]], i128 [[EXP]])
-; CHECK-NEXT: [[IS_INF:%.*]] = fcmp oeq float [[SCALED]], +inf
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[IS_INF]], float 0.000000e+00, float [[SCALED]]
-; CHECK-NEXT: ret float [[SELECT]]
+; CHECK-NEXT: ret float [[SCALED]]
; CHECK: [[LARGE]]:
; CHECK-NEXT: ret float 0.000000e+00
;
diff --git a/llvm/test/Transforms/InstCombine/simplify-demanded-fpclass-ldexp.ll b/llvm/test/Transforms/InstCombine/simplify-demanded-fpclass-ldexp.ll
index 0ca11d09b56de..f63f05c49993e 100644
--- a/llvm/test/Transforms/InstCombine/simplify-demanded-fpclass-ldexp.ll
+++ b/llvm/test/Transforms/InstCombine/simplify-demanded-fpclass-ldexp.ll
@@ -487,3 +487,17 @@ define nofpclass(snan) float @qnan_result_demands_snan_src(i1 %cond, float %unkn
%ldexp = call float @llvm.ldexp.f32.i32(float %select, i32 %unknown.int)
ret float %ldexp
}
+
+; Make sure there's no assertion from trying to query
+; getIntegerBitWidth on the vector type.
+define <2 x float> @ldexp_v2f32_v2i32_fmul_crash(<2 x i32> %exp) {
+; CHECK-LABEL: define <2 x float> @ldexp_v2f32_v2i32_fmul_crash(
+; CHECK-SAME: <2 x i32> [[EXP:%.*]]) {
+; CHECK-NEXT: [[LDEXP:%.*]] = call nnan <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> splat (float 1.000000e+00), <2 x i32> [[EXP]])
+; CHECK-NEXT: [[MUL:%.*]] = fmul <2 x float> [[LDEXP]], zeroinitializer
+; CHECK-NEXT: ret <2 x float> [[MUL]]
+;
+ %ldexp = call <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> splat (float 1.0), <2 x i32> %exp)
+ %mul = fmul <2 x float> %ldexp, zeroinitializer
+ ret <2 x float> %mul
+}
>From dbd7511ad28819587fc9effc0086abf437279a84 Mon Sep 17 00:00:00 2001
From: Krzysztof Drewniak <Krzysztof.Drewniak at amd.com>
Date: Mon, 6 Jul 2026 11:38:46 -0700
Subject: [PATCH 40/46] [LowerBufferFatPointers] Correctly handle alignment
modes (#134329)
Previously, AMDGPULowerBufferFatPointers would emit unaligned buffer
loads/stores, even when such unaligned accesses were disabled (that is,
on non-HSA platforms).
In addition, the lowering did not respect the newly-added
amdgpu.buffer.oob.mode module flag, which now must be enabled in order
to vectorize unaligned loads from buffers.
This commit fixes both issues and adds tests.
---
.../AMDGPU/AMDGPULowerBufferFatPointers.cpp | 352 +-
.../buffer-fat-pointer-atomicrmw-fadd.ll | 2608 ++++++++-----
.../buffer-fat-pointer-atomicrmw-fmax.ll | 2134 +++++++----
.../buffer-fat-pointer-atomicrmw-fmin.ll | 2134 +++++++----
...ffer-fat-pointers-contents-legalization.ll | 24 +-
.../AMDGPU/buffer-fat-pointers-memcpy.ll | 20 +-
llvm/test/CodeGen/AMDGPU/llc-pipeline.ll | 20 +
.../AMDGPU/lower-buffer-fat-pointers-calls.ll | 2 +-
...ointers-contents-legalization-alignment.ll | 3280 +++++++++++++++++
...ffer-fat-pointers-contents-legalization.ll | 141 +-
.../lower-buffer-fat-pointers-mem-transfer.ll | 2010 +++++++---
...fer-fat-pointers-unoptimized-debug-data.ll | 8 +-
llvm/test/CodeGen/AMDGPU/memset-pattern.ll | 113 +-
13 files changed, 10077 insertions(+), 2769 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization-alignment.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
index f74f6f1baa576..dd2f7075c8df2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
@@ -225,6 +225,8 @@
#include "llvm/ADT/SetOperations.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Analysis/InstSimplifyFolder.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/Utils/Local.h"
#include "llvm/CodeGen/TargetPassConfig.h"
@@ -241,6 +243,7 @@
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Operator.h"
+#include "llvm/IR/PassManager.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/ReplaceConstant.h"
#include "llvm/IR/ValueHandle.h"
@@ -251,6 +254,7 @@
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/MathExtras.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
@@ -260,6 +264,9 @@
using namespace llvm;
+using GetTTIFn = function_ref<const TargetTransformInfo *(Function &)>;
+using GetSEFn = function_ref<ScalarEvolution *(Function &)>;
+
static constexpr unsigned BufferOffsetWidth = 32;
namespace {
@@ -432,7 +439,9 @@ class StoreFatPtrsAsIntsAndExpandMemcpyVisitor
IRBuilder<InstSimplifyFolder> IRB;
- const TargetMachine *TM;
+ // Used for memcpy() lowering.
+ const TargetTransformInfo *TTI;
+ ScalarEvolution *SE;
// Convert all the buffer fat pointers within the input value to inttegers
// so that it can be stored in memory.
@@ -445,10 +454,10 @@ class StoreFatPtrsAsIntsAndExpandMemcpyVisitor
public:
StoreFatPtrsAsIntsAndExpandMemcpyVisitor(BufferFatPtrToIntTypeMap *TypeMap,
const DataLayout &DL,
- LLVMContext &Ctx,
- const TargetMachine *TM)
- : TypeMap(TypeMap), IRB(Ctx, InstSimplifyFolder(DL)), TM(TM) {}
- bool processFunction(Function &F);
+ LLVMContext &Ctx)
+ : TypeMap(TypeMap), IRB(Ctx, InstSimplifyFolder(DL)) {}
+ bool processFunction(Function &F, const TargetTransformInfo *TTI,
+ ScalarEvolution *SE);
bool visitInstruction(Instruction &I) { return false; }
bool visitAllocaInst(AllocaInst &I);
@@ -534,7 +543,10 @@ Value *StoreFatPtrsAsIntsAndExpandMemcpyVisitor::intsToFatPtrs(
return Ret;
}
-bool StoreFatPtrsAsIntsAndExpandMemcpyVisitor::processFunction(Function &F) {
+bool StoreFatPtrsAsIntsAndExpandMemcpyVisitor::processFunction(
+ Function &F, const TargetTransformInfo *TTI, ScalarEvolution *SE) {
+ this->TTI = TTI;
+ this->SE = SE;
bool Changed = false;
// Process memcpy-like instructions after the main iteration because they can
// invalidate iterators.
@@ -549,6 +561,8 @@ bool StoreFatPtrsAsIntsAndExpandMemcpyVisitor::processFunction(Function &F) {
Changed |= visit(cast<Instruction>(VH));
}
ConvertedForStore.clear();
+ this->TTI = nullptr;
+ this->SE = nullptr;
return Changed;
}
@@ -615,8 +629,7 @@ bool StoreFatPtrsAsIntsAndExpandMemcpyVisitor::visitMemCpyInst(
if (MCI.getSourceAddressSpace() != AMDGPUAS::BUFFER_FAT_POINTER &&
MCI.getDestAddressSpace() != AMDGPUAS::BUFFER_FAT_POINTER)
return false;
- llvm::expandMemCpyAsLoop(&MCI,
- TM->getTargetTransformInfo(*MCI.getFunction()));
+ llvm::expandMemCpyAsLoop(&MCI, *TTI, SE);
MCI.eraseFromParent();
return true;
}
@@ -635,8 +648,7 @@ bool StoreFatPtrsAsIntsAndExpandMemcpyVisitor::visitMemSetInst(
MemSetInst &MSI) {
if (MSI.getDestAddressSpace() != AMDGPUAS::BUFFER_FAT_POINTER)
return false;
- llvm::expandMemSetAsLoop(&MSI,
- TM->getTargetTransformInfo(*MSI.getFunction()));
+ llvm::expandMemSetAsLoop(&MSI, TTI);
MSI.eraseFromParent();
return true;
}
@@ -645,8 +657,7 @@ bool StoreFatPtrsAsIntsAndExpandMemcpyVisitor::visitMemSetPatternInst(
MemSetPatternInst &MSPI) {
if (MSPI.getDestAddressSpace() != AMDGPUAS::BUFFER_FAT_POINTER)
return false;
- llvm::expandMemSetPatternAsLoop(
- &MSPI, TM->getTargetTransformInfo(*MSPI.getFunction()));
+ llvm::expandMemSetPatternAsLoop(&MSPI, *TTI);
MSPI.eraseFromParent();
return true;
}
@@ -667,6 +678,11 @@ namespace {
/// Note that this doesn't handle complex data strucures, but, in the future,
/// the aggregate load splitter from SROA could be refactored to allow for that
/// case.
+///
+/// Note that, if we can prove that the initial value of the pointer offset is 0
+/// and that the load/store won't wrap from the left or won't have bounds checks
+/// that straddle a word boundary, we can emit some of the strict bounds
+/// checking pessimizations even in strict OOB mode, and we attempt to do so.
class LegalizeBufferContentTypesVisitor
: public InstVisitor<LegalizeBufferContentTypesVisitor, bool> {
friend class InstVisitor<LegalizeBufferContentTypesVisitor, bool>;
@@ -675,18 +691,73 @@ class LegalizeBufferContentTypesVisitor
const DataLayout &DL;
+ ScalarEvolution *SE = nullptr;
+
+ // Map base (non-GEP'd) pointers to the number of records they have, if known.
+ // If a pointer is known to have a starting offset of 0 but it wasn't known to
+ // have a number of records (ex. it was `addrspacecast` from a buffer
+ // resource), it will be present in this map, but the key will be null.
+ // Otherwise, there will be no map entry.
+ ValueToValueMapTy ZeroBasePointerToNumRecords;
+
+ // Subtarget info, needed for determining what cache control bits to set.
+ const TargetMachine *TM;
+ const GCNSubtarget *ST = nullptr;
+
/// If T is [N x U], where U is a scalar type, return the vector type
/// <N x U>, otherwise, return T.
Type *scalarArrayTypeAsVector(Type *MaybeArrayType);
Value *arrayToVector(Value *V, Type *TargetType, const Twine &Name);
Value *vectorToArray(Value *V, Type *OrigType, const Twine &Name);
+ /// Analyze how a given buffer access could be out of bounds. Used to optimize
+ /// the strict splitting used in strict bounds checking mode.
+ struct OobProperties {
+ // Offset is far enough from all-1s that we won't get wrapping around to 0.
+ bool NoWrapFromMax = false;
+ // Offset is either entirely in-bounds or entirely out of bounds.
+ bool NoPartialOOB = false;
+
+ OobProperties() = delete;
+ // Needed for some Clangs.
+ OobProperties(bool NoWrapFromMax, bool NoPartialOOB)
+ : NoWrapFromMax(NoWrapFromMax), NoPartialOOB(NoPartialOOB) {}
+ };
+ OobProperties analyzeOobProperties(Value *Ptr, Type *Ty, uint64_t ByteOffset);
+
/// Break up the loads of a struct into the loads of its components
+ /// Return the maximum allowed load/store width for the given type and
+ /// alignment combination based on subtarget flags.
+ /// 1. If unaligned accesses are not enabled, then any load/store that is less
+ /// than word-aligned has to be handled one byte or ushort at a time.
+ /// 2. If relaxed OOB mode is not set, we must ensure that the in-bounds
+ /// part of a partially out of bounds read/write is performed correctly. This
+ /// means that any load that isn't naturally aligned has to be split into
+ /// parts that are naturally aligned, so that, after bitcasting, we don't have
+ /// unaligned loads that could discard valid data.
+ ///
+ /// For example, if we're loading a <8 x i8>, that's actually a load of a <2 x
+ /// i32>, and if we load from an align(2) address, that address might be 2
+ /// bytes from the end of the buffer. The hardware will, when performing the
+ /// <2 x i32> load, mask off the entire first word, causing the two in-bounds
+ /// bytes to be masked off. However,if we know the offset can't be too close
+ /// to the number of records in the buffer (if known), we can skip this
+ /// expansion.
+ ///
+ /// Unlike the complete disablement of unaligned accesses from point 1,
+ /// this does not apply to unaligned scalars, but will apply to cases like
+ /// `load <2 x i32>, align 4` since the left elemenvt might be out of bounds.
+ /// Note that if the we know that the base offset is known to be
+ /// less than `uint32_max - byte_size(Ty)`, we can skip these alignment
+ /// checks.
+ uint64_t maxIntrinsicWidth(Type *Ty, Align A, OobProperties OobProps);
+
/// Convert a vector or scalar type that can't be operated on by buffer
/// intrinsics to one that would be legal through bitcasts and/or truncation.
- /// Uses the wider of i32, i16, or i8 where possible.
- Type *legalNonAggregateFor(Type *T);
+ /// Uses the wider of i32, i16, or i8 where possible, clamping to the maximum
+ /// allowed width under the alignment rules and subtarget flags.
+ Type *legalNonAggregateForMemOp(Type *T, uint64_t MaxWidth);
Value *makeLegalNonAggregate(Value *V, Type *TargetType, const Twine &Name);
Value *makeIllegalNonAggregate(Value *V, Type *OrigType, const Twine &Name);
@@ -700,8 +771,9 @@ class LegalizeBufferContentTypesVisitor
/// Return the [index, length] pairs into which `T` needs to be cut to form
/// legal buffer load or store operations. Clears `Slices`. Creates an empty
/// `Slices` for non-vector inputs and creates one slice if no slicing will be
- /// needed.
- void getVecSlices(Type *T, SmallVectorImpl<VecSlice> &Slices);
+ /// needed. No slice may be larger than `MaxWidth`.
+ void getVecSlices(Type *T, uint64_t MaxWidth,
+ SmallVectorImpl<VecSlice> &Slices);
Value *extractSlice(Value *Vec, VecSlice S, const Twine &Name);
Value *insertSlice(Value *Whole, Value *Part, VecSlice S, const Twine &Name);
@@ -729,10 +801,15 @@ class LegalizeBufferContentTypesVisitor
bool visitLoadInst(LoadInst &LI);
bool visitStoreInst(StoreInst &SI);
+ // Record base pointer data and num_records (if known).
+ bool visitIntrinsicInst(IntrinsicInst &II);
+ bool visitAddrSpaceCastInst(AddrSpaceCastInst &ASCI);
+
public:
- LegalizeBufferContentTypesVisitor(const DataLayout &DL, LLVMContext &Ctx)
- : IRB(Ctx, InstSimplifyFolder(DL)), DL(DL) {}
- bool processFunction(Function &F);
+ LegalizeBufferContentTypesVisitor(const DataLayout &DL, LLVMContext &Ctx,
+ const TargetMachine *TM)
+ : IRB(Ctx, InstSimplifyFolder(DL)), DL(DL), TM(TM) {}
+ bool processFunction(Function &F, ScalarEvolution *SE);
};
} // namespace
@@ -778,9 +855,115 @@ Value *LegalizeBufferContentTypesVisitor::vectorToArray(Value *V,
return ArrayRes;
}
-Type *LegalizeBufferContentTypesVisitor::legalNonAggregateFor(Type *T) {
+LegalizeBufferContentTypesVisitor::OobProperties
+LegalizeBufferContentTypesVisitor::analyzeOobProperties(Value *Ptr, Type *Ty,
+ uint64_t ByteOffset) {
+ OobProperties Result(false, false);
+
+ if (ST->hasRelaxedBufferOOBMode())
+ return OobProperties(true, true);
+
+ if (!SE)
+ return Result;
+ if (!SE->isSCEVable(Ptr->getType()))
+ return Result;
+ const SCEV *PtrOp = SE->getSCEV(Ptr);
+ if (ByteOffset > 0)
+ PtrOp = SE->getAddExpr(PtrOp, SE->getConstant(IRB.getInt32(ByteOffset)));
+ const auto *PtrBase = dyn_cast<SCEVUnknown>(SE->getPointerBase(PtrOp));
+ if (!PtrBase)
+ return Result;
+ Value *PtrBaseVal = PtrBase->getValue();
+ // We don't know if the offset field started at 0, so there's no safe analysis
+ // we can do. If it weren't for the fact that nuw / inbounds / ... are
+ // properties of the pointer, we might be able to use hem, but loads where the
+ // address computation for sub-parts of the loaded type wraps the address
+ // space are explicitly in scope here so there's not much we can do inside
+ // functions that can't "see" the fat pointer creation.
+ auto NumRecordsIfKnown = ZeroBasePointerToNumRecords.find(PtrBaseVal);
+ if (NumRecordsIfKnown == ZeroBasePointerToNumRecords.end())
+ return Result;
+
+ unsigned TypeSize = DL.getTypeStoreSize(Ty).getKnownMinValue();
+ const SCEV *PtrDiff = SE->getMinusSCEV(PtrOp, PtrBase);
+ APInt MaxNoWrapOffset = APInt::getAllOnes(BufferOffsetWidth) - TypeSize;
+ if (SE->isKnownNonNegative(PtrDiff) ||
+ SE->getUnsignedRangeMax(PtrDiff).ule(MaxNoWrapOffset))
+ Result.NoWrapFromMax = true;
+
+ // If we know that the pointer is zero-based but not what its upper bound is,
+ // we'll need to split up underaligned loads of small types.
+ if (!NumRecordsIfKnown->second)
+ return Result;
+ const SCEV *NumRecords = SE->getSCEV(NumRecordsIfKnown->second);
+ // All-1s is (per ISA or as a consequence of the bonud)check rules, depending
+ // on arcihtecture) no bounds check.
+ if (NumRecords->isAllOnesValue())
+ Result.NoPartialOOB = true;
+
+ const SCEV *BoundsDiff;
+ if (ST->has45BitNumRecordsBufferResource()) {
+ const SCEV *PtrDiffExt =
+ SE->getNoopOrZeroExtend(PtrDiff, NumRecords->getType());
+ BoundsDiff = SE->getMinusSCEV(NumRecords, PtrDiffExt);
+ } else {
+ const SCEV *NumRecordsI32 =
+ SE->getTruncateOrNoop(NumRecords, IRB.getInt32Ty());
+ BoundsDiff = SE->getMinusSCEV(NumRecordsI32, PtrDiff);
+ }
+
+ if (SE->getSignedRangeMin(BoundsDiff).sge(TypeSize) ||
+ SE->isKnownNonPositive(BoundsDiff))
+ Result.NoPartialOOB = true;
+ return Result;
+}
+
+uint64_t
+LegalizeBufferContentTypesVisitor::maxIntrinsicWidth(Type *T, Align A,
+ OobProperties OobProps) {
+ Align Result(16);
+ if (!ST->hasUnalignedBufferAccessEnabled() && A < Align(4))
+ Result = A;
+ auto *VT = dyn_cast<VectorType>(T);
+ if (!ST->hasRelaxedBufferOOBMode() && VT) {
+ TypeSize ElemBits = DL.getTypeSizeInBits(VT->getElementType());
+ if (ElemBits.isKnownMultipleOf(32)) {
+ // Word-sized operations are bounds-checked per word. So, the only case we
+ // have to worry about is stores that start out of bounds and then go in,
+ // and those can only become in-bounds on a multiple of their alignment.
+ // Therefore, we can use the declared alignment of the operation as the
+ // maximum width, rounding up to 4.
+ if (!OobProps.NoWrapFromMax)
+ Result = std::min(Result, std::max(A, Align(4)));
+ } else if ((ElemBits.isKnownMultipleOf(8) ||
+ isPowerOf2_64(ElemBits.getKnownMinValue()))) {
+ // To ensure correct behavior for sub-word types, we must always scalarize
+ // unaligned loads of sub-word types. For example, if you load
+ // a <4 x i8> from offset 7 in an 8-byte buffer, expecting the vector
+ // to be padded out with 0s after that last byte, you'll get all 0s
+ // instead. To prevent this behavior when not requested, de-vectorize such
+ // loads.
+ //
+ // If we knew that the value that triggers bounds checks was a multiple of
+ // 4 along with the access being word-aligned, we could avoid the
+ // scalarization here, as the bitcast wouldn't change any check behavior,
+ // but we don't currently try to analyze this.
+ //
+ // Strict OOB checking isn't supported if the size of each element is a
+ // non-power-of-2 value less than 8, since there's no feasible way to
+ // apply such a strict bounds check.
+ if (!OobProps.NoPartialOOB)
+ Result =
+ commonAlignment(Result, divideCeil(ElemBits.getKnownMinValue(), 8));
+ }
+ }
+ return Result.value() * 8;
+}
+
+Type *LegalizeBufferContentTypesVisitor::legalNonAggregateForMemOp(
+ Type *T, uint64_t MaxWidth) {
TypeSize Size = DL.getTypeStoreSizeInBits(T);
- // Implicitly zero-extend to the next byte if needed
+ // Implicitly zero-extend to the next byte if needed.
if (!DL.typeSizeEqualsStoreSize(T))
T = IRB.getIntNTy(Size.getFixedValue());
Type *ElemTy = T->getScalarType();
@@ -790,15 +973,16 @@ Type *LegalizeBufferContentTypesVisitor::legalNonAggregateFor(Type *T) {
return T;
}
unsigned ElemSize = DL.getTypeSizeInBits(ElemTy).getFixedValue();
- if (isPowerOf2_32(ElemSize) && ElemSize >= 16 && ElemSize <= 128) {
+ if (isPowerOf2_32(ElemSize) && ElemSize >= 16 && ElemSize <= MaxWidth) {
// [vectors of] anything that's 16/32/64/128 bits can be cast and split into
- // legal buffer operations.
+ // legal buffer operations, except that we might need to cut them into
+ // smaller values if we're not allowed to do unaligned vector loads.
return T;
}
Type *BestVectorElemType = nullptr;
- if (Size.isKnownMultipleOf(32))
+ if (Size.isKnownMultipleOf(32) && MaxWidth >= 32)
BestVectorElemType = IRB.getInt32Ty();
- else if (Size.isKnownMultipleOf(16))
+ else if (Size.isKnownMultipleOf(16) && MaxWidth >= 16)
BestVectorElemType = IRB.getInt16Ty();
else
BestVectorElemType = IRB.getInt8Ty();
@@ -871,7 +1055,7 @@ Type *LegalizeBufferContentTypesVisitor::intrinsicTypeFor(Type *LegalType) {
}
void LegalizeBufferContentTypesVisitor::getVecSlices(
- Type *T, SmallVectorImpl<VecSlice> &Slices) {
+ Type *T, uint64_t MaxWidth, SmallVectorImpl<VecSlice> &Slices) {
Slices.clear();
auto *VT = dyn_cast<FixedVectorType>(T);
if (!VT)
@@ -892,8 +1076,8 @@ void LegalizeBufferContentTypesVisitor::getVecSlices(
uint64_t TotalElems = VT->getNumElements();
uint64_t Index = 0;
- auto TrySlice = [&](unsigned MaybeLen) {
- if (MaybeLen > 0 && Index + MaybeLen <= TotalElems) {
+ auto TrySlice = [&](unsigned MaybeLen, unsigned Width) {
+ if (MaybeLen > 0 && Width <= MaxWidth && Index + MaybeLen <= TotalElems) {
VecSlice Slice{/*Index=*/Index, /*Length=*/MaybeLen};
Slices.push_back(Slice);
Index += MaybeLen;
@@ -902,9 +1086,9 @@ void LegalizeBufferContentTypesVisitor::getVecSlices(
return false;
};
while (Index < TotalElems) {
- TrySlice(ElemsPer4Words) || TrySlice(ElemsPer3Words) ||
- TrySlice(ElemsPer2Words) || TrySlice(ElemsPerWord) ||
- TrySlice(ElemsPerShort) || TrySlice(ElemsPerByte);
+ TrySlice(ElemsPer4Words, 128) || TrySlice(ElemsPer3Words, 96) ||
+ TrySlice(ElemsPer2Words, 64) || TrySlice(ElemsPerWord, 32) ||
+ TrySlice(ElemsPerShort, 16) || TrySlice(ElemsPerByte, 8);
}
}
@@ -991,11 +1175,15 @@ bool LegalizeBufferContentTypesVisitor::visitLoadImpl(
// Typical case
+ Align PartAlign = commonAlignment(OrigLI.getAlign(), AggByteOff);
Type *ArrayAsVecType = scalarArrayTypeAsVector(PartType);
- Type *LegalType = legalNonAggregateFor(ArrayAsVecType);
+ OobProperties OobProps =
+ analyzeOobProperties(OrigLI.getPointerOperand(), PartType, AggByteOff);
+ uint64_t MaxWidth = maxIntrinsicWidth(ArrayAsVecType, PartAlign, OobProps);
+ Type *LegalType = legalNonAggregateForMemOp(ArrayAsVecType, MaxWidth);
SmallVector<VecSlice> Slices;
- getVecSlices(LegalType, Slices);
+ getVecSlices(LegalType, MaxWidth, Slices);
bool HasSlices = Slices.size() > 1;
bool IsAggPart = !AggIdxs.empty();
Value *LoadsRes;
@@ -1032,7 +1220,8 @@ bool LegalizeBufferContentTypesVisitor::visitLoadImpl(
Value *NewPtr = IRB.CreateGEP(
IRB.getInt8Ty(), OrigLI.getPointerOperand(), IRB.getInt32(ByteOffset),
OrigPtr->getName() + ".off.ptr." + Twine(ByteOffset),
- GEPNoWrapFlags::noUnsignedWrap());
+ ST->hasRelaxedBufferOOBMode() ? GEPNoWrapFlags::noUnsignedWrap()
+ : GEPNoWrapFlags::none());
Type *LoadableType = intrinsicTypeFor(SliceType);
LoadInst *NewLI = IRB.CreateAlignedLoad(
LoadableType, NewPtr, commonAlignment(OrigLI.getAlign(), ByteOffset),
@@ -1121,13 +1310,17 @@ std::pair<bool, bool> LegalizeBufferContentTypesVisitor::visitStoreImpl(
NewData = arrayToVector(NewData, ArrayAsVecType, Name);
}
- Type *LegalType = legalNonAggregateFor(ArrayAsVecType);
+ Align PartAlign = commonAlignment(OrigSI.getAlign(), AggByteOff);
+ OobProperties OobProps =
+ analyzeOobProperties(OrigSI.getPointerOperand(), PartType, AggByteOff);
+ uint64_t MaxWidth = maxIntrinsicWidth(ArrayAsVecType, PartAlign, OobProps);
+ Type *LegalType = legalNonAggregateForMemOp(ArrayAsVecType, MaxWidth);
if (LegalType != ArrayAsVecType) {
NewData = makeLegalNonAggregate(NewData, LegalType, Name);
}
SmallVector<VecSlice> Slices;
- getVecSlices(LegalType, Slices);
+ getVecSlices(LegalType, MaxWidth, Slices);
bool NeedToSplit = Slices.size() > 1 || IsAggPart;
if (!NeedToSplit) {
Type *StorableType = intrinsicTypeFor(LegalType);
@@ -1148,10 +1341,11 @@ std::pair<bool, bool> LegalizeBufferContentTypesVisitor::visitStoreImpl(
Type *SliceType =
S.Length != 1 ? FixedVectorType::get(ElemType, S.Length) : ElemType;
int64_t ByteOffset = AggByteOff + S.Index * ElemBytes;
- Value *NewPtr =
- IRB.CreateGEP(IRB.getInt8Ty(), OrigPtr, IRB.getInt32(ByteOffset),
- OrigPtr->getName() + ".part." + Twine(S.Index),
- GEPNoWrapFlags::noUnsignedWrap());
+ Value *NewPtr = IRB.CreateGEP(
+ IRB.getInt8Ty(), OrigPtr, IRB.getInt32(ByteOffset),
+ OrigPtr->getName() + ".part." + Twine(S.Index),
+ ST->hasRelaxedBufferOOBMode() ? GEPNoWrapFlags::noUnsignedWrap()
+ : GEPNoWrapFlags::none());
Value *DataSlice = extractSlice(NewData, S, Name);
Type *StorableType = intrinsicTypeFor(SliceType);
DataSlice = IRB.CreateBitCast(DataSlice, StorableType,
@@ -1179,12 +1373,37 @@ bool LegalizeBufferContentTypesVisitor::visitStoreInst(StoreInst &SI) {
return Changed;
}
-bool LegalizeBufferContentTypesVisitor::processFunction(Function &F) {
+bool LegalizeBufferContentTypesVisitor::visitAddrSpaceCastInst(
+ AddrSpaceCastInst &AI) {
+ if (AI.getSrcAddressSpace() != AMDGPUAS::BUFFER_RESOURCE ||
+ AI.getDestAddressSpace() != AMDGPUAS::BUFFER_FAT_POINTER)
+ return false;
+ Value *Src = AI.getPointerOperand();
+ auto Record = ZeroBasePointerToNumRecords.find(Src);
+ if (Record != ZeroBasePointerToNumRecords.end())
+ ZeroBasePointerToNumRecords.insert({&AI, Record->second});
+ else
+ ZeroBasePointerToNumRecords.insert({&AI, nullptr});
+ return false;
+}
+
+bool LegalizeBufferContentTypesVisitor::visitIntrinsicInst(IntrinsicInst &II) {
+ if (II.getIntrinsicID() != Intrinsic::amdgcn_make_buffer_rsrc)
+ return false;
+ ZeroBasePointerToNumRecords.insert({&II, II.getOperand(2)});
+ return false;
+}
+
+bool LegalizeBufferContentTypesVisitor::processFunction(Function &F,
+ ScalarEvolution *SE) {
+ this->SE = SE;
+ ST = &TM->getSubtarget<GCNSubtarget>(F);
bool Changed = false;
- // Note, memory transfer intrinsics won't
for (Instruction &I : make_early_inc_range(instructions(F))) {
Changed |= visit(I);
}
+ ZeroBasePointerToNumRecords.clear();
+ this->SE = nullptr;
return Changed;
}
@@ -2374,7 +2593,7 @@ class AMDGPULowerBufferFatPointers : public ModulePass {
AMDGPULowerBufferFatPointers() : ModulePass(ID) {}
- bool run(Module &M, const TargetMachine &TM);
+ bool run(Module &M, const TargetMachine &TM, GetTTIFn GetTTI, GetSEFn GetSE);
bool runOnModule(Module &M) override;
void getAnalysisUsage(AnalysisUsage &AU) const override;
@@ -2466,7 +2685,8 @@ static void makeCloneInPraceMap(Function *F, ValueToValueMapTy &CloneMap) {
}
}
-bool AMDGPULowerBufferFatPointers::run(Module &M, const TargetMachine &TM) {
+bool AMDGPULowerBufferFatPointers::run(Module &M, const TargetMachine &TM,
+ GetTTIFn GetTTI, GetSEFn GetSE) {
bool Changed = false;
const DataLayout &DL = M.getDataLayout();
// Record the functions which need to be remapped.
@@ -2533,16 +2753,18 @@ bool AMDGPULowerBufferFatPointers::run(Module &M, const TargetMachine &TM) {
}
StoreFatPtrsAsIntsAndExpandMemcpyVisitor MemOpsRewrite(&IntTM, DL,
- M.getContext(), &TM);
- LegalizeBufferContentTypesVisitor BufferContentsTypeRewrite(DL,
- M.getContext());
+ M.getContext());
+ LegalizeBufferContentTypesVisitor BufferContentsTypeRewrite(
+ DL, M.getContext(), &TM);
for (Function &F : M.functions()) {
bool InterfaceChange = hasFatPointerInterface(F, &StructTM);
bool BodyChanges = containsBufferFatPointers(F, &StructTM);
- Changed |= MemOpsRewrite.processFunction(F);
+ const TargetTransformInfo *TTI = GetTTI(F);
+ ScalarEvolution *SE = GetSE(F);
+ Changed |= MemOpsRewrite.processFunction(F, TTI, SE);
if (InterfaceChange || BodyChanges) {
NeedsRemap.push_back(std::make_pair(&F, InterfaceChange));
- Changed |= BufferContentsTypeRewrite.processFunction(F);
+ Changed |= BufferContentsTypeRewrite.processFunction(F, SE);
}
}
if (NeedsRemap.empty())
@@ -2598,7 +2820,17 @@ bool AMDGPULowerBufferFatPointers::run(Module &M, const TargetMachine &TM) {
bool AMDGPULowerBufferFatPointers::runOnModule(Module &M) {
TargetPassConfig &TPC = getAnalysis<TargetPassConfig>();
const TargetMachine &TM = TPC.getTM<TargetMachine>();
- return run(M, TM);
+ auto GetTTI = [&](Function &F) -> const TargetTransformInfo * {
+ if (F.isDeclaration())
+ return nullptr;
+ return &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
+ };
+ auto GetSE = [&](Function &F) -> ScalarEvolution * {
+ if (F.isDeclaration())
+ return nullptr;
+ return &getAnalysis<ScalarEvolutionWrapperPass>(F).getSE();
+ };
+ return run(M, TM, GetTTI, GetSE);
}
char AMDGPULowerBufferFatPointers::ID = 0;
@@ -2607,12 +2839,16 @@ char &llvm::AMDGPULowerBufferFatPointersID = AMDGPULowerBufferFatPointers::ID;
void AMDGPULowerBufferFatPointers::getAnalysisUsage(AnalysisUsage &AU) const {
AU.addRequired<TargetPassConfig>();
+ AU.addRequired<TargetTransformInfoWrapperPass>();
+ AU.addRequired<ScalarEvolutionWrapperPass>();
}
#define PASS_DESC "Lower buffer fat pointer operations to buffer resources"
INITIALIZE_PASS_BEGIN(AMDGPULowerBufferFatPointers, DEBUG_TYPE, PASS_DESC,
false, false)
INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
+INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
INITIALIZE_PASS_END(AMDGPULowerBufferFatPointers, DEBUG_TYPE, PASS_DESC, false,
false)
#undef PASS_DESC
@@ -2623,6 +2859,18 @@ ModulePass *llvm::createAMDGPULowerBufferFatPointersPass() {
PreservedAnalyses
AMDGPULowerBufferFatPointersPass::run(Module &M, ModuleAnalysisManager &MA) {
- return AMDGPULowerBufferFatPointers().run(M, TM) ? PreservedAnalyses::none()
- : PreservedAnalyses::all();
+ auto &FA = MA.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
+ auto GetTTI = [&](Function &F) -> const TargetTransformInfo * {
+ if (F.isDeclaration())
+ return nullptr;
+ return &FA.getResult<TargetIRAnalysis>(F);
+ };
+ auto GetSE = [&](Function &F) -> ScalarEvolution * {
+ if (F.isDeclaration())
+ return nullptr;
+ return &FA.getResult<ScalarEvolutionAnalysis>(F);
+ };
+ return AMDGPULowerBufferFatPointers().run(M, TM, GetTTI, GetSE)
+ ? PreservedAnalyses::none()
+ : PreservedAnalyses::all();
}
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index 7df50850f6357..05d2195de63d6 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -6883,45 +6883,90 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, s16
-; GFX11-NEXT: v_mov_b32_e32 v0, s16
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-NEXT: .LBB19_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v5, v0
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_add_f16 v4, v5, v2
-; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB19_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, s16
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v0, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v4, v5, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB19_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s16 :: v_dual_mov_b32 v3, s16
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-FAKE16-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v0
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v4, v5, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB19_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v2, v0
; GFX10-NEXT: v_mov_b32_e32 v0, s20
-; GFX10-NEXT: v_mov_b32_e32 v3, s20
+; GFX10-NEXT: v_mov_b32_e32 v1, s20
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v0
; GFX10-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v5, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_pk_add_f16 v4, v5, v2
@@ -6953,12 +6998,17 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v2, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1026
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: s_waitcnt vmcnt(1)
+; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_lshl_or_b32 v0, v4, 16, v0
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v5, v0
; GFX908-NEXT: v_pk_add_f16 v4, v5, v2
; GFX908-NEXT: v_mov_b32_e32 v0, v4
@@ -6979,12 +7029,17 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s20
-; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_mov_b32_e32 v1, s20
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v0, v0, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mov_b32_e32 v3, s20
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v5, v0
; GFX8-NEXT: v_add_f16_sdwa v0, v5, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_add_f16_e32 v1, v5, v2
@@ -7006,16 +7061,17 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_mov_b32_e32 v4, s20
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -7047,18 +7103,18 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, s20
+; GFX6-NEXT: buffer_load_ushort v1, v2, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_mov_b32_e32 v4, s6
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v1
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -7118,45 +7174,89 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fin
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s16
-; GFX11-NEXT: v_mov_b32_e32 v3, s16
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 offen offset:1024
-; GFX11-NEXT: .LBB20_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_pk_add_f16 v1, v2, v0
-; GFX11-NEXT: v_mov_b32_e32 v5, v2
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v4, v1
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
-; GFX11-NEXT: v_mov_b32_e32 v2, v4
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB20_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s16
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v2, v2, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v1, v2, v0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v1
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB20_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s16
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v2, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v2, 16, v1
+; GFX11-FAKE16-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v1, v2, v0
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v1
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v4
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB20_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s20
-; GFX10-NEXT: v_mov_b32_e32 v3, s20
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshl_or_b32 v2, v4, 16, v1
; GFX10-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_pk_add_f16 v1, v2, v0
; GFX10-NEXT: v_mov_b32_e32 v5, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -7196,12 +7296,17 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fin
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s20
-; GFX8-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mov_b32_e32 v3, s20
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_or_b32_e32 v2, v1, v2
; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_add_f16_sdwa v1, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_add_f16_e32 v4, v2, v0
; GFX8-NEXT: v_or_b32_e32 v1, v4, v1
@@ -7223,16 +7328,17 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fin
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_mov_b32_e32 v4, s20
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -7261,18 +7367,18 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fin
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, s20
+; GFX6-NEXT: buffer_load_ushort v1, v2, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_mov_b32_e32 v4, s6
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v1
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -7367,65 +7473,159 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_mov_b32 s6, s5
-; GFX11-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-NEXT: v_readfirstlane_b32 s2, v2
-; GFX11-NEXT: v_readfirstlane_b32 s3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-NEXT: buffer_load_b32 v8, v4, s[0:3], 0 offen offset:1024
-; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX11-NEXT: s_cbranch_execnz .LBB21_1
-; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB21_3: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Loop Header: Depth=1
-; GFX11-NEXT: ; Child Loop BB21_4 Depth 2
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_pk_add_f16 v7, v8, v5
-; GFX11-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_mov_b32 s6, s5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_mov_b32_e32 v6, v7
-; GFX11-NEXT: v_mov_b32_e32 v7, v8
-; GFX11-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
-; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-NEXT: v_readfirstlane_b32 s2, v2
-; GFX11-NEXT: v_readfirstlane_b32 s3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
-; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX11-NEXT: s_cbranch_execnz .LBB21_4
-; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
-; GFX11-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
-; GFX11-NEXT: v_mov_b32_e32 v8, v6
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB21_3
-; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: v_mov_b32_e32 v0, v6
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_u16 v8, v4, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_1
+; GFX11-TRUE16-NEXT: ; %bb.2:
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_u16 v6, v4, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_3
+; GFX11-TRUE16-NEXT: ; %bb.4:
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v6.l
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB21_5: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: ; Child Loop BB21_6 Depth 2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v7, v8, v5
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v8
+; GFX11-TRUE16-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
+; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_6
+; GFX11-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_5
+; GFX11-TRUE16-NEXT: ; %bb.8: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v6
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_u16 v6, v4, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_1
+; GFX11-FAKE16-NEXT: ; %bb.2:
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_3
+; GFX11-FAKE16-NEXT: ; %bb.4:
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_perm_b32 v8, v7, v6, 0x5040100
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB21_5: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: ; Child Loop BB21_6 Depth 2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v7, v8, v5
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v7
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v8
+; GFX11-FAKE16-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
+; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_6
+; GFX11-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_5
+; GFX11-FAKE16-NEXT: ; %bb.8: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v6
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -7441,23 +7641,40 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX10-NEXT: buffer_load_dword v8, v4, s[4:7], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v6, v4, s[4:7], 0 offen offset:1024
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB21_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
-; GFX10-NEXT: .LBB21_3: ; %atomicrmw.start
-; GFX10-NEXT: ; =>This Loop Header: Depth=1
-; GFX10-NEXT: ; Child Loop BB21_4 Depth 2
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s10, s9
+; GFX10-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_ushort v7, v4, s[4:7], 0 offen offset:1026
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX10-NEXT: s_cbranch_execnz .LBB21_3
+; GFX10-NEXT: ; %bb.4:
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_perm_b32 v8, v7, v6, 0x5040100
+; GFX10-NEXT: .LBB21_5: ; %atomicrmw.start
+; GFX10-NEXT: ; =>This Loop Header: Depth=1
+; GFX10-NEXT: ; Child Loop BB21_6 Depth 2
; GFX10-NEXT: v_pk_add_f16 v7, v8, v5
; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_mov_b32_e32 v6, v7
; GFX10-NEXT: v_mov_b32_e32 v7, v8
-; GFX10-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX10-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
@@ -7469,8 +7686,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 glc
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
-; GFX10-NEXT: s_cbranch_execnz .LBB21_4
-; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX10-NEXT: s_cbranch_execnz .LBB21_6
+; GFX10-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -7481,8 +7698,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
-; GFX10-NEXT: s_cbranch_execnz .LBB21_3
-; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX10-NEXT: s_cbranch_execnz .LBB21_5
+; GFX10-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_mov_b32_e32 v0, v6
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -7527,21 +7744,39 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: buffer_load_dword v8, v4, s[8:11], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB21_1
; GFX908-NEXT: ; %bb.2:
; GFX908-NEXT: s_mov_b64 exec, s[6:7]
+; GFX908-NEXT: s_mov_b64 s[6:7], exec
+; GFX908-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: v_readfirstlane_b32 s8, v0
+; GFX908-NEXT: v_readfirstlane_b32 s9, v1
+; GFX908-NEXT: v_readfirstlane_b32 s10, v2
+; GFX908-NEXT: v_readfirstlane_b32 s11, v3
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX908-NEXT: s_nop 0
+; GFX908-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1026
+; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_cbranch_execnz .LBB21_3
+; GFX908-NEXT: ; %bb.4:
+; GFX908-NEXT: s_mov_b64 exec, s[6:7]
+; GFX908-NEXT: s_mov_b32 s4, 0x5040100
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_perm_b32 v8, v7, v6, s4
; GFX908-NEXT: s_mov_b64 s[6:7], 0
-; GFX908-NEXT: .LBB21_3: ; %atomicrmw.start
+; GFX908-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Loop Header: Depth=1
-; GFX908-NEXT: ; Child Loop BB21_4 Depth 2
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: ; Child Loop BB21_6 Depth 2
; GFX908-NEXT: v_pk_add_f16 v7, v8, v5
; GFX908-NEXT: v_mov_b32_e32 v6, v7
; GFX908-NEXT: s_mov_b64 s[12:13], exec
; GFX908-NEXT: v_mov_b32_e32 v7, v8
-; GFX908-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX908-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
; GFX908-NEXT: v_readfirstlane_b32 s9, v1
@@ -7554,8 +7789,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB21_4
-; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX908-NEXT: s_cbranch_execnz .LBB21_6
+; GFX908-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
@@ -7563,8 +7798,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX908-NEXT: v_mov_b32_e32 v8, v6
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB21_3
-; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX908-NEXT: s_cbranch_execnz .LBB21_5
+; GFX908-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v6
; GFX908-NEXT: s_setpc_b64 s[30:31]
@@ -7583,23 +7818,41 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_nop 0
-; GFX8-NEXT: buffer_load_dword v8, v4, s[8:11], 0 offen offset:1024
+; GFX8-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB21_1
; GFX8-NEXT: ; %bb.2:
; GFX8-NEXT: s_mov_b64 exec, s[6:7]
+; GFX8-NEXT: s_mov_b64 s[6:7], exec
+; GFX8-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_readfirstlane_b32 s8, v0
+; GFX8-NEXT: v_readfirstlane_b32 s9, v1
+; GFX8-NEXT: v_readfirstlane_b32 s10, v2
+; GFX8-NEXT: v_readfirstlane_b32 s11, v3
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX8-NEXT: s_nop 0
+; GFX8-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1026
+; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB21_3
+; GFX8-NEXT: ; %bb.4:
+; GFX8-NEXT: s_mov_b64 exec, s[6:7]
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX8-NEXT: v_or_b32_e32 v8, v6, v7
; GFX8-NEXT: s_mov_b64 s[6:7], 0
-; GFX8-NEXT: .LBB21_3: ; %atomicrmw.start
+; GFX8-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Loop Header: Depth=1
-; GFX8-NEXT: ; Child Loop BB21_4 Depth 2
-; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: ; Child Loop BB21_6 Depth 2
; GFX8-NEXT: v_add_f16_sdwa v6, v8, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_add_f16_e32 v7, v8, v5
; GFX8-NEXT: v_or_b32_e32 v7, v7, v6
; GFX8-NEXT: v_mov_b32_e32 v6, v7
; GFX8-NEXT: s_mov_b64 s[12:13], exec
; GFX8-NEXT: v_mov_b32_e32 v7, v8
-; GFX8-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX8-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
; GFX8-NEXT: v_readfirstlane_b32 s9, v1
@@ -7612,8 +7865,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB21_4
-; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX8-NEXT: s_cbranch_execnz .LBB21_6
+; GFX8-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
@@ -7621,8 +7874,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX8-NEXT: v_mov_b32_e32 v8, v6
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB21_3
-; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX8-NEXT: s_cbranch_execnz .LBB21_5
+; GFX8-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v6
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -7641,34 +7894,48 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: buffer_load_dword v6, v4, s[8:11], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB21_1
; GFX7-NEXT: ; %bb.2:
; GFX7-NEXT: s_mov_b64 exec, s[6:7]
+; GFX7-NEXT: s_mov_b64 s[6:7], exec
+; GFX7-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: v_readfirstlane_b32 s8, v0
+; GFX7-NEXT: v_readfirstlane_b32 s9, v1
+; GFX7-NEXT: v_readfirstlane_b32 s10, v2
+; GFX7-NEXT: v_readfirstlane_b32 s11, v3
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX7-NEXT: buffer_load_ushort v10, v4, s[8:11], 0 offen offset:1026
+; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_cbranch_execnz .LBB21_3
+; GFX7-NEXT: ; %bb.4:
+; GFX7-NEXT: s_mov_b64 exec, s[6:7]
; GFX7-NEXT: v_cvt_f32_f16_e32 v9, v7
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v6
; GFX7-NEXT: s_mov_b64 s[6:7], 0
-; GFX7-NEXT: .LBB21_3: ; %atomicrmw.start
+; GFX7-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Loop Header: Depth=1
-; GFX7-NEXT: ; Child Loop BB21_4 Depth 2
-; GFX7-NEXT: v_cvt_f32_f16_e32 v8, v7
-; GFX7-NEXT: v_cvt_f32_f16_e32 v10, v6
+; GFX7-NEXT: ; Child Loop BB21_6 Depth 2
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v10
+; GFX7-NEXT: v_cvt_f32_f16_e32 v8, v6
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX7-NEXT: v_add_f32_e32 v8, v8, v9
-; GFX7-NEXT: v_add_f32_e32 v10, v10, v5
-; GFX7-NEXT: v_cvt_f16_f32_e32 v11, v8
-; GFX7-NEXT: v_cvt_f16_f32_e32 v10, v10
-; GFX7-NEXT: v_or_b32_e32 v8, v6, v7
; GFX7-NEXT: s_mov_b64 s[12:13], exec
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v11
-; GFX7-NEXT: v_or_b32_e32 v7, v10, v6
+; GFX7-NEXT: v_add_f32_e32 v7, v7, v9
+; GFX7-NEXT: v_add_f32_e32 v8, v8, v5
+; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v7
+; GFX7-NEXT: v_cvt_f16_f32_e32 v11, v8
+; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v10
+; GFX7-NEXT: v_or_b32_e32 v8, v6, v8
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; GFX7-NEXT: v_or_b32_e32 v7, v11, v6
; GFX7-NEXT: v_mov_b32_e32 v6, v7
; GFX7-NEXT: v_mov_b32_e32 v7, v8
-; GFX7-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX7-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX7-NEXT: ; => This Inner Loop Header: Depth=2
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
; GFX7-NEXT: v_readfirstlane_b32 s9, v1
@@ -7681,20 +7948,20 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB21_4
-; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX7-NEXT: s_cbranch_execnz .LBB21_6
+; GFX7-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
-; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v10, 16, v6
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX7-NEXT: s_cbranch_execnz .LBB21_3
-; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX7-NEXT: s_cbranch_execnz .LBB21_5
+; GFX7-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v7
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v10
; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
@@ -7713,36 +7980,50 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: buffer_load_dword v6, v4, s[8:11], 0 offen offset:1024
-; GFX6-NEXT: ; implicit-def: $vgpr4
+; GFX6-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB21_1
; GFX6-NEXT: ; %bb.2:
; GFX6-NEXT: s_mov_b64 exec, s[6:7]
+; GFX6-NEXT: s_mov_b64 s[6:7], exec
+; GFX6-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: v_readfirstlane_b32 s8, v0
+; GFX6-NEXT: v_readfirstlane_b32 s9, v1
+; GFX6-NEXT: v_readfirstlane_b32 s10, v2
+; GFX6-NEXT: v_readfirstlane_b32 s11, v3
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX6-NEXT: buffer_load_ushort v11, v4, s[8:11], 0 offen offset:1026
+; GFX6-NEXT: ; implicit-def: $vgpr4
+; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX6-NEXT: s_cbranch_execnz .LBB21_3
+; GFX6-NEXT: ; %bb.4:
+; GFX6-NEXT: s_mov_b64 exec, s[6:7]
; GFX6-NEXT: v_cvt_f32_f16_e32 v9, v7
; GFX6-NEXT: v_cvt_f32_f16_e32 v10, v5
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
; GFX6-NEXT: s_mov_b64 s[6:7], 0
-; GFX6-NEXT: .LBB21_3: ; %atomicrmw.start
+; GFX6-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Loop Header: Depth=1
-; GFX6-NEXT: ; Child Loop BB21_4 Depth 2
-; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v4
+; GFX6-NEXT: ; Child Loop BB21_6 Depth 2
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v11
+; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v6
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v6
; GFX6-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_add_f32_e32 v5, v5, v9
-; GFX6-NEXT: v_add_f32_e32 v7, v7, v10
-; GFX6-NEXT: v_cvt_f16_f32_e32 v11, v5
-; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v7
-; GFX6-NEXT: v_or_b32_e32 v5, v6, v4
; GFX6-NEXT: s_mov_b64 s[12:13], exec
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v11
+; GFX6-NEXT: v_add_f32_e32 v4, v4, v9
+; GFX6-NEXT: v_add_f32_e32 v5, v5, v10
+; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v5
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v11
+; GFX6-NEXT: v_or_b32_e32 v5, v6, v5
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX6-NEXT: v_or_b32_e32 v4, v7, v4
; GFX6-NEXT: v_mov_b32_e32 v7, v5
; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX6-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX6-NEXT: ; => This Inner Loop Header: Depth=2
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
; GFX6-NEXT: v_readfirstlane_b32 s9, v1
@@ -7755,20 +8036,20 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v8, s[8:11], 0 offen glc
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_cbranch_execnz .LBB21_4
-; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX6-NEXT: s_cbranch_execnz .LBB21_6
+; GFX6-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v11, 16, v6
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX6-NEXT: s_cbranch_execnz .LBB21_3
-; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX6-NEXT: s_cbranch_execnz .LBB21_5
+; GFX6-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v6
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v11
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
@@ -7803,45 +8084,90 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, s16
-; GFX11-NEXT: v_mov_b32_e32 v0, s16
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-NEXT: .LBB22_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v5, v0
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_add_f16 v4, v5, v2
-; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB22_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, s16
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v0, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v4, v5, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB22_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s16 :: v_dual_mov_b32 v3, s16
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-FAKE16-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v0
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v4, v5, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB22_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v2, v0
; GFX10-NEXT: v_mov_b32_e32 v0, s20
-; GFX10-NEXT: v_mov_b32_e32 v3, s20
+; GFX10-NEXT: v_mov_b32_e32 v1, s20
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v0
; GFX10-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v5, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_pk_add_f16 v4, v5, v2
@@ -7864,12 +8190,17 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, v0
; GFX90A-NEXT: v_mov_b32_e32 v0, s20
-; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1026
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_waitcnt vmcnt(1)
+; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshl_or_b32 v0, v4, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: v_pk_add_f16 v4, v5, v2
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
@@ -7889,12 +8220,17 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v2, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1026
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: s_waitcnt vmcnt(1)
+; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_lshl_or_b32 v0, v4, 16, v0
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v5, v0
; GFX908-NEXT: v_pk_add_f16 v4, v5, v2
; GFX908-NEXT: v_mov_b32_e32 v0, v4
@@ -7915,12 +8251,17 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s20
-; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_mov_b32_e32 v1, s20
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v0, v0, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mov_b32_e32 v3, s20
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v5, v0
; GFX8-NEXT: v_add_f16_sdwa v0, v5, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_add_f16_e32 v1, v5, v2
@@ -7942,16 +8283,17 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_mov_b32_e32 v4, s20
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -7983,18 +8325,18 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, s20
+; GFX6-NEXT: buffer_load_ushort v1, v2, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_mov_b32_e32 v4, s6
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v1
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -8054,45 +8396,89 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s16
-; GFX11-NEXT: v_mov_b32_e32 v3, s16
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 offen offset:1024
-; GFX11-NEXT: .LBB23_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_pk_add_f16 v1, v2, v0
-; GFX11-NEXT: v_mov_b32_e32 v5, v2
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v4, v1
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
-; GFX11-NEXT: v_mov_b32_e32 v2, v4
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB23_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s16
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v2, v2, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v1, v2, v0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v1
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB23_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s16
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v2, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v2, 16, v1
+; GFX11-FAKE16-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v1, v2, v0
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v1
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v4
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB23_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s20
-; GFX10-NEXT: v_mov_b32_e32 v3, s20
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshl_or_b32 v2, v4, 16, v1
; GFX10-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_pk_add_f16 v1, v2, v0
; GFX10-NEXT: v_mov_b32_e32 v5, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -8114,11 +8500,17 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: buffer_load_dword v3, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
+; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_waitcnt vmcnt(1)
+; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshl_or_b32 v3, v4, 16, v1
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_add_f16 v2, v3, v0
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v1, s[16:19], 0 offen offset:1024 glc
@@ -8137,12 +8529,17 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: s_waitcnt vmcnt(1)
+; GFX908-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_lshl_or_b32 v2, v4, 16, v1
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_pk_add_f16 v1, v2, v0
; GFX908-NEXT: v_mov_b32_e32 v5, v2
; GFX908-NEXT: v_mov_b32_e32 v4, v1
@@ -8162,12 +8559,17 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s20
-; GFX8-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mov_b32_e32 v3, s20
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_or_b32_e32 v2, v1, v2
; GFX8-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_add_f16_sdwa v1, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_add_f16_e32 v4, v2, v0
; GFX8-NEXT: v_or_b32_e32 v1, v4, v1
@@ -8189,16 +8591,17 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_mov_b32_e32 v4, s20
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -8227,18 +8630,18 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, s20
+; GFX6-NEXT: buffer_load_ushort v1, v2, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_mov_b32_e32 v4, s6
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v1
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -8295,45 +8698,90 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_remote_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, s16
-; GFX11-NEXT: v_mov_b32_e32 v0, s16
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-NEXT: .LBB24_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v5, v0
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_add_f16 v4, v5, v2
-; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB24_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_remote_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, s16
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v0, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v4, v5, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB24_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_remote_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s16 :: v_dual_mov_b32 v3, s16
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-FAKE16-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v0
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v4, v5, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB24_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v2, v0
; GFX10-NEXT: v_mov_b32_e32 v0, s20
-; GFX10-NEXT: v_mov_b32_e32 v3, s20
+; GFX10-NEXT: v_mov_b32_e32 v1, s20
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v0
; GFX10-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v5, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_pk_add_f16 v4, v5, v2
@@ -8356,12 +8804,17 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, v0
; GFX90A-NEXT: v_mov_b32_e32 v0, s20
-; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1026
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_waitcnt vmcnt(1)
+; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshl_or_b32 v0, v4, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: v_pk_add_f16 v4, v5, v2
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
@@ -8381,12 +8834,17 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v2, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1026
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: s_waitcnt vmcnt(1)
+; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_lshl_or_b32 v0, v4, 16, v0
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v5, v0
; GFX908-NEXT: v_pk_add_f16 v4, v5, v2
; GFX908-NEXT: v_mov_b32_e32 v0, v4
@@ -8407,12 +8865,17 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s20
-; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_mov_b32_e32 v1, s20
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v0, v0, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mov_b32_e32 v3, s20
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v5, v0
; GFX8-NEXT: v_add_f16_sdwa v0, v5, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_add_f16_e32 v1, v5, v2
@@ -8434,16 +8897,17 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_mov_b32_e32 v4, s20
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -8475,18 +8939,18 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_remote_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, s20
+; GFX6-NEXT: buffer_load_ushort v1, v2, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_mov_b32_e32 v4, s6
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v1
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -8546,45 +9010,89 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_remote_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s16
-; GFX11-NEXT: v_mov_b32_e32 v3, s16
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 offen offset:1024
-; GFX11-NEXT: .LBB25_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_pk_add_f16 v1, v2, v0
-; GFX11-NEXT: v_mov_b32_e32 v5, v2
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v4, v1
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
-; GFX11-NEXT: v_mov_b32_e32 v2, v4
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB25_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_remote_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s16
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v2, v2, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v1, v2, v0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v1
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB25_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_remote_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s16
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v2, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v2, 16, v1
+; GFX11-FAKE16-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v1, v2, v0
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v1
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v4
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB25_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s20
-; GFX10-NEXT: v_mov_b32_e32 v3, s20
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshl_or_b32 v2, v4, 16, v1
; GFX10-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_pk_add_f16 v1, v2, v0
; GFX10-NEXT: v_mov_b32_e32 v5, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -8606,11 +9114,17 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: buffer_load_dword v3, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
+; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_waitcnt vmcnt(1)
+; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshl_or_b32 v3, v4, 16, v1
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_add_f16 v2, v3, v0
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v1, s[16:19], 0 offen offset:1024 glc
@@ -8629,12 +9143,17 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: s_waitcnt vmcnt(1)
+; GFX908-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_lshl_or_b32 v2, v4, 16, v1
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_pk_add_f16 v1, v2, v0
; GFX908-NEXT: v_mov_b32_e32 v5, v2
; GFX908-NEXT: v_mov_b32_e32 v4, v1
@@ -8654,12 +9173,17 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s20
-; GFX8-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mov_b32_e32 v3, s20
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_or_b32_e32 v2, v1, v2
; GFX8-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_add_f16_sdwa v1, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_add_f16_e32 v4, v2, v0
; GFX8-NEXT: v_or_b32_e32 v1, v4, v1
@@ -8681,16 +9205,17 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_mov_b32_e32 v4, s20
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -8719,18 +9244,18 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_remote_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, s20
+; GFX6-NEXT: buffer_load_ushort v1, v2, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_mov_b32_e32 v4, s6
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v1
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -8783,18 +9308,22 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-NEXT: v_mov_b32_e32 v0, s16
-; GFX942-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
+; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
-; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
-; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
+; GFX942-NEXT: s_waitcnt vmcnt(1)
+; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_lshl_or_b32 v0, v5, 16, v0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v7, v0
; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v7
; GFX942-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
@@ -8830,37 +9359,41 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_lshlrev_b32 v3, 16, v1
-; GFX11-TRUE16-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: buffer_load_u16 v4, v0, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v0
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -8879,39 +9412,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_and_b32 v3, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v0
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v1, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v1, v7, v9 :: v_dual_lshlrev_b32 v0, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v0, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v5, v8, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_perm_b32 v5, v1, v0, 0x7060302
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
@@ -8930,16 +9469,21 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, s20
-; GFX10-NEXT: v_mov_b32_e32 v4, s20
+; GFX10-NEXT: v_mov_b32_e32 v1, s20
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v6
@@ -8974,18 +9518,22 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: v_mov_b32_e32 v0, s20
-; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_waitcnt vmcnt(1)
+; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshl_or_b32 v0, v5, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v0
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v7
; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
@@ -9017,18 +9565,22 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, v0
-; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
-; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
+; GFX908-NEXT: s_waitcnt vmcnt(1)
+; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_lshl_or_b32 v0, v5, 16, v0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v0
; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v6
; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
@@ -9061,16 +9613,20 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s20
-; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_mov_b32_e32 v1, s20
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX8-NEXT: s_mov_b64 s[6:7], 0
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
@@ -9106,8 +9662,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX7-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -9115,9 +9673,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -9154,7 +9713,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
@@ -9163,9 +9723,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX6-NEXT: s_waitcnt vmcnt(1)
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX6-NEXT: v_mov_b32_e32 v2, s6
; GFX6-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -9223,16 +9784,21 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v1, s16
-; GFX942-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: s_mov_b64 s[6:7], 0
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
+; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX942-NEXT: s_mov_b64 s[6:7], 0
+; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
+; GFX942-NEXT: s_waitcnt vmcnt(1)
+; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_lshl_or_b32 v1, v5, 16, v0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX942-NEXT: v_add_f32_e32 v0, v0, v2
@@ -9267,21 +9833,25 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_and_b32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: buffer_load_u16 v4, v1, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, v5, v2 :: v_dual_add_f32 v0, v0, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -9313,22 +9883,30 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_lshlrev_b32 v2, 16, v0
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_and_b32 v3, 0xffff0000, v0
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
-; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v5, v3 :: v_dual_add_f32 v0, v0, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v0, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
@@ -9361,14 +9939,20 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s20
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
+; GFX10-NEXT: s_mov_b32 s5, 0
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshl_or_b32 v1, v4, 16, v1
; GFX10-NEXT: v_mov_b32_e32 v4, s20
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
; GFX10-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -9404,16 +9988,21 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_waitcnt vmcnt(1)
+; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshl_or_b32 v1, v5, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX90A-NEXT: v_add_f32_e32 v0, v0, v2
@@ -9446,16 +10035,21 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
+; GFX908-NEXT: s_waitcnt vmcnt(1)
+; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_lshl_or_b32 v1, v5, 16, v0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX908-NEXT: v_add_f32_e32 v0, v0, v2
@@ -9489,14 +10083,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s20
-; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX8-NEXT: v_add_f32_e32 v0, v0, v2
@@ -9532,8 +10131,10 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX7-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -9541,9 +10142,10 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -9576,7 +10178,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
@@ -9585,9 +10188,10 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX6-NEXT: s_waitcnt vmcnt(1)
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX6-NEXT: v_mov_b32_e32 v2, s6
; GFX6-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -9671,20 +10275,38 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX942-NEXT: buffer_load_dword v9, v4, s[4:7], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v6, v4, s[4:7], 0 offen offset:1024
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB28_1
; GFX942-NEXT: ; %bb.2:
; GFX942-NEXT: s_mov_b64 exec, s[2:3]
+; GFX942-NEXT: s_mov_b64 s[2:3], exec
+; GFX942-NEXT: .LBB28_3: ; =>This Inner Loop Header: Depth=1
+; GFX942-NEXT: v_readfirstlane_b32 s4, v0
+; GFX942-NEXT: v_readfirstlane_b32 s5, v1
+; GFX942-NEXT: v_readfirstlane_b32 s6, v2
+; GFX942-NEXT: v_readfirstlane_b32 s7, v3
+; GFX942-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
+; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT: buffer_load_ushort v7, v4, s[4:7], 0 offen offset:1026
+; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_cbranch_execnz .LBB28_3
+; GFX942-NEXT: ; %bb.4:
+; GFX942-NEXT: s_mov_b64 exec, s[2:3]
+; GFX942-NEXT: s_mov_b32 s0, 0x5040100
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_perm_b32 v9, v7, v6, s0
; GFX942-NEXT: s_mov_b64 s[2:3], 0
; GFX942-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX942-NEXT: s_movk_i32 s10, 0x7fff
; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX942-NEXT: s_mov_b32 s11, 0x7060302
-; GFX942-NEXT: .LBB28_3: ; %atomicrmw.start
+; GFX942-NEXT: .LBB28_5: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Loop Header: Depth=1
-; GFX942-NEXT: ; Child Loop BB28_4 Depth 2
-; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: ; Child Loop BB28_6 Depth 2
; GFX942-NEXT: v_lshlrev_b32_e32 v6, 16, v9
; GFX942-NEXT: v_add_f32_e32 v6, v6, v10
; GFX942-NEXT: v_bfe_u32 v7, v6, 16, 1
@@ -9705,7 +10327,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX942-NEXT: v_cndmask_b32_e32 v7, v8, v11, vcc
; GFX942-NEXT: v_perm_b32 v8, v7, v6, s11
; GFX942-NEXT: v_mov_b64_e32 v[6:7], v[8:9]
-; GFX942-NEXT: .LBB28_4: ; Parent Loop BB28_3 Depth=1
+; GFX942-NEXT: .LBB28_6: ; Parent Loop BB28_5 Depth=1
; GFX942-NEXT: ; => This Inner Loop Header: Depth=2
; GFX942-NEXT: v_readfirstlane_b32 s4, v0
; GFX942-NEXT: v_readfirstlane_b32 s5, v1
@@ -9719,8 +10341,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 sc0
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB28_4
-; GFX942-NEXT: ; %bb.5: ; in Loop: Header=BB28_3 Depth=1
+; GFX942-NEXT: s_cbranch_execnz .LBB28_6
+; GFX942-NEXT: ; %bb.7: ; in Loop: Header=BB28_5 Depth=1
; GFX942-NEXT: s_mov_b64 exec, s[8:9]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v6, v9
@@ -9728,8 +10350,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX942-NEXT: v_mov_b32_e32 v9, v6
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_cbranch_execnz .LBB28_3
-; GFX942-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX942-NEXT: s_cbranch_execnz .LBB28_5
+; GFX942-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-NEXT: v_mov_b32_e32 v0, v6
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -9748,28 +10370,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX11-TRUE16-NEXT: ; %bb.2:
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: .LBB28_3: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_u16 v6, v4, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_3
+; GFX11-TRUE16-NEXT: ; %bb.4:
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB28_3: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: .LBB28_5: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: ; Child Loop BB28_4 Depth 2
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; Child Loop BB28_6 Depth 2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, v6, v8 :: v_dual_add_f32 v5, v5, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v6, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
@@ -9784,7 +10423,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.h
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v7
-; GFX11-TRUE16-NEXT: .LBB28_4: ; Parent Loop BB28_3 Depth=1
+; GFX11-TRUE16-NEXT: .LBB28_6: ; Parent Loop BB28_5 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -9796,8 +10435,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_4
-; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB28_3 Depth=1
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_6
+; GFX11-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB28_5 Depth=1
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
@@ -9807,8 +10446,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_3
-; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_5
+; GFX11-TRUE16-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v5
@@ -9828,28 +10467,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v6, v4, s[0:3], 0 offen offset:1024
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX11-FAKE16-NEXT: ; %bb.2:
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: .LBB28_3: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB28_3
+; GFX11-FAKE16-NEXT: ; %bb.4:
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB28_3: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: .LBB28_5: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: ; Child Loop BB28_4 Depth 2
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; Child Loop BB28_6 Depth 2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_add_f32 v6, v6, v9 :: v_dual_add_f32 v5, v5, v8
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_bfe_u32 v11, v6, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_bfe_u32 v10, v5, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
@@ -9864,7 +10520,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: v_perm_b32 v6, v6, v5, 0x7060302
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v7
-; GFX11-FAKE16-NEXT: .LBB28_4: ; Parent Loop BB28_3 Depth=1
+; GFX11-FAKE16-NEXT: .LBB28_6: ; Parent Loop BB28_5 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -9876,8 +10532,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB28_4
-; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB28_3 Depth=1
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB28_6
+; GFX11-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB28_5 Depth=1
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
@@ -9887,8 +10543,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB28_3
-; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB28_5
+; GFX11-FAKE16-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v5
@@ -9908,18 +10564,35 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX10-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v6, v4, s[4:7], 0 offen offset:1024
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB28_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s10, s9
+; GFX10-NEXT: .LBB28_3: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_ushort v7, v4, s[4:7], 0 offen offset:1026
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX10-NEXT: s_cbranch_execnz .LBB28_3
+; GFX10-NEXT: ; %bb.4:
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; GFX10-NEXT: .LBB28_3: ; %atomicrmw.start
+; GFX10-NEXT: .LBB28_5: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
-; GFX10-NEXT: ; Child Loop BB28_4 Depth 2
-; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: ; Child Loop BB28_6 Depth 2
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX10-NEXT: s_mov_b32 s9, exec_lo
@@ -9940,7 +10613,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX10-NEXT: v_perm_b32 v6, v6, v5, 0x7060302
; GFX10-NEXT: v_mov_b32_e32 v5, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v7
-; GFX10-NEXT: .LBB28_4: ; Parent Loop BB28_3 Depth=1
+; GFX10-NEXT: .LBB28_6: ; Parent Loop BB28_5 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
@@ -9952,8 +10625,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[4:7], 0 offen offset:1024 glc
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
-; GFX10-NEXT: s_cbranch_execnz .LBB28_4
-; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB28_3 Depth=1
+; GFX10-NEXT: s_cbranch_execnz .LBB28_6
+; GFX10-NEXT: ; %bb.7: ; in Loop: Header=BB28_5 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -9964,8 +10637,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
-; GFX10-NEXT: s_cbranch_execnz .LBB28_3
-; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX10-NEXT: s_cbranch_execnz .LBB28_5
+; GFX10-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -9984,20 +10657,38 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_nop 0
-; GFX90A-NEXT: buffer_load_dword v9, v4, s[8:11], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB28_1
; GFX90A-NEXT: ; %bb.2:
; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
+; GFX90A-NEXT: s_mov_b64 s[6:7], exec
+; GFX90A-NEXT: .LBB28_3: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
+; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
+; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
+; GFX90A-NEXT: v_readfirstlane_b32 s11, v3
+; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX90A-NEXT: s_nop 0
+; GFX90A-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1026
+; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_cbranch_execnz .LBB28_3
+; GFX90A-NEXT: ; %bb.4:
+; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
+; GFX90A-NEXT: s_mov_b32 s4, 0x5040100
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_perm_b32 v9, v7, v6, s4
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX90A-NEXT: s_movk_i32 s14, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX90A-NEXT: s_mov_b32 s15, 0x7060302
-; GFX90A-NEXT: .LBB28_3: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB28_5: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
-; GFX90A-NEXT: ; Child Loop BB28_4 Depth 2
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; Child Loop BB28_6 Depth 2
; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v9
; GFX90A-NEXT: v_add_f32_e32 v6, v6, v10
; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
@@ -10015,7 +10706,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX90A-NEXT: v_perm_b32 v8, v7, v6, s15
; GFX90A-NEXT: s_mov_b64 s[12:13], exec
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[8:9], v[8:9] op_sel:[0,1]
-; GFX90A-NEXT: .LBB28_4: ; Parent Loop BB28_3 Depth=1
+; GFX90A-NEXT: .LBB28_6: ; Parent Loop BB28_5 Depth=1
; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
@@ -10028,8 +10719,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB28_4
-; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB28_3 Depth=1
+; GFX90A-NEXT: s_cbranch_execnz .LBB28_6
+; GFX90A-NEXT: ; %bb.7: ; in Loop: Header=BB28_5 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v6, v9
@@ -10037,8 +10728,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX90A-NEXT: v_mov_b32_e32 v9, v6
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB28_3
-; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX90A-NEXT: s_cbranch_execnz .LBB28_5
+; GFX90A-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v6
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -10057,20 +10748,38 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB28_1
; GFX908-NEXT: ; %bb.2:
; GFX908-NEXT: s_mov_b64 exec, s[6:7]
+; GFX908-NEXT: s_mov_b64 s[6:7], exec
+; GFX908-NEXT: .LBB28_3: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: v_readfirstlane_b32 s8, v0
+; GFX908-NEXT: v_readfirstlane_b32 s9, v1
+; GFX908-NEXT: v_readfirstlane_b32 s10, v2
+; GFX908-NEXT: v_readfirstlane_b32 s11, v3
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX908-NEXT: s_nop 0
+; GFX908-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1026
+; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_cbranch_execnz .LBB28_3
+; GFX908-NEXT: ; %bb.4:
+; GFX908-NEXT: s_mov_b64 exec, s[6:7]
+; GFX908-NEXT: s_mov_b32 s4, 0x5040100
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_perm_b32 v7, v7, v6, s4
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX908-NEXT: s_movk_i32 s14, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX908-NEXT: s_mov_b32 s15, 0x7060302
-; GFX908-NEXT: .LBB28_3: ; %atomicrmw.start
+; GFX908-NEXT: .LBB28_5: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Loop Header: Depth=1
-; GFX908-NEXT: ; Child Loop BB28_4 Depth 2
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: ; Child Loop BB28_6 Depth 2
; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX908-NEXT: v_add_f32_e32 v5, v5, v8
; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
@@ -10089,7 +10798,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX908-NEXT: v_mov_b32_e32 v5, v6
; GFX908-NEXT: s_mov_b64 s[12:13], exec
; GFX908-NEXT: v_mov_b32_e32 v6, v7
-; GFX908-NEXT: .LBB28_4: ; Parent Loop BB28_3 Depth=1
+; GFX908-NEXT: .LBB28_6: ; Parent Loop BB28_5 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
; GFX908-NEXT: v_readfirstlane_b32 s9, v1
@@ -10102,8 +10811,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB28_4
-; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB28_3 Depth=1
+; GFX908-NEXT: s_cbranch_execnz .LBB28_6
+; GFX908-NEXT: ; %bb.7: ; in Loop: Header=BB28_5 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
@@ -10111,8 +10820,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB28_3
-; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX908-NEXT: s_cbranch_execnz .LBB28_5
+; GFX908-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
@@ -10131,18 +10840,36 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_nop 0
-; GFX8-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
+; GFX8-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB28_1
; GFX8-NEXT: ; %bb.2:
; GFX8-NEXT: s_mov_b64 exec, s[6:7]
+; GFX8-NEXT: s_mov_b64 s[6:7], exec
+; GFX8-NEXT: .LBB28_3: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_readfirstlane_b32 s8, v0
+; GFX8-NEXT: v_readfirstlane_b32 s9, v1
+; GFX8-NEXT: v_readfirstlane_b32 s10, v2
+; GFX8-NEXT: v_readfirstlane_b32 s11, v3
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX8-NEXT: s_nop 0
+; GFX8-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1026
+; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB28_3
+; GFX8-NEXT: ; %bb.4:
+; GFX8-NEXT: s_mov_b64 exec, s[6:7]
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX8-NEXT: v_or_b32_e32 v7, v6, v7
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; GFX8-NEXT: .LBB28_3: ; %atomicrmw.start
+; GFX8-NEXT: .LBB28_5: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Loop Header: Depth=1
-; GFX8-NEXT: ; Child Loop BB28_4 Depth 2
-; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: ; Child Loop BB28_6 Depth 2
; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX8-NEXT: v_add_f32_e32 v5, v5, v8
; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
@@ -10164,7 +10891,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX8-NEXT: v_mov_b32_e32 v5, v6
; GFX8-NEXT: s_mov_b64 s[12:13], exec
; GFX8-NEXT: v_mov_b32_e32 v6, v7
-; GFX8-NEXT: .LBB28_4: ; Parent Loop BB28_3 Depth=1
+; GFX8-NEXT: .LBB28_6: ; Parent Loop BB28_5 Depth=1
; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
; GFX8-NEXT: v_readfirstlane_b32 s9, v1
@@ -10177,8 +10904,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB28_4
-; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB28_3 Depth=1
+; GFX8-NEXT: s_cbranch_execnz .LBB28_6
+; GFX8-NEXT: ; %bb.7: ; in Loop: Header=BB28_5 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
@@ -10186,8 +10913,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX8-NEXT: v_mov_b32_e32 v7, v5
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB28_3
-; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX8-NEXT: s_cbranch_execnz .LBB28_5
+; GFX8-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -10196,7 +10923,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: s_mov_b64 s[6:7], exec
; GFX7-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
@@ -10207,36 +10934,51 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: buffer_load_dword v8, v4, s[8:11], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1024
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB28_1
; GFX7-NEXT: ; %bb.2:
; GFX7-NEXT: s_mov_b64 exec, s[6:7]
-; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: s_mov_b64 s[6:7], exec
+; GFX7-NEXT: .LBB28_3: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: v_readfirstlane_b32 s8, v0
+; GFX7-NEXT: v_readfirstlane_b32 s9, v1
+; GFX7-NEXT: v_readfirstlane_b32 s10, v2
+; GFX7-NEXT: v_readfirstlane_b32 s11, v3
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX7-NEXT: buffer_load_ushort v8, v4, s[8:11], 0 offen offset:1026
+; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_cbranch_execnz .LBB28_3
+; GFX7-NEXT: ; %bb.4:
+; GFX7-NEXT: s_mov_b64 exec, s[6:7]
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v6
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v8
; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX7-NEXT: s_mov_b64 s[6:7], 0
-; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v6
-; GFX7-NEXT: .LBB28_3: ; %atomicrmw.start
+; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
+; GFX7-NEXT: .LBB28_5: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Loop Header: Depth=1
-; GFX7-NEXT: ; Child Loop BB28_4 Depth 2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v8
-; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v8
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v6
-; GFX7-NEXT: v_add_f32_e32 v5, v5, v10
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v9
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v7, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v8
+; GFX7-NEXT: ; Child Loop BB28_6 Depth 2
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v7
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v8
+; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
+; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX7-NEXT: v_add_f32_e32 v8, v8, v10
+; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX7-NEXT: v_add_f32_e32 v5, v5, v9
+; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v8
; GFX7-NEXT: v_alignbit_b32 v6, v7, v6, 16
+; GFX7-NEXT: v_alignbit_b32 v5, v8, v5, 16
; GFX7-NEXT: v_mov_b32_e32 v8, v6
; GFX7-NEXT: s_mov_b64 s[12:13], exec
; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: .LBB28_4: ; Parent Loop BB28_3 Depth=1
+; GFX7-NEXT: .LBB28_6: ; Parent Loop BB28_5 Depth=1
; GFX7-NEXT: ; => This Inner Loop Header: Depth=2
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
; GFX7-NEXT: v_readfirstlane_b32 s9, v1
@@ -10249,22 +10991,22 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v4, s[8:11], 0 offen offset:1024 glc
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB28_4
-; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB28_3 Depth=1
+; GFX7-NEXT: s_cbranch_execnz .LBB28_6
+; GFX7-NEXT: ; %bb.7: ; in Loop: Header=BB28_5 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
+; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v7
+; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX7-NEXT: s_cbranch_execnz .LBB28_3
-; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX7-NEXT: s_cbranch_execnz .LBB28_5
+; GFX7-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v8
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v8
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v7
; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
@@ -10284,23 +11026,38 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
-; GFX6-NEXT: ; implicit-def: $vgpr4
+; GFX6-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1024
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB28_1
; GFX6-NEXT: ; %bb.2:
; GFX6-NEXT: s_mov_b64 exec, s[6:7]
+; GFX6-NEXT: s_mov_b64 s[6:7], exec
+; GFX6-NEXT: .LBB28_3: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: v_readfirstlane_b32 s8, v0
+; GFX6-NEXT: v_readfirstlane_b32 s9, v1
+; GFX6-NEXT: v_readfirstlane_b32 s10, v2
+; GFX6-NEXT: v_readfirstlane_b32 s11, v3
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX6-NEXT: buffer_load_ushort v9, v4, s[8:11], 0 offen offset:1026
+; GFX6-NEXT: ; implicit-def: $vgpr4
+; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX6-NEXT: s_cbranch_execnz .LBB28_3
+; GFX6-NEXT: ; %bb.4:
+; GFX6-NEXT: s_mov_b64 exec, s[6:7]
; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v9
; GFX6-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v6
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v7
; GFX6-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX6-NEXT: s_mov_b64 s[6:7], 0
; GFX6-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
-; GFX6-NEXT: .LBB28_3: ; %atomicrmw.start
+; GFX6-NEXT: .LBB28_5: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Loop Header: Depth=1
-; GFX6-NEXT: ; Child Loop BB28_4 Depth 2
+; GFX6-NEXT: ; Child Loop BB28_6 Depth 2
; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v7
; GFX6-NEXT: v_mul_f32_e32 v7, 1.0, v4
; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v7
@@ -10314,7 +11071,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX6-NEXT: v_mov_b32_e32 v7, v5
; GFX6-NEXT: s_mov_b64 s[12:13], exec
; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: .LBB28_4: ; Parent Loop BB28_3 Depth=1
+; GFX6-NEXT: .LBB28_6: ; Parent Loop BB28_5 Depth=1
; GFX6-NEXT: ; => This Inner Loop Header: Depth=2
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
; GFX6-NEXT: v_readfirstlane_b32 s9, v1
@@ -10327,8 +11084,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v8, s[8:11], 0 offen glc
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_cbranch_execnz .LBB28_4
-; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB28_3 Depth=1
+; GFX6-NEXT: s_cbranch_execnz .LBB28_6
+; GFX6-NEXT: ; %bb.7: ; in Loop: Header=BB28_5 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
@@ -10338,8 +11095,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX6-NEXT: v_lshlrev_b32_e32 v7, 16, v6
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX6-NEXT: s_cbranch_execnz .LBB28_3
-; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX6-NEXT: s_cbranch_execnz .LBB28_5
+; GFX6-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v4
; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
@@ -10369,18 +11126,22 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-NEXT: v_mov_b32_e32 v0, s16
-; GFX942-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
+; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
-; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
-; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
+; GFX942-NEXT: s_waitcnt vmcnt(1)
+; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_lshl_or_b32 v0, v5, 16, v0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v7, v0
; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v7
; GFX942-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
@@ -10416,37 +11177,41 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_lshlrev_b32 v3, 16, v1
-; GFX11-TRUE16-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: buffer_load_u16 v4, v0, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v0
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -10465,39 +11230,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_and_b32 v3, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v0
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v1, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v1, v7, v9 :: v_dual_lshlrev_b32 v0, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v0, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v5, v8, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_perm_b32 v5, v1, v0, 0x7060302
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
@@ -10516,16 +11287,21 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, s20
-; GFX10-NEXT: v_mov_b32_e32 v4, s20
+; GFX10-NEXT: v_mov_b32_e32 v1, s20
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v6
@@ -10560,18 +11336,22 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: v_mov_b32_e32 v0, s20
-; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_waitcnt vmcnt(1)
+; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshl_or_b32 v0, v5, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v0
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v7
; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
@@ -10603,18 +11383,22 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, v0
-; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
-; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
+; GFX908-NEXT: s_waitcnt vmcnt(1)
+; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_lshl_or_b32 v0, v5, 16, v0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v0
; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v6
; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
@@ -10647,16 +11431,20 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s20
-; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_mov_b32_e32 v1, s20
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX8-NEXT: s_mov_b64 s[6:7], 0
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
@@ -10692,8 +11480,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX7-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -10701,9 +11491,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -10740,7 +11531,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
@@ -10749,9 +11541,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX6-NEXT: s_waitcnt vmcnt(1)
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX6-NEXT: v_mov_b32_e32 v2, s6
; GFX6-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -10809,16 +11602,21 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v1, s16
-; GFX942-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: s_mov_b64 s[6:7], 0
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
+; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX942-NEXT: s_mov_b64 s[6:7], 0
+; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
+; GFX942-NEXT: s_waitcnt vmcnt(1)
+; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_lshl_or_b32 v1, v5, 16, v0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX942-NEXT: v_add_f32_e32 v0, v0, v2
@@ -10853,21 +11651,25 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_and_b32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: buffer_load_u16 v4, v1, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, v5, v2 :: v_dual_add_f32 v0, v0, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -10899,22 +11701,30 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_lshlrev_b32 v2, 16, v0
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_and_b32 v3, 0xffff0000, v0
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
-; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v5, v3 :: v_dual_add_f32 v0, v0, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v0, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
@@ -10947,14 +11757,20 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s20
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
+; GFX10-NEXT: s_mov_b32 s5, 0
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshl_or_b32 v1, v4, 16, v1
; GFX10-NEXT: v_mov_b32_e32 v4, s20
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
; GFX10-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -10990,16 +11806,21 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_waitcnt vmcnt(1)
+; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshl_or_b32 v1, v5, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX90A-NEXT: v_add_f32_e32 v0, v0, v2
@@ -11032,16 +11853,21 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
+; GFX908-NEXT: s_waitcnt vmcnt(1)
+; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_lshl_or_b32 v1, v5, 16, v0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX908-NEXT: v_add_f32_e32 v0, v0, v2
@@ -11075,14 +11901,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s20
-; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX8-NEXT: v_add_f32_e32 v0, v0, v2
@@ -11118,8 +11949,10 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX7-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -11127,9 +11960,10 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -11162,7 +11996,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
@@ -11171,9 +12006,10 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX6-NEXT: s_waitcnt vmcnt(1)
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX6-NEXT: v_mov_b32_e32 v2, s6
; GFX6-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -11226,18 +12062,22 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-NEXT: v_mov_b32_e32 v0, s16
-; GFX942-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
+; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
-; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
-; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
+; GFX942-NEXT: s_waitcnt vmcnt(1)
+; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_lshl_or_b32 v0, v5, 16, v0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v7, v0
; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v7
; GFX942-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
@@ -11273,37 +12113,41 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_lshlrev_b32 v3, 16, v1
-; GFX11-TRUE16-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: buffer_load_u16 v4, v0, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v0
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -11322,39 +12166,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_and_b32 v3, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v0
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v1, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v1, v7, v9 :: v_dual_lshlrev_b32 v0, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v0, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v5, v8, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_perm_b32 v5, v1, v0, 0x7060302
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
@@ -11373,16 +12223,21 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, s20
-; GFX10-NEXT: v_mov_b32_e32 v4, s20
+; GFX10-NEXT: v_mov_b32_e32 v1, s20
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v6
@@ -11417,18 +12272,22 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: v_mov_b32_e32 v0, s20
-; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_waitcnt vmcnt(1)
+; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshl_or_b32 v0, v5, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v0
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v7
; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
@@ -11460,18 +12319,22 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, v0
-; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
-; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
+; GFX908-NEXT: s_waitcnt vmcnt(1)
+; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_lshl_or_b32 v0, v5, 16, v0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v0
; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v6
; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
@@ -11504,16 +12367,20 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s20
-; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_mov_b32_e32 v1, s20
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX8-NEXT: s_mov_b64 s[6:7], 0
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
@@ -11549,8 +12416,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX7-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -11558,9 +12427,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -11597,7 +12467,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
@@ -11606,9 +12477,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX6-NEXT: s_waitcnt vmcnt(1)
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX6-NEXT: v_mov_b32_e32 v2, s6
; GFX6-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -11666,16 +12538,21 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v1, s16
-; GFX942-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: s_mov_b64 s[6:7], 0
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
+; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX942-NEXT: s_mov_b64 s[6:7], 0
+; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
+; GFX942-NEXT: s_waitcnt vmcnt(1)
+; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_lshl_or_b32 v1, v5, 16, v0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX942-NEXT: v_add_f32_e32 v0, v0, v2
@@ -11710,21 +12587,25 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_and_b32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: buffer_load_u16 v4, v1, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, v5, v2 :: v_dual_add_f32 v0, v0, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -11756,22 +12637,30 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_lshlrev_b32 v2, 16, v0
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_and_b32 v3, 0xffff0000, v0
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
-; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v5, v3 :: v_dual_add_f32 v0, v0, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v0, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
@@ -11804,14 +12693,20 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s20
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
+; GFX10-NEXT: s_mov_b32 s5, 0
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshl_or_b32 v1, v4, 16, v1
; GFX10-NEXT: v_mov_b32_e32 v4, s20
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
; GFX10-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -11847,16 +12742,21 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_waitcnt vmcnt(1)
+; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshl_or_b32 v1, v5, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX90A-NEXT: v_add_f32_e32 v0, v0, v2
@@ -11889,16 +12789,21 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
+; GFX908-NEXT: s_waitcnt vmcnt(1)
+; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_lshl_or_b32 v1, v5, 16, v0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX908-NEXT: v_add_f32_e32 v0, v0, v2
@@ -11932,14 +12837,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s20
-; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX8-NEXT: v_add_f32_e32 v0, v0, v2
@@ -11975,8 +12885,10 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX7-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -11984,9 +12896,10 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -12019,7 +12932,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
@@ -12028,9 +12942,10 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX6-NEXT: s_waitcnt vmcnt(1)
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX6-NEXT: v_mov_b32_e32 v2, s6
; GFX6-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -12084,16 +12999,21 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v1, s16
-; GFX942-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: s_mov_b64 s[6:7], 0
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
+; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX942-NEXT: s_mov_b64 s[6:7], 0
+; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
+; GFX942-NEXT: s_waitcnt vmcnt(1)
+; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_lshl_or_b32 v1, v5, 16, v0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX942-NEXT: v_add_f32_e32 v0, v0, v2
@@ -12128,21 +13048,25 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_and_b32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: buffer_load_u16 v4, v1, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, v5, v2 :: v_dual_add_f32 v0, v0, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -12174,22 +13098,30 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_lshlrev_b32 v2, 16, v0
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_and_b32 v3, 0xffff0000, v0
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
-; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v5, v3 :: v_dual_add_f32 v0, v0, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v0, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
@@ -12222,14 +13154,20 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s20
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
+; GFX10-NEXT: s_mov_b32 s5, 0
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshl_or_b32 v1, v4, 16, v1
; GFX10-NEXT: v_mov_b32_e32 v4, s20
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
; GFX10-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -12265,16 +13203,21 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_waitcnt vmcnt(1)
+; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshl_or_b32 v1, v5, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX90A-NEXT: v_add_f32_e32 v0, v0, v2
@@ -12307,16 +13250,21 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
+; GFX908-NEXT: s_waitcnt vmcnt(1)
+; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_lshl_or_b32 v1, v5, 16, v0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX908-NEXT: v_add_f32_e32 v0, v0, v2
@@ -12350,14 +13298,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s20
-; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX8-NEXT: v_add_f32_e32 v0, v0, v2
@@ -12393,8 +13346,10 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX7-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -12402,9 +13357,10 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -12437,7 +13393,8 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
@@ -12446,9 +13403,10 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX6-NEXT: s_waitcnt vmcnt(1)
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX6-NEXT: v_mov_b32_e32 v2, s6
; GFX6-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index 08e1a59245fd6..29bde07836045 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -6004,53 +6004,99 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; --------------------------------------------------------------------
define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
-; GFX12-NEXT: v_mov_b32_e32 v3, s16
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-NEXT: v_pk_max_num_f16 v2, v1, v1
-; GFX12-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen offset:1024
-; GFX12-NEXT: .LBB16_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v5, v0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v0, v5, v5
-; GFX12-NEXT: v_pk_max_num_f16 v4, v0, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB16_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-TRUE16-NEXT: s_clause 0x1
+; GFX12-TRUE16-NEXT: buffer_load_u16 v3, v0, s[0:3], null offen offset:1026
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v2, v1, v1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, s16
+; GFX12-TRUE16-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v0, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v4, v0, v2
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB16_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-FAKE16-NEXT: s_clause 0x1
+; GFX12-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1024
+; GFX12-FAKE16-NEXT: buffer_load_u16 v3, v2, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v2, v0, v0
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v0, v3, 16, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, s16
+; GFX12-FAKE16-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, v0
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v0, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v4, v0, v2
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB16_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-NEXT: v_mov_b32_e32 v0, s16
-; GFX942-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
+; GFX942-NEXT: buffer_load_ushort v3, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_pk_max_f16 v2, v0, v0
; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: v_pk_max_f16 v2, v1, v1
+; GFX942-NEXT: s_waitcnt vmcnt(1)
+; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_lshl_or_b32 v0, v4, 16, v0
; GFX942-NEXT: v_mov_b32_e32 v3, s16
; GFX942-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v5, v0
; GFX942-NEXT: v_pk_max_f16 v0, v5, v5
; GFX942-NEXT: buffer_wbl2 sc1
@@ -6069,50 +6115,94 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
-; GFX11-NEXT: v_mov_b32_e32 v3, s16
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_pk_max_f16 v2, v1, v1
-; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-NEXT: .LBB16_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v5, v0
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v0, v5, v5
-; GFX11-NEXT: v_pk_max_f16 v4, v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB16_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: buffer_load_u16 v3, v0, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v2, v1, v1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s16
+; GFX11-TRUE16-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v0, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v4, v0, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB16_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v3, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v2, v0, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v3, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s16
+; GFX11-FAKE16-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v0
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v0, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v4, v0, v2
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB16_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, s20
-; GFX10-NEXT: v_mov_b32_e32 v3, s20
+; GFX10-NEXT: v_mov_b32_e32 v1, s20
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: v_pk_max_f16 v2, v1, v1
-; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: v_pk_max_f16 v2, v0, v0
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v1
; GFX10-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v5, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_pk_max_f16 v0, v5, v5
@@ -6134,15 +6224,19 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: v_mov_b32_e32 v0, s20
-; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
+; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_pk_max_f16 v2, v0, v0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_pk_max_f16 v2, v1, v1
+; GFX90A-NEXT: s_waitcnt vmcnt(1)
+; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshl_or_b32 v0, v4, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: v_pk_max_f16 v0, v5, v5
; GFX90A-NEXT: v_pk_max_f16 v4, v0, v2
@@ -6161,15 +6255,19 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, v0
-; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_pk_max_f16 v2, v0, v0
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_pk_max_f16 v2, v1, v1
+; GFX908-NEXT: s_waitcnt vmcnt(1)
+; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_lshl_or_b32 v0, v4, 16, v0
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v5, v0
; GFX908-NEXT: v_pk_max_f16 v0, v5, v5
; GFX908-NEXT: v_pk_max_f16 v4, v0, v2
@@ -6189,16 +6287,20 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s20
-; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_mov_b32_e32 v1, s20
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_max_f16_sdwa v2, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v3, v0, v0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_max_f16_sdwa v2, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v3, v1, v1
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
; GFX8-NEXT: v_max_f16_sdwa v0, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v1, v6, v6
@@ -6222,16 +6324,17 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_mov_b32_e32 v4, s20
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -6263,18 +6366,18 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, s20
+; GFX6-NEXT: buffer_load_ushort v1, v2, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_mov_b32_e32 v4, s6
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v1
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -6309,50 +6412,99 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
}
define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v1, s16
-; GFX12-NEXT: v_pk_max_num_f16 v2, v0, v0
-; GFX12-NEXT: v_mov_b32_e32 v3, s16
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: buffer_load_b32 v1, v1, s[0:3], null offen offset:1024
-; GFX12-NEXT: .LBB17_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v0, v1, v1
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v0, v0, v2
-; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX12-NEXT: v_mov_b32_e32 v1, v4
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB17_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, s16
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v2, v0, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-TRUE16-NEXT: s_clause 0x1
+; GFX12-TRUE16-NEXT: buffer_load_u16 v3, v1, s[0:3], null offen offset:1026
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, s16
+; GFX12-TRUE16-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v0, v1, v1
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v0, v0, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB17_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-FAKE16-NEXT: s_clause 0x1
+; GFX12-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1024
+; GFX12-FAKE16-NEXT: buffer_load_u16 v3, v2, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v2, v0, v0
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, s16
+; GFX12-FAKE16-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v0, v1, v1
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v0, v0, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB17_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v1, s16
-; GFX942-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
+; GFX942-NEXT: buffer_load_ushort v3, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
; GFX942-NEXT: v_pk_max_f16 v2, v0, v0
+; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: s_waitcnt vmcnt(1)
+; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_lshl_or_b32 v1, v4, 16, v0
; GFX942-NEXT: v_mov_b32_e32 v3, s16
; GFX942-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_pk_max_f16 v0, v1, v1
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -6371,47 +6523,94 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s16
-; GFX11-NEXT: v_pk_max_f16 v2, v0, v0
-; GFX11-NEXT: v_mov_b32_e32 v3, s16
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen offset:1024
-; GFX11-NEXT: .LBB17_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_pk_max_f16 v0, v1, v1
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v0, v0, v2
-; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX11-NEXT: v_mov_b32_e32 v1, v4
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB17_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s16
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v2, v0, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: buffer_load_u16 v3, v1, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s16
+; GFX11-TRUE16-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v0, v0, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB17_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v3, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v2, v0, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s16
+; GFX11-FAKE16-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v0, v0, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB17_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s20
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
+; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
; GFX10-NEXT: v_pk_max_f16 v2, v0, v0
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
; GFX10-NEXT: v_mov_b32_e32 v3, s20
-; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshl_or_b32 v1, v4, 16, v1
; GFX10-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_pk_max_f16 v0, v1, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_pk_max_f16 v0, v0, v2
@@ -6434,13 +6633,18 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
+; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
; GFX90A-NEXT: v_pk_max_f16 v2, v0, v0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_waitcnt vmcnt(1)
+; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshl_or_b32 v1, v4, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v0, v1, v1
; GFX90A-NEXT: v_pk_max_f16 v0, v0, v2
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
@@ -6460,13 +6664,18 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
; GFX908-NEXT: v_pk_max_f16 v2, v0, v0
+; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: s_waitcnt vmcnt(1)
+; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_lshl_or_b32 v1, v4, 16, v0
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_pk_max_f16 v0, v1, v1
; GFX908-NEXT: v_pk_max_f16 v0, v0, v2
; GFX908-NEXT: v_mov_b32_e32 v5, v1
@@ -6487,14 +6696,19 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s20
-; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
; GFX8-NEXT: v_max_f16_sdwa v2, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v3, v0, v0
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f16_sdwa v0, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v5, v1, v1
; GFX8-NEXT: v_max_f16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -6518,16 +6732,17 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_mov_b32_e32 v4, s20
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -6556,18 +6771,18 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, s20
+; GFX6-NEXT: buffer_load_ushort v1, v2, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_mov_b32_e32 v4, s6
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v1
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -6599,74 +6814,185 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
}
define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, exec_lo
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s5, s4
-; GFX12-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_readfirstlane_b32 s0, v0
-; GFX12-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-NEXT: v_readfirstlane_b32 s2, v2
-; GFX12-NEXT: v_readfirstlane_b32 s3, v3
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_load_b32 v7, v4, s[0:3], null offen offset:1024
-; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB18_1
-; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s4
-; GFX12-NEXT: v_pk_max_num_f16 v8, v5, v5
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB18_3: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Loop Header: Depth=1
-; GFX12-NEXT: ; Child Loop BB18_4 Depth 2
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v5, v7, v7
-; GFX12-NEXT: s_mov_b32 s5, exec_lo
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s6, s5
-; GFX12-NEXT: v_pk_max_num_f16 v6, v5, v8
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_mov_b32_e32 v5, v6
-; GFX12-NEXT: v_mov_b32_e32 v6, v7
-; GFX12-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
-; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-NEXT: v_readfirstlane_b32 s0, v0
-; GFX12-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-NEXT: v_readfirstlane_b32 s2, v2
-; GFX12-NEXT: v_readfirstlane_b32 s3, v3
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX12-NEXT: s_cbranch_execnz .LBB18_4
-; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX12-NEXT: s_mov_b32 exec_lo, s5
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX12-NEXT: v_mov_b32_e32 v7, v5
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB18_3
-; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: v_mov_b32_e32 v0, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
+; GFX12-TRUE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: buffer_load_u16 v7, v4, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_1
+; GFX12-TRUE16-NEXT: ; %bb.2:
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
+; GFX12-TRUE16-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: buffer_load_u16 v6, v4, s[0:3], null offen offset:1026
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_3
+; GFX12-TRUE16-NEXT: ; %bb.4:
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v8, v5, v5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-TRUE16-NEXT: .LBB18_5: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: ; Child Loop BB18_6 Depth 2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v5, v7, v7
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v6, v5, v8
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v7
+; GFX12-TRUE16-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
+; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_6
+; GFX12-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v5
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_5
+; GFX12-TRUE16-NEXT: ; %bb.8: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
+; GFX12-FAKE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: buffer_load_u16 v6, v4, s[0:3], null offen offset:1024
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_1
+; GFX12-FAKE16-NEXT: ; %bb.2:
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
+; GFX12-FAKE16-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: buffer_load_u16 v7, v4, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_3
+; GFX12-FAKE16-NEXT: ; %bb.4:
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v8, v5, v5
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-FAKE16-NEXT: .LBB18_5: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: ; Child Loop BB18_6 Depth 2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v5, v7, v7
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v6, v5, v8
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, v6
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v7
+; GFX12-FAKE16-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
+; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_6
+; GFX12-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v7, v5
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_5
+; GFX12-FAKE16-NEXT: ; %bb.8: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -6682,24 +7008,42 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX942-NEXT: buffer_load_dword v9, v4, s[4:7], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v6, v4, s[4:7], 0 offen offset:1024
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB18_1
; GFX942-NEXT: ; %bb.2:
; GFX942-NEXT: s_mov_b64 exec, s[2:3]
+; GFX942-NEXT: s_mov_b64 s[2:3], exec
+; GFX942-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
+; GFX942-NEXT: v_readfirstlane_b32 s4, v0
+; GFX942-NEXT: v_readfirstlane_b32 s5, v1
+; GFX942-NEXT: v_readfirstlane_b32 s6, v2
+; GFX942-NEXT: v_readfirstlane_b32 s7, v3
+; GFX942-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
+; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT: buffer_load_ushort v7, v4, s[4:7], 0 offen offset:1026
+; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_cbranch_execnz .LBB18_3
+; GFX942-NEXT: ; %bb.4:
+; GFX942-NEXT: s_mov_b64 exec, s[2:3]
+; GFX942-NEXT: s_mov_b32 s0, 0x5040100
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_perm_b32 v9, v7, v6, s0
; GFX942-NEXT: s_mov_b64 s[2:3], 0
; GFX942-NEXT: v_pk_max_f16 v5, v5, v5
-; GFX942-NEXT: .LBB18_3: ; %atomicrmw.start
+; GFX942-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Loop Header: Depth=1
-; GFX942-NEXT: ; Child Loop BB18_4 Depth 2
-; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: ; Child Loop BB18_6 Depth 2
; GFX942-NEXT: v_pk_max_f16 v6, v9, v9
; GFX942-NEXT: s_mov_b64 s[8:9], exec
; GFX942-NEXT: v_pk_max_f16 v8, v6, v5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b64_e32 v[6:7], v[8:9]
-; GFX942-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
+; GFX942-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX942-NEXT: ; => This Inner Loop Header: Depth=2
; GFX942-NEXT: v_readfirstlane_b32 s4, v0
; GFX942-NEXT: v_readfirstlane_b32 s5, v1
@@ -6713,8 +7057,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 sc0
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB18_4
-; GFX942-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
+; GFX942-NEXT: s_cbranch_execnz .LBB18_6
+; GFX942-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX942-NEXT: s_mov_b64 exec, s[8:9]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v6, v9
@@ -6722,73 +7066,171 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX942-NEXT: v_mov_b32_e32 v9, v6
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_cbranch_execnz .LBB18_3
-; GFX942-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX942-NEXT: s_cbranch_execnz .LBB18_5
+; GFX942-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-NEXT: v_mov_b32_e32 v0, v6
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_mov_b32 s6, s5
-; GFX11-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-NEXT: v_readfirstlane_b32 s2, v2
-; GFX11-NEXT: v_readfirstlane_b32 s3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
-; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX11-NEXT: s_cbranch_execnz .LBB18_1
-; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-NEXT: v_pk_max_f16 v8, v5, v5
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB18_3: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Loop Header: Depth=1
-; GFX11-NEXT: ; Child Loop BB18_4 Depth 2
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_pk_max_f16 v5, v7, v7
-; GFX11-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_mov_b32 s6, s5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v6, v5, v8
-; GFX11-NEXT: v_mov_b32_e32 v5, v6
-; GFX11-NEXT: v_mov_b32_e32 v6, v7
-; GFX11-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
-; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-NEXT: v_readfirstlane_b32 s2, v2
-; GFX11-NEXT: v_readfirstlane_b32 s3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
-; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX11-NEXT: s_cbranch_execnz .LBB18_4
-; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX11-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX11-NEXT: v_mov_b32_e32 v7, v5
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB18_3
-; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: v_mov_b32_e32 v0, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_1
+; GFX11-TRUE16-NEXT: ; %bb.2:
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_u16 v6, v4, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_3
+; GFX11-TRUE16-NEXT: ; %bb.4:
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v8, v5, v5
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB18_5: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: ; Child Loop BB18_6 Depth 2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v5, v7, v7
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v6, v5, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v7
+; GFX11-TRUE16-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
+; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_6
+; GFX11-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v5
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_5
+; GFX11-TRUE16-NEXT: ; %bb.8: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_u16 v6, v4, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_1
+; GFX11-FAKE16-NEXT: ; %bb.2:
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_3
+; GFX11-FAKE16-NEXT: ; %bb.4:
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v8, v5, v5
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB18_5: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: ; Child Loop BB18_6 Depth 2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v5, v7, v7
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v6, v5, v8
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v6
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v7
+; GFX11-FAKE16-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
+; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_6
+; GFX11-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v5
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_5
+; GFX11-FAKE16-NEXT: ; %bb.8: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -6804,17 +7246,34 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX10-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v6, v4, s[4:7], 0 offen offset:1024
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB18_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s10, s9
+; GFX10-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_ushort v7, v4, s[4:7], 0 offen offset:1026
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX10-NEXT: s_cbranch_execnz .LBB18_3
+; GFX10-NEXT: ; %bb.4:
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
; GFX10-NEXT: v_pk_max_f16 v8, v5, v5
-; GFX10-NEXT: .LBB18_3: ; %atomicrmw.start
+; GFX10-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
-; GFX10-NEXT: ; Child Loop BB18_4 Depth 2
-; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: ; Child Loop BB18_6 Depth 2
; GFX10-NEXT: v_pk_max_f16 v5, v7, v7
; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -6822,7 +7281,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX10-NEXT: v_pk_max_f16 v6, v5, v8
; GFX10-NEXT: v_mov_b32_e32 v5, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v7
-; GFX10-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
+; GFX10-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
@@ -6834,8 +7293,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[4:7], 0 offen offset:1024 glc
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
-; GFX10-NEXT: s_cbranch_execnz .LBB18_4
-; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
+; GFX10-NEXT: s_cbranch_execnz .LBB18_6
+; GFX10-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -6846,8 +7305,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
-; GFX10-NEXT: s_cbranch_execnz .LBB18_3
-; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX10-NEXT: s_cbranch_execnz .LBB18_5
+; GFX10-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -6866,22 +7325,40 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_nop 0
-; GFX90A-NEXT: buffer_load_dword v9, v4, s[8:11], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB18_1
; GFX90A-NEXT: ; %bb.2:
; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
+; GFX90A-NEXT: s_mov_b64 s[6:7], exec
+; GFX90A-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
+; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
+; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
+; GFX90A-NEXT: v_readfirstlane_b32 s11, v3
+; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX90A-NEXT: s_nop 0
+; GFX90A-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1026
+; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_cbranch_execnz .LBB18_3
+; GFX90A-NEXT: ; %bb.4:
+; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
+; GFX90A-NEXT: s_mov_b32 s4, 0x5040100
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_perm_b32 v9, v7, v6, s4
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_pk_max_f16 v5, v5, v5
-; GFX90A-NEXT: .LBB18_3: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
-; GFX90A-NEXT: ; Child Loop BB18_4 Depth 2
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; Child Loop BB18_6 Depth 2
; GFX90A-NEXT: v_pk_max_f16 v6, v9, v9
; GFX90A-NEXT: v_pk_max_f16 v8, v6, v5
; GFX90A-NEXT: s_mov_b64 s[12:13], exec
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[8:9], v[8:9] op_sel:[0,1]
-; GFX90A-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
+; GFX90A-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
@@ -6894,8 +7371,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB18_4
-; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
+; GFX90A-NEXT: s_cbranch_execnz .LBB18_6
+; GFX90A-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v6, v9
@@ -6903,8 +7380,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX90A-NEXT: v_mov_b32_e32 v9, v6
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB18_3
-; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX90A-NEXT: s_cbranch_execnz .LBB18_5
+; GFX90A-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v6
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -6923,23 +7400,41 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB18_1
; GFX908-NEXT: ; %bb.2:
; GFX908-NEXT: s_mov_b64 exec, s[6:7]
+; GFX908-NEXT: s_mov_b64 s[6:7], exec
+; GFX908-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: v_readfirstlane_b32 s8, v0
+; GFX908-NEXT: v_readfirstlane_b32 s9, v1
+; GFX908-NEXT: v_readfirstlane_b32 s10, v2
+; GFX908-NEXT: v_readfirstlane_b32 s11, v3
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX908-NEXT: s_nop 0
+; GFX908-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1026
+; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_cbranch_execnz .LBB18_3
+; GFX908-NEXT: ; %bb.4:
+; GFX908-NEXT: s_mov_b64 exec, s[6:7]
+; GFX908-NEXT: s_mov_b32 s4, 0x5040100
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_perm_b32 v7, v7, v6, s4
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: v_pk_max_f16 v8, v5, v5
-; GFX908-NEXT: .LBB18_3: ; %atomicrmw.start
+; GFX908-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Loop Header: Depth=1
-; GFX908-NEXT: ; Child Loop BB18_4 Depth 2
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: ; Child Loop BB18_6 Depth 2
; GFX908-NEXT: v_pk_max_f16 v5, v7, v7
; GFX908-NEXT: v_pk_max_f16 v6, v5, v8
; GFX908-NEXT: v_mov_b32_e32 v5, v6
; GFX908-NEXT: s_mov_b64 s[12:13], exec
; GFX908-NEXT: v_mov_b32_e32 v6, v7
-; GFX908-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
+; GFX908-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
; GFX908-NEXT: v_readfirstlane_b32 s9, v1
@@ -6952,8 +7447,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB18_4
-; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
+; GFX908-NEXT: s_cbranch_execnz .LBB18_6
+; GFX908-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
@@ -6961,8 +7456,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB18_3
-; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX908-NEXT: s_cbranch_execnz .LBB18_5
+; GFX908-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
@@ -6981,18 +7476,36 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_nop 0
-; GFX8-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
+; GFX8-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB18_1
; GFX8-NEXT: ; %bb.2:
; GFX8-NEXT: s_mov_b64 exec, s[6:7]
+; GFX8-NEXT: s_mov_b64 s[6:7], exec
+; GFX8-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_readfirstlane_b32 s8, v0
+; GFX8-NEXT: v_readfirstlane_b32 s9, v1
+; GFX8-NEXT: v_readfirstlane_b32 s10, v2
+; GFX8-NEXT: v_readfirstlane_b32 s11, v3
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX8-NEXT: s_nop 0
+; GFX8-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1026
+; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB18_3
+; GFX8-NEXT: ; %bb.4:
+; GFX8-NEXT: s_mov_b64 exec, s[6:7]
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX8-NEXT: v_or_b32_e32 v7, v6, v7
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_max_f16_sdwa v8, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v9, v5, v5
-; GFX8-NEXT: .LBB18_3: ; %atomicrmw.start
+; GFX8-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Loop Header: Depth=1
-; GFX8-NEXT: ; Child Loop BB18_4 Depth 2
-; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: ; Child Loop BB18_6 Depth 2
; GFX8-NEXT: v_max_f16_sdwa v5, v7, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v6, v7, v7
; GFX8-NEXT: v_max_f16_sdwa v5, v5, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -7001,7 +7514,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX8-NEXT: v_mov_b32_e32 v5, v6
; GFX8-NEXT: s_mov_b64 s[12:13], exec
; GFX8-NEXT: v_mov_b32_e32 v6, v7
-; GFX8-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
+; GFX8-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
; GFX8-NEXT: v_readfirstlane_b32 s9, v1
@@ -7014,8 +7527,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB18_4
-; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
+; GFX8-NEXT: s_cbranch_execnz .LBB18_6
+; GFX8-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
@@ -7023,8 +7536,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX8-NEXT: v_mov_b32_e32 v7, v5
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB18_3
-; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX8-NEXT: s_cbranch_execnz .LBB18_5
+; GFX8-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -7043,34 +7556,48 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: buffer_load_dword v6, v4, s[8:11], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB18_1
; GFX7-NEXT: ; %bb.2:
; GFX7-NEXT: s_mov_b64 exec, s[6:7]
+; GFX7-NEXT: s_mov_b64 s[6:7], exec
+; GFX7-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: v_readfirstlane_b32 s8, v0
+; GFX7-NEXT: v_readfirstlane_b32 s9, v1
+; GFX7-NEXT: v_readfirstlane_b32 s10, v2
+; GFX7-NEXT: v_readfirstlane_b32 s11, v3
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX7-NEXT: buffer_load_ushort v10, v4, s[8:11], 0 offen offset:1026
+; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_cbranch_execnz .LBB18_3
+; GFX7-NEXT: ; %bb.4:
+; GFX7-NEXT: s_mov_b64 exec, s[6:7]
; GFX7-NEXT: v_cvt_f32_f16_e32 v9, v7
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v6
; GFX7-NEXT: s_mov_b64 s[6:7], 0
-; GFX7-NEXT: .LBB18_3: ; %atomicrmw.start
+; GFX7-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Loop Header: Depth=1
-; GFX7-NEXT: ; Child Loop BB18_4 Depth 2
-; GFX7-NEXT: v_cvt_f32_f16_e32 v8, v7
-; GFX7-NEXT: v_cvt_f32_f16_e32 v10, v6
+; GFX7-NEXT: ; Child Loop BB18_6 Depth 2
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v10
+; GFX7-NEXT: v_cvt_f32_f16_e32 v8, v6
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX7-NEXT: v_max_f32_e32 v8, v8, v9
-; GFX7-NEXT: v_max_f32_e32 v10, v10, v5
-; GFX7-NEXT: v_cvt_f16_f32_e32 v11, v8
-; GFX7-NEXT: v_cvt_f16_f32_e32 v10, v10
-; GFX7-NEXT: v_or_b32_e32 v8, v6, v7
; GFX7-NEXT: s_mov_b64 s[12:13], exec
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v11
-; GFX7-NEXT: v_or_b32_e32 v7, v10, v6
+; GFX7-NEXT: v_max_f32_e32 v7, v7, v9
+; GFX7-NEXT: v_max_f32_e32 v8, v8, v5
+; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v7
+; GFX7-NEXT: v_cvt_f16_f32_e32 v11, v8
+; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v10
+; GFX7-NEXT: v_or_b32_e32 v8, v6, v8
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; GFX7-NEXT: v_or_b32_e32 v7, v11, v6
; GFX7-NEXT: v_mov_b32_e32 v6, v7
; GFX7-NEXT: v_mov_b32_e32 v7, v8
-; GFX7-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
+; GFX7-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX7-NEXT: ; => This Inner Loop Header: Depth=2
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
; GFX7-NEXT: v_readfirstlane_b32 s9, v1
@@ -7083,20 +7610,20 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB18_4
-; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
+; GFX7-NEXT: s_cbranch_execnz .LBB18_6
+; GFX7-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
-; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v10, 16, v6
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX7-NEXT: s_cbranch_execnz .LBB18_3
-; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX7-NEXT: s_cbranch_execnz .LBB18_5
+; GFX7-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v7
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v10
; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
@@ -7115,36 +7642,50 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: buffer_load_dword v6, v4, s[8:11], 0 offen offset:1024
-; GFX6-NEXT: ; implicit-def: $vgpr4
+; GFX6-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB18_1
; GFX6-NEXT: ; %bb.2:
; GFX6-NEXT: s_mov_b64 exec, s[6:7]
+; GFX6-NEXT: s_mov_b64 s[6:7], exec
+; GFX6-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: v_readfirstlane_b32 s8, v0
+; GFX6-NEXT: v_readfirstlane_b32 s9, v1
+; GFX6-NEXT: v_readfirstlane_b32 s10, v2
+; GFX6-NEXT: v_readfirstlane_b32 s11, v3
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX6-NEXT: buffer_load_ushort v11, v4, s[8:11], 0 offen offset:1026
+; GFX6-NEXT: ; implicit-def: $vgpr4
+; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX6-NEXT: s_cbranch_execnz .LBB18_3
+; GFX6-NEXT: ; %bb.4:
+; GFX6-NEXT: s_mov_b64 exec, s[6:7]
; GFX6-NEXT: v_cvt_f32_f16_e32 v9, v7
; GFX6-NEXT: v_cvt_f32_f16_e32 v10, v5
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
; GFX6-NEXT: s_mov_b64 s[6:7], 0
-; GFX6-NEXT: .LBB18_3: ; %atomicrmw.start
+; GFX6-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Loop Header: Depth=1
-; GFX6-NEXT: ; Child Loop BB18_4 Depth 2
-; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v4
+; GFX6-NEXT: ; Child Loop BB18_6 Depth 2
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v11
+; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v6
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v6
; GFX6-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_max_f32_e32 v5, v5, v9
-; GFX6-NEXT: v_max_f32_e32 v7, v7, v10
-; GFX6-NEXT: v_cvt_f16_f32_e32 v11, v5
-; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v7
-; GFX6-NEXT: v_or_b32_e32 v5, v6, v4
; GFX6-NEXT: s_mov_b64 s[12:13], exec
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v11
+; GFX6-NEXT: v_max_f32_e32 v4, v4, v9
+; GFX6-NEXT: v_max_f32_e32 v5, v5, v10
+; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v5
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v11
+; GFX6-NEXT: v_or_b32_e32 v5, v6, v5
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX6-NEXT: v_or_b32_e32 v4, v7, v4
; GFX6-NEXT: v_mov_b32_e32 v7, v5
; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
+; GFX6-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX6-NEXT: ; => This Inner Loop Header: Depth=2
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
; GFX6-NEXT: v_readfirstlane_b32 s9, v1
@@ -7157,20 +7698,20 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v8, s[8:11], 0 offen glc
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_cbranch_execnz .LBB18_4
-; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
+; GFX6-NEXT: s_cbranch_execnz .LBB18_6
+; GFX6-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v11, 16, v6
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX6-NEXT: s_cbranch_execnz .LBB18_3
-; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX6-NEXT: s_cbranch_execnz .LBB18_5
+; GFX6-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v6
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v11
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
@@ -7193,37 +7734,41 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_lshlrev_b32 v3, 16, v1
-; GFX12-TRUE16-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: s_clause 0x1
+; GFX12-TRUE16-NEXT: buffer_load_u16 v4, v0, s[0:3], null offen offset:1026
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], null offen offset:1024
; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX12-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v0
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v1, v1, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v0, v0, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -7244,39 +7789,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX12-FAKE16-NEXT: s_clause 0x1
+; GFX12-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1024
+; GFX12-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_and_b32 v3, 0xffff0000, v1
-; GFX12-FAKE16-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen offset:1024
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX12-FAKE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v0
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v1, v1, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v1, 16, 1
; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_dual_cndmask_b32 v1, v7, v9 :: v_dual_lshlrev_b32 v0, 16, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v0, v0, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v0, 16, 1
; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v0, v5, v8, s4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_perm_b32 v5, v1, v0, 0x7060302
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
@@ -7293,18 +7844,22 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-NEXT: v_mov_b32_e32 v0, s16
-; GFX942-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
+; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
-; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
-; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
+; GFX942-NEXT: s_waitcnt vmcnt(1)
+; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_lshl_or_b32 v0, v5, 16, v0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v7, v0
; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v7
; GFX942-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
@@ -7340,37 +7895,41 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_lshlrev_b32 v3, 16, v1
-; GFX11-TRUE16-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: buffer_load_u16 v4, v0, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v0
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -7389,39 +7948,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_and_b32 v3, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v0
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_max_f32_e32 v1, v1, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v1, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v1, v7, v9 :: v_dual_lshlrev_b32 v0, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_max_f32_e32 v0, v0, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v0, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v5, v8, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_perm_b32 v5, v1, v0, 0x7060302
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
@@ -7440,16 +8005,21 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, s20
-; GFX10-NEXT: v_mov_b32_e32 v4, s20
+; GFX10-NEXT: v_mov_b32_e32 v1, s20
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v6
@@ -7484,18 +8054,22 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: v_mov_b32_e32 v0, s20
-; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_waitcnt vmcnt(1)
+; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshl_or_b32 v0, v5, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v0
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v7
; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
@@ -7527,18 +8101,22 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, v0
-; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
-; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
+; GFX908-NEXT: s_waitcnt vmcnt(1)
+; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_lshl_or_b32 v0, v5, 16, v0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v0
; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v6
; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
@@ -7571,16 +8149,20 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s20
-; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_mov_b32_e32 v1, s20
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX8-NEXT: s_mov_b64 s[6:7], 0
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
@@ -7616,18 +8198,21 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX7-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7664,18 +8249,20 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v0
+; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v0
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX6-NEXT: s_waitcnt vmcnt(1)
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX6-NEXT: v_mov_b32_e32 v2, s6
; GFX6-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7723,19 +8310,23 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_and_b32 v2, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v1, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: s_clause 0x1
+; GFX12-TRUE16-NEXT: buffer_load_u16 v4, v1, s[0:3], null offen offset:1026
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX12-TRUE16-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_dual_max_num_f32 v5, v5, v2 :: v_dual_max_num_f32 v0, v0, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -7771,20 +8362,28 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_lshlrev_b32 v2, 16, v0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_and_b32 v3, 0xffff0000, v0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v1, s[0:3], null offen offset:1024
+; GFX12-FAKE16-NEXT: s_clause 0x1
+; GFX12-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1024
+; GFX12-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX12-FAKE16-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_dual_max_num_f32 v5, v5, v3 :: v_dual_max_num_f32 v0, v0, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v0, 16, 1
; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
@@ -7817,16 +8416,21 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v1, s16
-; GFX942-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: s_mov_b64 s[6:7], 0
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
+; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX942-NEXT: s_mov_b64 s[6:7], 0
+; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
+; GFX942-NEXT: s_waitcnt vmcnt(1)
+; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_lshl_or_b32 v1, v5, 16, v0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX942-NEXT: v_max_f32_e32 v0, v0, v2
@@ -7861,21 +8465,25 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_and_b32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: buffer_load_u16 v4, v1, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_max_f32 v5, v5, v2 :: v_dual_max_f32 v0, v0, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -7907,22 +8515,30 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_lshlrev_b32 v2, 16, v0
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_and_b32 v3, 0xffff0000, v0
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
-; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_max_f32 v5, v5, v3 :: v_dual_max_f32 v0, v0, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v0, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
@@ -7955,14 +8571,20 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s20
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
+; GFX10-NEXT: s_mov_b32 s5, 0
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshl_or_b32 v1, v4, 16, v1
; GFX10-NEXT: v_mov_b32_e32 v4, s20
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
; GFX10-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -7998,16 +8620,21 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_waitcnt vmcnt(1)
+; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshl_or_b32 v1, v5, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX90A-NEXT: v_max_f32_e32 v0, v0, v2
@@ -8040,16 +8667,21 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
+; GFX908-NEXT: s_waitcnt vmcnt(1)
+; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_lshl_or_b32 v1, v5, 16, v0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX908-NEXT: v_max_f32_e32 v0, v0, v2
@@ -8083,14 +8715,19 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s20
-; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX8-NEXT: v_max_f32_e32 v0, v0, v2
@@ -8126,18 +8763,21 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX7-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8170,18 +8810,20 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v0
+; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v0
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX6-NEXT: s_waitcnt vmcnt(1)
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX6-NEXT: v_mov_b32_e32 v2, s6
; GFX6-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8237,18 +8879,39 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v4, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: buffer_load_u16 v7, v4, s[0:3], null offen offset:1024
; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX12-TRUE16-NEXT: ; %bb.2:
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
+; GFX12-TRUE16-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: buffer_load_u16 v6, v4, s[0:3], null offen offset:1026
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_3
+; GFX12-TRUE16-NEXT: ; %bb.4:
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-TRUE16-NEXT: .LBB21_3: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: ; Child Loop BB21_4 Depth 2
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; Child Loop BB21_6 Depth 2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
@@ -8274,7 +8937,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v7
-; GFX12-TRUE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX12-TRUE16-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -8287,8 +8950,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_4
-; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_6
+; GFX12-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
@@ -8297,8 +8960,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_3
-; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_5
+; GFX12-TRUE16-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v5
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -8323,18 +8986,39 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v7, v4, s[0:3], null offen offset:1024
+; GFX12-FAKE16-NEXT: buffer_load_u16 v6, v4, s[0:3], null offen offset:1024
; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX12-FAKE16-NEXT: ; %bb.2:
; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
+; GFX12-FAKE16-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: buffer_load_u16 v7, v4, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_3
+; GFX12-FAKE16-NEXT: ; %bb.4:
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX12-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-FAKE16-NEXT: .LBB21_3: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: ; Child Loop BB21_4 Depth 2
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; Child Loop BB21_6 Depth 2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
@@ -8360,7 +9044,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, v6
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v7
-; GFX12-FAKE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX12-FAKE16-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -8373,8 +9057,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_4
-; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_6
+; GFX12-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
@@ -8383,8 +9067,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_3
-; GFX12-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_5
+; GFX12-FAKE16-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, v5
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -8403,20 +9087,38 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX942-NEXT: buffer_load_dword v9, v4, s[4:7], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v6, v4, s[4:7], 0 offen offset:1024
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB21_1
; GFX942-NEXT: ; %bb.2:
; GFX942-NEXT: s_mov_b64 exec, s[2:3]
+; GFX942-NEXT: s_mov_b64 s[2:3], exec
+; GFX942-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX942-NEXT: v_readfirstlane_b32 s4, v0
+; GFX942-NEXT: v_readfirstlane_b32 s5, v1
+; GFX942-NEXT: v_readfirstlane_b32 s6, v2
+; GFX942-NEXT: v_readfirstlane_b32 s7, v3
+; GFX942-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
+; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT: buffer_load_ushort v7, v4, s[4:7], 0 offen offset:1026
+; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_cbranch_execnz .LBB21_3
+; GFX942-NEXT: ; %bb.4:
+; GFX942-NEXT: s_mov_b64 exec, s[2:3]
+; GFX942-NEXT: s_mov_b32 s0, 0x5040100
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_perm_b32 v9, v7, v6, s0
; GFX942-NEXT: s_mov_b64 s[2:3], 0
; GFX942-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX942-NEXT: s_movk_i32 s10, 0x7fff
; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX942-NEXT: s_mov_b32 s11, 0x7060302
-; GFX942-NEXT: .LBB21_3: ; %atomicrmw.start
+; GFX942-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Loop Header: Depth=1
-; GFX942-NEXT: ; Child Loop BB21_4 Depth 2
-; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: ; Child Loop BB21_6 Depth 2
; GFX942-NEXT: v_lshlrev_b32_e32 v6, 16, v9
; GFX942-NEXT: v_max_f32_e32 v6, v6, v10
; GFX942-NEXT: v_bfe_u32 v7, v6, 16, 1
@@ -8437,7 +9139,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX942-NEXT: v_cndmask_b32_e32 v7, v8, v11, vcc
; GFX942-NEXT: v_perm_b32 v8, v7, v6, s11
; GFX942-NEXT: v_mov_b64_e32 v[6:7], v[8:9]
-; GFX942-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX942-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX942-NEXT: ; => This Inner Loop Header: Depth=2
; GFX942-NEXT: v_readfirstlane_b32 s4, v0
; GFX942-NEXT: v_readfirstlane_b32 s5, v1
@@ -8451,8 +9153,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 sc0
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB21_4
-; GFX942-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX942-NEXT: s_cbranch_execnz .LBB21_6
+; GFX942-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX942-NEXT: s_mov_b64 exec, s[8:9]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v6, v9
@@ -8460,8 +9162,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX942-NEXT: v_mov_b32_e32 v9, v6
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_cbranch_execnz .LBB21_3
-; GFX942-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX942-NEXT: s_cbranch_execnz .LBB21_5
+; GFX942-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-NEXT: v_mov_b32_e32 v0, v6
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -8480,28 +9182,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-TRUE16-NEXT: ; %bb.2:
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_u16 v6, v4, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_3
+; GFX11-TRUE16-NEXT: ; %bb.4:
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB21_3: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: ; Child Loop BB21_4 Depth 2
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; Child Loop BB21_6 Depth 2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_max_f32 v6, v6, v8 :: v_dual_max_f32 v5, v5, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v6, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
@@ -8516,7 +9235,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.h
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v7
-; GFX11-TRUE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX11-TRUE16-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -8528,8 +9247,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_4
-; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_6
+; GFX11-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
@@ -8539,8 +9258,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_3
-; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_5
+; GFX11-TRUE16-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v5
@@ -8560,28 +9279,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v6, v4, s[0:3], 0 offen offset:1024
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-FAKE16-NEXT: ; %bb.2:
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_3
+; GFX11-FAKE16-NEXT: ; %bb.4:
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB21_3: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: ; Child Loop BB21_4 Depth 2
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; Child Loop BB21_6 Depth 2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_max_f32 v6, v6, v9 :: v_dual_max_f32 v5, v5, v8
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_bfe_u32 v11, v6, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_bfe_u32 v10, v5, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
@@ -8596,7 +9332,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: v_perm_b32 v6, v6, v5, 0x7060302
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v7
-; GFX11-FAKE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX11-FAKE16-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -8608,8 +9344,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_4
-; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_6
+; GFX11-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
@@ -8619,8 +9355,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_3
-; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_5
+; GFX11-FAKE16-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v5
@@ -8640,18 +9376,35 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX10-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v6, v4, s[4:7], 0 offen offset:1024
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB21_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s10, s9
+; GFX10-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_ushort v7, v4, s[4:7], 0 offen offset:1026
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX10-NEXT: s_cbranch_execnz .LBB21_3
+; GFX10-NEXT: ; %bb.4:
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; GFX10-NEXT: .LBB21_3: ; %atomicrmw.start
+; GFX10-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
-; GFX10-NEXT: ; Child Loop BB21_4 Depth 2
-; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: ; Child Loop BB21_6 Depth 2
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX10-NEXT: s_mov_b32 s9, exec_lo
@@ -8672,7 +9425,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX10-NEXT: v_perm_b32 v6, v6, v5, 0x7060302
; GFX10-NEXT: v_mov_b32_e32 v5, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v7
-; GFX10-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX10-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
@@ -8684,8 +9437,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[4:7], 0 offen offset:1024 glc
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
-; GFX10-NEXT: s_cbranch_execnz .LBB21_4
-; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX10-NEXT: s_cbranch_execnz .LBB21_6
+; GFX10-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -8696,8 +9449,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
-; GFX10-NEXT: s_cbranch_execnz .LBB21_3
-; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX10-NEXT: s_cbranch_execnz .LBB21_5
+; GFX10-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -8716,20 +9469,38 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_nop 0
-; GFX90A-NEXT: buffer_load_dword v9, v4, s[8:11], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB21_1
; GFX90A-NEXT: ; %bb.2:
; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
+; GFX90A-NEXT: s_mov_b64 s[6:7], exec
+; GFX90A-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
+; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
+; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
+; GFX90A-NEXT: v_readfirstlane_b32 s11, v3
+; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX90A-NEXT: s_nop 0
+; GFX90A-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1026
+; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_cbranch_execnz .LBB21_3
+; GFX90A-NEXT: ; %bb.4:
+; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
+; GFX90A-NEXT: s_mov_b32 s4, 0x5040100
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_perm_b32 v9, v7, v6, s4
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX90A-NEXT: s_movk_i32 s14, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX90A-NEXT: s_mov_b32 s15, 0x7060302
-; GFX90A-NEXT: .LBB21_3: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
-; GFX90A-NEXT: ; Child Loop BB21_4 Depth 2
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; Child Loop BB21_6 Depth 2
; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v9
; GFX90A-NEXT: v_max_f32_e32 v6, v6, v10
; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
@@ -8747,7 +9518,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX90A-NEXT: v_perm_b32 v8, v7, v6, s15
; GFX90A-NEXT: s_mov_b64 s[12:13], exec
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[8:9], v[8:9] op_sel:[0,1]
-; GFX90A-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX90A-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
@@ -8760,8 +9531,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB21_4
-; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX90A-NEXT: s_cbranch_execnz .LBB21_6
+; GFX90A-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v6, v9
@@ -8769,8 +9540,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX90A-NEXT: v_mov_b32_e32 v9, v6
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB21_3
-; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX90A-NEXT: s_cbranch_execnz .LBB21_5
+; GFX90A-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v6
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -8789,20 +9560,38 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB21_1
; GFX908-NEXT: ; %bb.2:
; GFX908-NEXT: s_mov_b64 exec, s[6:7]
+; GFX908-NEXT: s_mov_b64 s[6:7], exec
+; GFX908-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: v_readfirstlane_b32 s8, v0
+; GFX908-NEXT: v_readfirstlane_b32 s9, v1
+; GFX908-NEXT: v_readfirstlane_b32 s10, v2
+; GFX908-NEXT: v_readfirstlane_b32 s11, v3
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX908-NEXT: s_nop 0
+; GFX908-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1026
+; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_cbranch_execnz .LBB21_3
+; GFX908-NEXT: ; %bb.4:
+; GFX908-NEXT: s_mov_b64 exec, s[6:7]
+; GFX908-NEXT: s_mov_b32 s4, 0x5040100
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_perm_b32 v7, v7, v6, s4
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX908-NEXT: s_movk_i32 s14, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX908-NEXT: s_mov_b32 s15, 0x7060302
-; GFX908-NEXT: .LBB21_3: ; %atomicrmw.start
+; GFX908-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Loop Header: Depth=1
-; GFX908-NEXT: ; Child Loop BB21_4 Depth 2
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: ; Child Loop BB21_6 Depth 2
; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX908-NEXT: v_max_f32_e32 v5, v5, v8
; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
@@ -8821,7 +9610,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX908-NEXT: v_mov_b32_e32 v5, v6
; GFX908-NEXT: s_mov_b64 s[12:13], exec
; GFX908-NEXT: v_mov_b32_e32 v6, v7
-; GFX908-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX908-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
; GFX908-NEXT: v_readfirstlane_b32 s9, v1
@@ -8834,8 +9623,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB21_4
-; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX908-NEXT: s_cbranch_execnz .LBB21_6
+; GFX908-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
@@ -8843,8 +9632,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB21_3
-; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX908-NEXT: s_cbranch_execnz .LBB21_5
+; GFX908-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
@@ -8863,18 +9652,36 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_nop 0
-; GFX8-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
+; GFX8-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB21_1
; GFX8-NEXT: ; %bb.2:
; GFX8-NEXT: s_mov_b64 exec, s[6:7]
+; GFX8-NEXT: s_mov_b64 s[6:7], exec
+; GFX8-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_readfirstlane_b32 s8, v0
+; GFX8-NEXT: v_readfirstlane_b32 s9, v1
+; GFX8-NEXT: v_readfirstlane_b32 s10, v2
+; GFX8-NEXT: v_readfirstlane_b32 s11, v3
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX8-NEXT: s_nop 0
+; GFX8-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1026
+; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB21_3
+; GFX8-NEXT: ; %bb.4:
+; GFX8-NEXT: s_mov_b64 exec, s[6:7]
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX8-NEXT: v_or_b32_e32 v7, v6, v7
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; GFX8-NEXT: .LBB21_3: ; %atomicrmw.start
+; GFX8-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Loop Header: Depth=1
-; GFX8-NEXT: ; Child Loop BB21_4 Depth 2
-; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: ; Child Loop BB21_6 Depth 2
; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX8-NEXT: v_max_f32_e32 v5, v5, v8
; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
@@ -8896,7 +9703,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX8-NEXT: v_mov_b32_e32 v5, v6
; GFX8-NEXT: s_mov_b64 s[12:13], exec
; GFX8-NEXT: v_mov_b32_e32 v6, v7
-; GFX8-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX8-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
; GFX8-NEXT: v_readfirstlane_b32 s9, v1
@@ -8909,8 +9716,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB21_4
-; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX8-NEXT: s_cbranch_execnz .LBB21_6
+; GFX8-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
@@ -8918,8 +9725,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX8-NEXT: v_mov_b32_e32 v7, v5
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB21_3
-; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX8-NEXT: s_cbranch_execnz .LBB21_5
+; GFX8-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -8939,22 +9746,37 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1024
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB21_1
; GFX7-NEXT: ; %bb.2:
; GFX7-NEXT: s_mov_b64 exec, s[6:7]
+; GFX7-NEXT: s_mov_b64 s[6:7], exec
+; GFX7-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: v_readfirstlane_b32 s8, v0
+; GFX7-NEXT: v_readfirstlane_b32 s9, v1
+; GFX7-NEXT: v_readfirstlane_b32 s10, v2
+; GFX7-NEXT: v_readfirstlane_b32 s11, v3
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX7-NEXT: buffer_load_ushort v8, v4, s[8:11], 0 offen offset:1026
+; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_cbranch_execnz .LBB21_3
+; GFX7-NEXT: ; %bb.4:
+; GFX7-NEXT: s_mov_b64 exec, s[6:7]
; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
+; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
-; GFX7-NEXT: .LBB21_3: ; %atomicrmw.start
+; GFX7-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Loop Header: Depth=1
-; GFX7-NEXT: ; Child Loop BB21_4 Depth 2
+; GFX7-NEXT: ; Child Loop BB21_6 Depth 2
; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v8
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
@@ -8968,7 +9790,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX7-NEXT: v_mov_b32_e32 v8, v6
; GFX7-NEXT: s_mov_b64 s[12:13], exec
; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX7-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX7-NEXT: ; => This Inner Loop Header: Depth=2
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
; GFX7-NEXT: v_readfirstlane_b32 s9, v1
@@ -8981,8 +9803,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v4, s[8:11], 0 offen offset:1024 glc
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB21_4
-; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX7-NEXT: s_cbranch_execnz .LBB21_6
+; GFX7-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
@@ -8991,8 +9813,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX7-NEXT: s_cbranch_execnz .LBB21_3
-; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX7-NEXT: s_cbranch_execnz .LBB21_5
+; GFX7-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v8
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
@@ -9016,24 +9838,40 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: buffer_load_dword v9, v4, s[8:11], 0 offen offset:1024
-; GFX6-NEXT: ; implicit-def: $vgpr4
+; GFX6-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1024
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB21_1
; GFX6-NEXT: ; %bb.2:
; GFX6-NEXT: s_mov_b64 exec, s[6:7]
+; GFX6-NEXT: s_mov_b64 s[6:7], exec
+; GFX6-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: v_readfirstlane_b32 s8, v0
+; GFX6-NEXT: v_readfirstlane_b32 s9, v1
+; GFX6-NEXT: v_readfirstlane_b32 s10, v2
+; GFX6-NEXT: v_readfirstlane_b32 s11, v3
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX6-NEXT: buffer_load_ushort v9, v4, s[8:11], 0 offen offset:1026
+; GFX6-NEXT: ; implicit-def: $vgpr4
+; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX6-NEXT: s_cbranch_execnz .LBB21_3
+; GFX6-NEXT: ; %bb.4:
+; GFX6-NEXT: s_mov_b64 exec, s[6:7]
; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v6
; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v9
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v9
+; GFX6-NEXT: v_lshlrev_b32_e32 v11, 16, v9
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v7
; GFX6-NEXT: s_mov_b64 s[6:7], 0
; GFX6-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
; GFX6-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
-; GFX6-NEXT: .LBB21_3: ; %atomicrmw.start
+; GFX6-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Loop Header: Depth=1
-; GFX6-NEXT: ; Child Loop BB21_4 Depth 2
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v7
+; GFX6-NEXT: ; Child Loop BB21_6 Depth 2
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v11
+; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
; GFX6-NEXT: v_mul_f32_e32 v7, 1.0, v4
; GFX6-NEXT: v_max_f32_e32 v6, v6, v9
@@ -9046,7 +9884,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX6-NEXT: v_mov_b32_e32 v7, v5
; GFX6-NEXT: s_mov_b64 s[12:13], exec
; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX6-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX6-NEXT: ; => This Inner Loop Header: Depth=2
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
; GFX6-NEXT: v_readfirstlane_b32 s9, v1
@@ -9059,24 +9897,24 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v8, s[8:11], 0 offen glc
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_cbranch_execnz .LBB21_4
-; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX6-NEXT: s_cbranch_execnz .LBB21_6
+; GFX6-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v6
+; GFX6-NEXT: v_and_b32_e32 v11, 0xffff0000, v6
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX6-NEXT: s_cbranch_execnz .LBB21_3
-; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX6-NEXT: s_cbranch_execnz .LBB21_5
+; GFX6-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v7
+; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v11
; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v4
; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 256
%result = atomicrmw fmax ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index be032802e75d1..d0d8327947e0f 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -6004,53 +6004,99 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; --------------------------------------------------------------------
define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
-; GFX12-NEXT: v_mov_b32_e32 v3, s16
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-NEXT: v_pk_max_num_f16 v2, v1, v1
-; GFX12-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen offset:1024
-; GFX12-NEXT: .LBB16_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v5, v0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v0, v5, v5
-; GFX12-NEXT: v_pk_min_num_f16 v4, v0, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB16_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-TRUE16-NEXT: s_clause 0x1
+; GFX12-TRUE16-NEXT: buffer_load_u16 v3, v0, s[0:3], null offen offset:1026
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v2, v1, v1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, s16
+; GFX12-TRUE16-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v0, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_pk_min_num_f16 v4, v0, v2
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB16_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-FAKE16-NEXT: s_clause 0x1
+; GFX12-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1024
+; GFX12-FAKE16-NEXT: buffer_load_u16 v3, v2, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v2, v0, v0
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v0, v3, 16, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, s16
+; GFX12-FAKE16-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, v0
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v0, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_pk_min_num_f16 v4, v0, v2
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB16_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-NEXT: v_mov_b32_e32 v0, s16
-; GFX942-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
+; GFX942-NEXT: buffer_load_ushort v3, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_pk_max_f16 v2, v0, v0
; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: v_pk_max_f16 v2, v1, v1
+; GFX942-NEXT: s_waitcnt vmcnt(1)
+; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_lshl_or_b32 v0, v4, 16, v0
; GFX942-NEXT: v_mov_b32_e32 v3, s16
; GFX942-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v5, v0
; GFX942-NEXT: v_pk_max_f16 v0, v5, v5
; GFX942-NEXT: buffer_wbl2 sc1
@@ -6069,50 +6115,94 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
-; GFX11-NEXT: v_mov_b32_e32 v3, s16
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_pk_max_f16 v2, v1, v1
-; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-NEXT: .LBB16_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v5, v0
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v0, v5, v5
-; GFX11-NEXT: v_pk_min_f16 v4, v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB16_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: buffer_load_u16 v3, v0, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v2, v1, v1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s16
+; GFX11-TRUE16-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v0, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_pk_min_f16 v4, v0, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB16_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v3, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v2, v0, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v3, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s16
+; GFX11-FAKE16-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v0
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v0, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_pk_min_f16 v4, v0, v2
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB16_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, s20
-; GFX10-NEXT: v_mov_b32_e32 v3, s20
+; GFX10-NEXT: v_mov_b32_e32 v1, s20
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: v_pk_max_f16 v2, v1, v1
-; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: v_pk_max_f16 v2, v0, v0
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v1
; GFX10-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v5, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_pk_max_f16 v0, v5, v5
@@ -6134,15 +6224,19 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: v_mov_b32_e32 v0, s20
-; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
+; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_pk_max_f16 v2, v0, v0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_pk_max_f16 v2, v1, v1
+; GFX90A-NEXT: s_waitcnt vmcnt(1)
+; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshl_or_b32 v0, v4, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: v_pk_max_f16 v0, v5, v5
; GFX90A-NEXT: v_pk_min_f16 v4, v0, v2
@@ -6161,15 +6255,19 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, v0
-; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_pk_max_f16 v2, v0, v0
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_pk_max_f16 v2, v1, v1
+; GFX908-NEXT: s_waitcnt vmcnt(1)
+; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_lshl_or_b32 v0, v4, 16, v0
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v5, v0
; GFX908-NEXT: v_pk_max_f16 v0, v5, v5
; GFX908-NEXT: v_pk_min_f16 v4, v0, v2
@@ -6189,16 +6287,20 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s20
-; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_mov_b32_e32 v1, s20
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_max_f16_sdwa v2, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v3, v0, v0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_max_f16_sdwa v2, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v3, v1, v1
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
; GFX8-NEXT: v_max_f16_sdwa v0, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v1, v6, v6
@@ -6222,16 +6324,17 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_mov_b32_e32 v4, s20
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -6263,18 +6366,18 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, s20
+; GFX6-NEXT: buffer_load_ushort v1, v2, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_mov_b32_e32 v4, s6
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v1
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -6309,50 +6412,99 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
}
define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v1, s16
-; GFX12-NEXT: v_pk_max_num_f16 v2, v0, v0
-; GFX12-NEXT: v_mov_b32_e32 v3, s16
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: buffer_load_b32 v1, v1, s[0:3], null offen offset:1024
-; GFX12-NEXT: .LBB17_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v0, v1, v1
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_pk_min_num_f16 v0, v0, v2
-; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX12-NEXT: v_mov_b32_e32 v1, v4
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB17_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, s16
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v2, v0, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-TRUE16-NEXT: s_clause 0x1
+; GFX12-TRUE16-NEXT: buffer_load_u16 v3, v1, s[0:3], null offen offset:1026
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, s16
+; GFX12-TRUE16-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v0, v1, v1
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: v_pk_min_num_f16 v0, v0, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB17_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-FAKE16-NEXT: s_clause 0x1
+; GFX12-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1024
+; GFX12-FAKE16-NEXT: buffer_load_u16 v3, v2, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v2, v0, v0
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, s16
+; GFX12-FAKE16-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v0, v1, v1
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: v_pk_min_num_f16 v0, v0, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB17_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v1, s16
-; GFX942-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
+; GFX942-NEXT: buffer_load_ushort v3, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
; GFX942-NEXT: v_pk_max_f16 v2, v0, v0
+; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: s_waitcnt vmcnt(1)
+; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_lshl_or_b32 v1, v4, 16, v0
; GFX942-NEXT: v_mov_b32_e32 v3, s16
; GFX942-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_pk_max_f16 v0, v1, v1
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -6371,47 +6523,94 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s16
-; GFX11-NEXT: v_pk_max_f16 v2, v0, v0
-; GFX11-NEXT: v_mov_b32_e32 v3, s16
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen offset:1024
-; GFX11-NEXT: .LBB17_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_pk_max_f16 v0, v1, v1
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_min_f16 v0, v0, v2
-; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX11-NEXT: v_mov_b32_e32 v1, v4
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB17_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s16
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v2, v0, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: buffer_load_u16 v3, v1, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s16
+; GFX11-TRUE16-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: v_pk_min_f16 v0, v0, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB17_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v3, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v2, v0, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s16
+; GFX11-FAKE16-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: v_pk_min_f16 v0, v0, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB17_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s20
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
+; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
; GFX10-NEXT: v_pk_max_f16 v2, v0, v0
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
; GFX10-NEXT: v_mov_b32_e32 v3, s20
-; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshl_or_b32 v1, v4, 16, v1
; GFX10-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_pk_max_f16 v0, v1, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_pk_min_f16 v0, v0, v2
@@ -6434,13 +6633,18 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
+; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
; GFX90A-NEXT: v_pk_max_f16 v2, v0, v0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_waitcnt vmcnt(1)
+; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshl_or_b32 v1, v4, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v0, v1, v1
; GFX90A-NEXT: v_pk_min_f16 v0, v0, v2
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
@@ -6460,13 +6664,18 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
; GFX908-NEXT: v_pk_max_f16 v2, v0, v0
+; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: s_waitcnt vmcnt(1)
+; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_lshl_or_b32 v1, v4, 16, v0
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_pk_max_f16 v0, v1, v1
; GFX908-NEXT: v_pk_min_f16 v0, v0, v2
; GFX908-NEXT: v_mov_b32_e32 v5, v1
@@ -6487,14 +6696,19 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s20
-; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
; GFX8-NEXT: v_max_f16_sdwa v2, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v3, v0, v0
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f16_sdwa v0, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v5, v1, v1
; GFX8-NEXT: v_min_f16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -6518,16 +6732,17 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_mov_b32_e32 v4, s20
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -6556,18 +6771,18 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, s20
+; GFX6-NEXT: buffer_load_ushort v1, v2, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_mov_b32_e32 v4, s6
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v1
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -6599,74 +6814,185 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
}
define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, <2 x half> %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, exec_lo
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s5, s4
-; GFX12-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_readfirstlane_b32 s0, v0
-; GFX12-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-NEXT: v_readfirstlane_b32 s2, v2
-; GFX12-NEXT: v_readfirstlane_b32 s3, v3
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_load_b32 v7, v4, s[0:3], null offen offset:1024
-; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB18_1
-; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s4
-; GFX12-NEXT: v_pk_max_num_f16 v8, v5, v5
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB18_3: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Loop Header: Depth=1
-; GFX12-NEXT: ; Child Loop BB18_4 Depth 2
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v5, v7, v7
-; GFX12-NEXT: s_mov_b32 s5, exec_lo
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s6, s5
-; GFX12-NEXT: v_pk_min_num_f16 v6, v5, v8
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_mov_b32_e32 v5, v6
-; GFX12-NEXT: v_mov_b32_e32 v6, v7
-; GFX12-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
-; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-NEXT: v_readfirstlane_b32 s0, v0
-; GFX12-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-NEXT: v_readfirstlane_b32 s2, v2
-; GFX12-NEXT: v_readfirstlane_b32 s3, v3
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX12-NEXT: s_cbranch_execnz .LBB18_4
-; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX12-NEXT: s_mov_b32 exec_lo, s5
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX12-NEXT: v_mov_b32_e32 v7, v5
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB18_3
-; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: v_mov_b32_e32 v0, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
+; GFX12-TRUE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: buffer_load_u16 v7, v4, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_1
+; GFX12-TRUE16-NEXT: ; %bb.2:
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
+; GFX12-TRUE16-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: buffer_load_u16 v6, v4, s[0:3], null offen offset:1026
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_3
+; GFX12-TRUE16-NEXT: ; %bb.4:
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v8, v5, v5
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-TRUE16-NEXT: .LBB18_5: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: ; Child Loop BB18_6 Depth 2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v5, v7, v7
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX12-TRUE16-NEXT: v_pk_min_num_f16 v6, v5, v8
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v7
+; GFX12-TRUE16-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
+; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_6
+; GFX12-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v5
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_5
+; GFX12-TRUE16-NEXT: ; %bb.8: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
+; GFX12-FAKE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: buffer_load_u16 v6, v4, s[0:3], null offen offset:1024
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_1
+; GFX12-FAKE16-NEXT: ; %bb.2:
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
+; GFX12-FAKE16-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: buffer_load_u16 v7, v4, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_3
+; GFX12-FAKE16-NEXT: ; %bb.4:
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v8, v5, v5
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-FAKE16-NEXT: .LBB18_5: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: ; Child Loop BB18_6 Depth 2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v5, v7, v7
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX12-FAKE16-NEXT: v_pk_min_num_f16 v6, v5, v8
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, v6
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v7
+; GFX12-FAKE16-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
+; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_6
+; GFX12-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v7, v5
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_5
+; GFX12-FAKE16-NEXT: ; %bb.8: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -6682,24 +7008,42 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX942-NEXT: buffer_load_dword v9, v4, s[4:7], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v6, v4, s[4:7], 0 offen offset:1024
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB18_1
; GFX942-NEXT: ; %bb.2:
; GFX942-NEXT: s_mov_b64 exec, s[2:3]
+; GFX942-NEXT: s_mov_b64 s[2:3], exec
+; GFX942-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
+; GFX942-NEXT: v_readfirstlane_b32 s4, v0
+; GFX942-NEXT: v_readfirstlane_b32 s5, v1
+; GFX942-NEXT: v_readfirstlane_b32 s6, v2
+; GFX942-NEXT: v_readfirstlane_b32 s7, v3
+; GFX942-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
+; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT: buffer_load_ushort v7, v4, s[4:7], 0 offen offset:1026
+; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_cbranch_execnz .LBB18_3
+; GFX942-NEXT: ; %bb.4:
+; GFX942-NEXT: s_mov_b64 exec, s[2:3]
+; GFX942-NEXT: s_mov_b32 s0, 0x5040100
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_perm_b32 v9, v7, v6, s0
; GFX942-NEXT: s_mov_b64 s[2:3], 0
; GFX942-NEXT: v_pk_max_f16 v5, v5, v5
-; GFX942-NEXT: .LBB18_3: ; %atomicrmw.start
+; GFX942-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Loop Header: Depth=1
-; GFX942-NEXT: ; Child Loop BB18_4 Depth 2
-; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: ; Child Loop BB18_6 Depth 2
; GFX942-NEXT: v_pk_max_f16 v6, v9, v9
; GFX942-NEXT: s_mov_b64 s[8:9], exec
; GFX942-NEXT: v_pk_min_f16 v8, v6, v5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b64_e32 v[6:7], v[8:9]
-; GFX942-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
+; GFX942-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX942-NEXT: ; => This Inner Loop Header: Depth=2
; GFX942-NEXT: v_readfirstlane_b32 s4, v0
; GFX942-NEXT: v_readfirstlane_b32 s5, v1
@@ -6713,8 +7057,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 sc0
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB18_4
-; GFX942-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
+; GFX942-NEXT: s_cbranch_execnz .LBB18_6
+; GFX942-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX942-NEXT: s_mov_b64 exec, s[8:9]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v6, v9
@@ -6722,73 +7066,171 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX942-NEXT: v_mov_b32_e32 v9, v6
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_cbranch_execnz .LBB18_3
-; GFX942-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX942-NEXT: s_cbranch_execnz .LBB18_5
+; GFX942-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-NEXT: v_mov_b32_e32 v0, v6
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_mov_b32 s6, s5
-; GFX11-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-NEXT: v_readfirstlane_b32 s2, v2
-; GFX11-NEXT: v_readfirstlane_b32 s3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
-; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX11-NEXT: s_cbranch_execnz .LBB18_1
-; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-NEXT: v_pk_max_f16 v8, v5, v5
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB18_3: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Loop Header: Depth=1
-; GFX11-NEXT: ; Child Loop BB18_4 Depth 2
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_pk_max_f16 v5, v7, v7
-; GFX11-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_mov_b32 s6, s5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_min_f16 v6, v5, v8
-; GFX11-NEXT: v_mov_b32_e32 v5, v6
-; GFX11-NEXT: v_mov_b32_e32 v6, v7
-; GFX11-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
-; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-NEXT: v_readfirstlane_b32 s2, v2
-; GFX11-NEXT: v_readfirstlane_b32 s3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
-; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX11-NEXT: s_cbranch_execnz .LBB18_4
-; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX11-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX11-NEXT: v_mov_b32_e32 v7, v5
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB18_3
-; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: v_mov_b32_e32 v0, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_1
+; GFX11-TRUE16-NEXT: ; %bb.2:
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_u16 v6, v4, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_3
+; GFX11-TRUE16-NEXT: ; %bb.4:
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v8, v5, v5
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB18_5: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: ; Child Loop BB18_6 Depth 2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v5, v7, v7
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: v_pk_min_f16 v6, v5, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v7
+; GFX11-TRUE16-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
+; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_6
+; GFX11-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v5
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_5
+; GFX11-TRUE16-NEXT: ; %bb.8: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_u16 v6, v4, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_1
+; GFX11-FAKE16-NEXT: ; %bb.2:
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_3
+; GFX11-FAKE16-NEXT: ; %bb.4:
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v8, v5, v5
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB18_5: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: ; Child Loop BB18_6 Depth 2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v5, v7, v7
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: v_pk_min_f16 v6, v5, v8
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v6
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v7
+; GFX11-FAKE16-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
+; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_6
+; GFX11-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v5
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_5
+; GFX11-FAKE16-NEXT: ; %bb.8: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -6804,17 +7246,34 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX10-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v6, v4, s[4:7], 0 offen offset:1024
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB18_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s10, s9
+; GFX10-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_ushort v7, v4, s[4:7], 0 offen offset:1026
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX10-NEXT: s_cbranch_execnz .LBB18_3
+; GFX10-NEXT: ; %bb.4:
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
; GFX10-NEXT: v_pk_max_f16 v8, v5, v5
-; GFX10-NEXT: .LBB18_3: ; %atomicrmw.start
+; GFX10-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
-; GFX10-NEXT: ; Child Loop BB18_4 Depth 2
-; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: ; Child Loop BB18_6 Depth 2
; GFX10-NEXT: v_pk_max_f16 v5, v7, v7
; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -6822,7 +7281,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX10-NEXT: v_pk_min_f16 v6, v5, v8
; GFX10-NEXT: v_mov_b32_e32 v5, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v7
-; GFX10-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
+; GFX10-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
@@ -6834,8 +7293,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[4:7], 0 offen offset:1024 glc
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
-; GFX10-NEXT: s_cbranch_execnz .LBB18_4
-; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
+; GFX10-NEXT: s_cbranch_execnz .LBB18_6
+; GFX10-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -6846,8 +7305,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
-; GFX10-NEXT: s_cbranch_execnz .LBB18_3
-; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX10-NEXT: s_cbranch_execnz .LBB18_5
+; GFX10-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -6866,22 +7325,40 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_nop 0
-; GFX90A-NEXT: buffer_load_dword v9, v4, s[8:11], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB18_1
; GFX90A-NEXT: ; %bb.2:
; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
+; GFX90A-NEXT: s_mov_b64 s[6:7], exec
+; GFX90A-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
+; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
+; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
+; GFX90A-NEXT: v_readfirstlane_b32 s11, v3
+; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX90A-NEXT: s_nop 0
+; GFX90A-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1026
+; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_cbranch_execnz .LBB18_3
+; GFX90A-NEXT: ; %bb.4:
+; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
+; GFX90A-NEXT: s_mov_b32 s4, 0x5040100
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_perm_b32 v9, v7, v6, s4
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_pk_max_f16 v5, v5, v5
-; GFX90A-NEXT: .LBB18_3: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
-; GFX90A-NEXT: ; Child Loop BB18_4 Depth 2
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; Child Loop BB18_6 Depth 2
; GFX90A-NEXT: v_pk_max_f16 v6, v9, v9
; GFX90A-NEXT: v_pk_min_f16 v8, v6, v5
; GFX90A-NEXT: s_mov_b64 s[12:13], exec
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[8:9], v[8:9] op_sel:[0,1]
-; GFX90A-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
+; GFX90A-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
@@ -6894,8 +7371,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB18_4
-; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
+; GFX90A-NEXT: s_cbranch_execnz .LBB18_6
+; GFX90A-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v6, v9
@@ -6903,8 +7380,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX90A-NEXT: v_mov_b32_e32 v9, v6
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB18_3
-; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX90A-NEXT: s_cbranch_execnz .LBB18_5
+; GFX90A-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v6
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -6923,23 +7400,41 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB18_1
; GFX908-NEXT: ; %bb.2:
; GFX908-NEXT: s_mov_b64 exec, s[6:7]
+; GFX908-NEXT: s_mov_b64 s[6:7], exec
+; GFX908-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: v_readfirstlane_b32 s8, v0
+; GFX908-NEXT: v_readfirstlane_b32 s9, v1
+; GFX908-NEXT: v_readfirstlane_b32 s10, v2
+; GFX908-NEXT: v_readfirstlane_b32 s11, v3
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX908-NEXT: s_nop 0
+; GFX908-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1026
+; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_cbranch_execnz .LBB18_3
+; GFX908-NEXT: ; %bb.4:
+; GFX908-NEXT: s_mov_b64 exec, s[6:7]
+; GFX908-NEXT: s_mov_b32 s4, 0x5040100
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_perm_b32 v7, v7, v6, s4
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: v_pk_max_f16 v8, v5, v5
-; GFX908-NEXT: .LBB18_3: ; %atomicrmw.start
+; GFX908-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Loop Header: Depth=1
-; GFX908-NEXT: ; Child Loop BB18_4 Depth 2
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: ; Child Loop BB18_6 Depth 2
; GFX908-NEXT: v_pk_max_f16 v5, v7, v7
; GFX908-NEXT: v_pk_min_f16 v6, v5, v8
; GFX908-NEXT: v_mov_b32_e32 v5, v6
; GFX908-NEXT: s_mov_b64 s[12:13], exec
; GFX908-NEXT: v_mov_b32_e32 v6, v7
-; GFX908-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
+; GFX908-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
; GFX908-NEXT: v_readfirstlane_b32 s9, v1
@@ -6952,8 +7447,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB18_4
-; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
+; GFX908-NEXT: s_cbranch_execnz .LBB18_6
+; GFX908-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
@@ -6961,8 +7456,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB18_3
-; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX908-NEXT: s_cbranch_execnz .LBB18_5
+; GFX908-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
@@ -6981,18 +7476,36 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_nop 0
-; GFX8-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
+; GFX8-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB18_1
; GFX8-NEXT: ; %bb.2:
; GFX8-NEXT: s_mov_b64 exec, s[6:7]
+; GFX8-NEXT: s_mov_b64 s[6:7], exec
+; GFX8-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_readfirstlane_b32 s8, v0
+; GFX8-NEXT: v_readfirstlane_b32 s9, v1
+; GFX8-NEXT: v_readfirstlane_b32 s10, v2
+; GFX8-NEXT: v_readfirstlane_b32 s11, v3
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX8-NEXT: s_nop 0
+; GFX8-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1026
+; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB18_3
+; GFX8-NEXT: ; %bb.4:
+; GFX8-NEXT: s_mov_b64 exec, s[6:7]
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX8-NEXT: v_or_b32_e32 v7, v6, v7
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_max_f16_sdwa v8, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v9, v5, v5
-; GFX8-NEXT: .LBB18_3: ; %atomicrmw.start
+; GFX8-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Loop Header: Depth=1
-; GFX8-NEXT: ; Child Loop BB18_4 Depth 2
-; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: ; Child Loop BB18_6 Depth 2
; GFX8-NEXT: v_max_f16_sdwa v5, v7, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v6, v7, v7
; GFX8-NEXT: v_min_f16_sdwa v5, v5, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -7001,7 +7514,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX8-NEXT: v_mov_b32_e32 v5, v6
; GFX8-NEXT: s_mov_b64 s[12:13], exec
; GFX8-NEXT: v_mov_b32_e32 v6, v7
-; GFX8-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
+; GFX8-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
; GFX8-NEXT: v_readfirstlane_b32 s9, v1
@@ -7014,8 +7527,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB18_4
-; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
+; GFX8-NEXT: s_cbranch_execnz .LBB18_6
+; GFX8-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
@@ -7023,8 +7536,8 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX8-NEXT: v_mov_b32_e32 v7, v5
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB18_3
-; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX8-NEXT: s_cbranch_execnz .LBB18_5
+; GFX8-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -7043,34 +7556,48 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: buffer_load_dword v6, v4, s[8:11], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB18_1
; GFX7-NEXT: ; %bb.2:
; GFX7-NEXT: s_mov_b64 exec, s[6:7]
+; GFX7-NEXT: s_mov_b64 s[6:7], exec
+; GFX7-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: v_readfirstlane_b32 s8, v0
+; GFX7-NEXT: v_readfirstlane_b32 s9, v1
+; GFX7-NEXT: v_readfirstlane_b32 s10, v2
+; GFX7-NEXT: v_readfirstlane_b32 s11, v3
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX7-NEXT: buffer_load_ushort v10, v4, s[8:11], 0 offen offset:1026
+; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_cbranch_execnz .LBB18_3
+; GFX7-NEXT: ; %bb.4:
+; GFX7-NEXT: s_mov_b64 exec, s[6:7]
; GFX7-NEXT: v_cvt_f32_f16_e32 v9, v7
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v6
; GFX7-NEXT: s_mov_b64 s[6:7], 0
-; GFX7-NEXT: .LBB18_3: ; %atomicrmw.start
+; GFX7-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Loop Header: Depth=1
-; GFX7-NEXT: ; Child Loop BB18_4 Depth 2
-; GFX7-NEXT: v_cvt_f32_f16_e32 v8, v7
-; GFX7-NEXT: v_cvt_f32_f16_e32 v10, v6
+; GFX7-NEXT: ; Child Loop BB18_6 Depth 2
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v10
+; GFX7-NEXT: v_cvt_f32_f16_e32 v8, v6
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX7-NEXT: v_min_f32_e32 v8, v8, v9
-; GFX7-NEXT: v_min_f32_e32 v10, v10, v5
-; GFX7-NEXT: v_cvt_f16_f32_e32 v11, v8
-; GFX7-NEXT: v_cvt_f16_f32_e32 v10, v10
-; GFX7-NEXT: v_or_b32_e32 v8, v6, v7
; GFX7-NEXT: s_mov_b64 s[12:13], exec
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v11
-; GFX7-NEXT: v_or_b32_e32 v7, v10, v6
+; GFX7-NEXT: v_min_f32_e32 v7, v7, v9
+; GFX7-NEXT: v_min_f32_e32 v8, v8, v5
+; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v7
+; GFX7-NEXT: v_cvt_f16_f32_e32 v11, v8
+; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v10
+; GFX7-NEXT: v_or_b32_e32 v8, v6, v8
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; GFX7-NEXT: v_or_b32_e32 v7, v11, v6
; GFX7-NEXT: v_mov_b32_e32 v6, v7
; GFX7-NEXT: v_mov_b32_e32 v7, v8
-; GFX7-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
+; GFX7-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX7-NEXT: ; => This Inner Loop Header: Depth=2
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
; GFX7-NEXT: v_readfirstlane_b32 s9, v1
@@ -7083,20 +7610,20 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB18_4
-; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
+; GFX7-NEXT: s_cbranch_execnz .LBB18_6
+; GFX7-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
-; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v10, 16, v6
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX7-NEXT: s_cbranch_execnz .LBB18_3
-; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX7-NEXT: s_cbranch_execnz .LBB18_5
+; GFX7-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v7
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v10
; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
@@ -7115,36 +7642,50 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: buffer_load_dword v6, v4, s[8:11], 0 offen offset:1024
-; GFX6-NEXT: ; implicit-def: $vgpr4
+; GFX6-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB18_1
; GFX6-NEXT: ; %bb.2:
; GFX6-NEXT: s_mov_b64 exec, s[6:7]
+; GFX6-NEXT: s_mov_b64 s[6:7], exec
+; GFX6-NEXT: .LBB18_3: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: v_readfirstlane_b32 s8, v0
+; GFX6-NEXT: v_readfirstlane_b32 s9, v1
+; GFX6-NEXT: v_readfirstlane_b32 s10, v2
+; GFX6-NEXT: v_readfirstlane_b32 s11, v3
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX6-NEXT: buffer_load_ushort v11, v4, s[8:11], 0 offen offset:1026
+; GFX6-NEXT: ; implicit-def: $vgpr4
+; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX6-NEXT: s_cbranch_execnz .LBB18_3
+; GFX6-NEXT: ; %bb.4:
+; GFX6-NEXT: s_mov_b64 exec, s[6:7]
; GFX6-NEXT: v_cvt_f32_f16_e32 v9, v7
; GFX6-NEXT: v_cvt_f32_f16_e32 v10, v5
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
; GFX6-NEXT: s_mov_b64 s[6:7], 0
-; GFX6-NEXT: .LBB18_3: ; %atomicrmw.start
+; GFX6-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Loop Header: Depth=1
-; GFX6-NEXT: ; Child Loop BB18_4 Depth 2
-; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v4
+; GFX6-NEXT: ; Child Loop BB18_6 Depth 2
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v11
+; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v6
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v6
; GFX6-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_min_f32_e32 v5, v5, v9
-; GFX6-NEXT: v_min_f32_e32 v7, v7, v10
-; GFX6-NEXT: v_cvt_f16_f32_e32 v11, v5
-; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v7
-; GFX6-NEXT: v_or_b32_e32 v5, v6, v4
; GFX6-NEXT: s_mov_b64 s[12:13], exec
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v11
+; GFX6-NEXT: v_min_f32_e32 v4, v4, v9
+; GFX6-NEXT: v_min_f32_e32 v5, v5, v10
+; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v5
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v11
+; GFX6-NEXT: v_or_b32_e32 v5, v6, v5
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX6-NEXT: v_or_b32_e32 v4, v7, v4
; GFX6-NEXT: v_mov_b32_e32 v7, v5
; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
+; GFX6-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX6-NEXT: ; => This Inner Loop Header: Depth=2
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
; GFX6-NEXT: v_readfirstlane_b32 s9, v1
@@ -7157,20 +7698,20 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v8, s[8:11], 0 offen glc
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_cbranch_execnz .LBB18_4
-; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
+; GFX6-NEXT: s_cbranch_execnz .LBB18_6
+; GFX6-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v11, 16, v6
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX6-NEXT: s_cbranch_execnz .LBB18_3
-; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX6-NEXT: s_cbranch_execnz .LBB18_5
+; GFX6-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v6
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v11
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
@@ -7193,37 +7734,41 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_lshlrev_b32 v3, 16, v1
-; GFX12-TRUE16-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: s_clause 0x1
+; GFX12-TRUE16-NEXT: buffer_load_u16 v4, v0, s[0:3], null offen offset:1026
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], null offen offset:1024
; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX12-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v0
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v1, v1, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v0, v0, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -7244,39 +7789,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX12-FAKE16-NEXT: s_clause 0x1
+; GFX12-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1024
+; GFX12-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_and_b32 v3, 0xffff0000, v1
-; GFX12-FAKE16-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen offset:1024
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX12-FAKE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v0
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v1, v1, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v1, 16, 1
; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_dual_cndmask_b32 v1, v7, v9 :: v_dual_lshlrev_b32 v0, 16, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v0, v0, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v0, 16, 1
; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v0, v5, v8, s4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_perm_b32 v5, v1, v0, 0x7060302
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
@@ -7293,18 +7844,22 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-NEXT: v_mov_b32_e32 v0, s16
-; GFX942-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
+; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
-; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
-; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
+; GFX942-NEXT: s_waitcnt vmcnt(1)
+; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_lshl_or_b32 v0, v5, 16, v0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v7, v0
; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v7
; GFX942-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
@@ -7340,37 +7895,41 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_lshlrev_b32 v3, 16, v1
-; GFX11-TRUE16-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: buffer_load_u16 v4, v0, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v0
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -7389,39 +7948,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_and_b32 v3, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v0
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_min_f32_e32 v1, v1, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v1, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v1, v7, v9 :: v_dual_lshlrev_b32 v0, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_min_f32_e32 v0, v0, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v0, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v5, v8, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_perm_b32 v5, v1, v0, 0x7060302
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
@@ -7440,16 +8005,21 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, s20
-; GFX10-NEXT: v_mov_b32_e32 v4, s20
+; GFX10-NEXT: v_mov_b32_e32 v1, s20
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v6
@@ -7484,18 +8054,22 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: v_mov_b32_e32 v0, s20
-; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_waitcnt vmcnt(1)
+; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshl_or_b32 v0, v5, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v0
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v7
; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
@@ -7527,18 +8101,22 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, v0
-; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
-; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
+; GFX908-NEXT: s_waitcnt vmcnt(1)
+; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_lshl_or_b32 v0, v5, 16, v0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v0
; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v6
; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
@@ -7571,16 +8149,20 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s20
-; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_mov_b32_e32 v1, s20
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX8-NEXT: s_mov_b64 s[6:7], 0
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v0
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
@@ -7616,18 +8198,21 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX7-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7664,18 +8249,20 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v0
+; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v0
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX6-NEXT: s_waitcnt vmcnt(1)
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX6-NEXT: v_mov_b32_e32 v2, s6
; GFX6-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7723,19 +8310,23 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_and_b32 v2, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v1, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: s_clause 0x1
+; GFX12-TRUE16-NEXT: buffer_load_u16 v4, v1, s[0:3], null offen offset:1026
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX12-TRUE16-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_dual_min_num_f32 v5, v5, v2 :: v_dual_min_num_f32 v0, v0, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -7771,20 +8362,28 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_lshlrev_b32 v2, 16, v0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_and_b32 v3, 0xffff0000, v0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v1, s[0:3], null offen offset:1024
+; GFX12-FAKE16-NEXT: s_clause 0x1
+; GFX12-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1024
+; GFX12-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX12-FAKE16-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_dual_min_num_f32 v5, v5, v3 :: v_dual_min_num_f32 v0, v0, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v0, 16, 1
; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
@@ -7817,16 +8416,21 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v1, s16
-; GFX942-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: s_mov_b64 s[6:7], 0
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
+; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX942-NEXT: s_mov_b64 s[6:7], 0
+; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
+; GFX942-NEXT: s_waitcnt vmcnt(1)
+; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_lshl_or_b32 v1, v5, 16, v0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX942-NEXT: v_min_f32_e32 v0, v0, v2
@@ -7861,21 +8465,25 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_and_b32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: buffer_load_u16 v4, v1, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_min_f32 v5, v5, v2 :: v_dual_min_f32 v0, v0, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -7907,22 +8515,30 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_lshlrev_b32 v2, 16, v0
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_and_b32 v3, 0xffff0000, v0
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
-; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_min_f32 v5, v5, v3 :: v_dual_min_f32 v0, v0, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v0, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
@@ -7955,14 +8571,20 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s20
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
+; GFX10-NEXT: s_mov_b32 s5, 0
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshl_or_b32 v1, v4, 16, v1
; GFX10-NEXT: v_mov_b32_e32 v4, s20
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
; GFX10-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -7998,16 +8620,21 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_waitcnt vmcnt(1)
+; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshl_or_b32 v1, v5, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX90A-NEXT: v_min_f32_e32 v0, v0, v2
@@ -8040,16 +8667,21 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
+; GFX908-NEXT: s_waitcnt vmcnt(1)
+; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_lshl_or_b32 v1, v5, 16, v0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX908-NEXT: v_min_f32_e32 v0, v0, v2
@@ -8083,14 +8715,19 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s20
-; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX8-NEXT: v_min_f32_e32 v0, v0, v2
@@ -8126,18 +8763,21 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: v_mov_b32_e32 v1, s20
-; GFX7-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX7-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8170,18 +8810,20 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v0
+; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v0
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX6-NEXT: s_waitcnt vmcnt(1)
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX6-NEXT: v_mov_b32_e32 v2, s6
; GFX6-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8237,18 +8879,39 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v4, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: buffer_load_u16 v7, v4, s[0:3], null offen offset:1024
; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX12-TRUE16-NEXT: ; %bb.2:
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
+; GFX12-TRUE16-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: buffer_load_u16 v6, v4, s[0:3], null offen offset:1026
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_3
+; GFX12-TRUE16-NEXT: ; %bb.4:
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-TRUE16-NEXT: .LBB21_3: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: ; Child Loop BB21_4 Depth 2
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: ; Child Loop BB21_6 Depth 2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
@@ -8274,7 +8937,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v7
-; GFX12-TRUE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX12-TRUE16-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -8287,8 +8950,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_4
-; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_6
+; GFX12-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
@@ -8297,8 +8960,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_3
-; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_5
+; GFX12-TRUE16-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v5
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -8323,18 +8986,39 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v7, v4, s[0:3], null offen offset:1024
+; GFX12-FAKE16-NEXT: buffer_load_u16 v6, v4, s[0:3], null offen offset:1024
; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX12-FAKE16-NEXT: ; %bb.2:
; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
+; GFX12-FAKE16-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: buffer_load_u16 v7, v4, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_3
+; GFX12-FAKE16-NEXT: ; %bb.4:
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX12-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-FAKE16-NEXT: .LBB21_3: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: ; Child Loop BB21_4 Depth 2
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: ; Child Loop BB21_6 Depth 2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
@@ -8360,7 +9044,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, v6
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v7
-; GFX12-FAKE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX12-FAKE16-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -8373,8 +9057,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_4
-; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_6
+; GFX12-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
@@ -8383,8 +9067,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_3
-; GFX12-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_5
+; GFX12-FAKE16-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, v5
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -8403,20 +9087,38 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX942-NEXT: buffer_load_dword v9, v4, s[4:7], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v6, v4, s[4:7], 0 offen offset:1024
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB21_1
; GFX942-NEXT: ; %bb.2:
; GFX942-NEXT: s_mov_b64 exec, s[2:3]
+; GFX942-NEXT: s_mov_b64 s[2:3], exec
+; GFX942-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX942-NEXT: v_readfirstlane_b32 s4, v0
+; GFX942-NEXT: v_readfirstlane_b32 s5, v1
+; GFX942-NEXT: v_readfirstlane_b32 s6, v2
+; GFX942-NEXT: v_readfirstlane_b32 s7, v3
+; GFX942-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
+; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT: buffer_load_ushort v7, v4, s[4:7], 0 offen offset:1026
+; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_cbranch_execnz .LBB21_3
+; GFX942-NEXT: ; %bb.4:
+; GFX942-NEXT: s_mov_b64 exec, s[2:3]
+; GFX942-NEXT: s_mov_b32 s0, 0x5040100
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_perm_b32 v9, v7, v6, s0
; GFX942-NEXT: s_mov_b64 s[2:3], 0
; GFX942-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX942-NEXT: s_movk_i32 s10, 0x7fff
; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX942-NEXT: s_mov_b32 s11, 0x7060302
-; GFX942-NEXT: .LBB21_3: ; %atomicrmw.start
+; GFX942-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Loop Header: Depth=1
-; GFX942-NEXT: ; Child Loop BB21_4 Depth 2
-; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: ; Child Loop BB21_6 Depth 2
; GFX942-NEXT: v_lshlrev_b32_e32 v6, 16, v9
; GFX942-NEXT: v_min_f32_e32 v6, v6, v10
; GFX942-NEXT: v_bfe_u32 v7, v6, 16, 1
@@ -8437,7 +9139,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX942-NEXT: v_cndmask_b32_e32 v7, v8, v11, vcc
; GFX942-NEXT: v_perm_b32 v8, v7, v6, s11
; GFX942-NEXT: v_mov_b64_e32 v[6:7], v[8:9]
-; GFX942-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX942-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX942-NEXT: ; => This Inner Loop Header: Depth=2
; GFX942-NEXT: v_readfirstlane_b32 s4, v0
; GFX942-NEXT: v_readfirstlane_b32 s5, v1
@@ -8451,8 +9153,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 sc0
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB21_4
-; GFX942-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX942-NEXT: s_cbranch_execnz .LBB21_6
+; GFX942-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX942-NEXT: s_mov_b64 exec, s[8:9]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v6, v9
@@ -8460,8 +9162,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX942-NEXT: v_mov_b32_e32 v9, v6
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_cbranch_execnz .LBB21_3
-; GFX942-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX942-NEXT: s_cbranch_execnz .LBB21_5
+; GFX942-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-NEXT: v_mov_b32_e32 v0, v6
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -8480,28 +9182,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-TRUE16-NEXT: ; %bb.2:
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_u16 v6, v4, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_3
+; GFX11-TRUE16-NEXT: ; %bb.4:
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB21_3: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: ; Child Loop BB21_4 Depth 2
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: ; Child Loop BB21_6 Depth 2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_min_f32 v6, v6, v8 :: v_dual_min_f32 v5, v5, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v6, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
@@ -8516,7 +9235,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.h
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v7
-; GFX11-TRUE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX11-TRUE16-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -8528,8 +9247,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_4
-; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_6
+; GFX11-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
@@ -8539,8 +9258,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_3
-; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_5
+; GFX11-TRUE16-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v5
@@ -8560,28 +9279,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v6, v4, s[0:3], 0 offen offset:1024
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-FAKE16-NEXT: ; %bb.2:
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_3
+; GFX11-FAKE16-NEXT: ; %bb.4:
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB21_3: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: ; Child Loop BB21_4 Depth 2
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: ; Child Loop BB21_6 Depth 2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_min_f32 v6, v6, v9 :: v_dual_min_f32 v5, v5, v8
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_bfe_u32 v11, v6, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_bfe_u32 v10, v5, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
@@ -8596,7 +9332,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: v_perm_b32 v6, v6, v5, 0x7060302
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v7
-; GFX11-FAKE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX11-FAKE16-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -8608,8 +9344,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_4
-; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_6
+; GFX11-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
@@ -8619,8 +9355,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_3
-; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_5
+; GFX11-FAKE16-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v5
@@ -8640,18 +9376,35 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX10-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v6, v4, s[4:7], 0 offen offset:1024
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB21_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s10, s9
+; GFX10-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_ushort v7, v4, s[4:7], 0 offen offset:1026
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
+; GFX10-NEXT: s_cbranch_execnz .LBB21_3
+; GFX10-NEXT: ; %bb.4:
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; GFX10-NEXT: .LBB21_3: ; %atomicrmw.start
+; GFX10-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
-; GFX10-NEXT: ; Child Loop BB21_4 Depth 2
-; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: ; Child Loop BB21_6 Depth 2
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX10-NEXT: s_mov_b32 s9, exec_lo
@@ -8672,7 +9425,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX10-NEXT: v_perm_b32 v6, v6, v5, 0x7060302
; GFX10-NEXT: v_mov_b32_e32 v5, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v7
-; GFX10-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX10-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
@@ -8684,8 +9437,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[4:7], 0 offen offset:1024 glc
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
-; GFX10-NEXT: s_cbranch_execnz .LBB21_4
-; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX10-NEXT: s_cbranch_execnz .LBB21_6
+; GFX10-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -8696,8 +9449,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
-; GFX10-NEXT: s_cbranch_execnz .LBB21_3
-; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX10-NEXT: s_cbranch_execnz .LBB21_5
+; GFX10-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -8716,20 +9469,38 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_nop 0
-; GFX90A-NEXT: buffer_load_dword v9, v4, s[8:11], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB21_1
; GFX90A-NEXT: ; %bb.2:
; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
+; GFX90A-NEXT: s_mov_b64 s[6:7], exec
+; GFX90A-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
+; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
+; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
+; GFX90A-NEXT: v_readfirstlane_b32 s11, v3
+; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX90A-NEXT: s_nop 0
+; GFX90A-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1026
+; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_cbranch_execnz .LBB21_3
+; GFX90A-NEXT: ; %bb.4:
+; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
+; GFX90A-NEXT: s_mov_b32 s4, 0x5040100
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_perm_b32 v9, v7, v6, s4
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX90A-NEXT: s_movk_i32 s14, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX90A-NEXT: s_mov_b32 s15, 0x7060302
-; GFX90A-NEXT: .LBB21_3: ; %atomicrmw.start
+; GFX90A-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
-; GFX90A-NEXT: ; Child Loop BB21_4 Depth 2
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; Child Loop BB21_6 Depth 2
; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v9
; GFX90A-NEXT: v_min_f32_e32 v6, v6, v10
; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
@@ -8747,7 +9518,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX90A-NEXT: v_perm_b32 v8, v7, v6, s15
; GFX90A-NEXT: s_mov_b64 s[12:13], exec
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[8:9], v[8:9] op_sel:[0,1]
-; GFX90A-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX90A-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
@@ -8760,8 +9531,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: s_cbranch_execnz .LBB21_4
-; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX90A-NEXT: s_cbranch_execnz .LBB21_6
+; GFX90A-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v6, v9
@@ -8769,8 +9540,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX90A-NEXT: v_mov_b32_e32 v9, v6
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_execnz .LBB21_3
-; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX90A-NEXT: s_cbranch_execnz .LBB21_5
+; GFX90A-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v6
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -8789,20 +9560,38 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB21_1
; GFX908-NEXT: ; %bb.2:
; GFX908-NEXT: s_mov_b64 exec, s[6:7]
+; GFX908-NEXT: s_mov_b64 s[6:7], exec
+; GFX908-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX908-NEXT: v_readfirstlane_b32 s8, v0
+; GFX908-NEXT: v_readfirstlane_b32 s9, v1
+; GFX908-NEXT: v_readfirstlane_b32 s10, v2
+; GFX908-NEXT: v_readfirstlane_b32 s11, v3
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX908-NEXT: s_nop 0
+; GFX908-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1026
+; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX908-NEXT: s_cbranch_execnz .LBB21_3
+; GFX908-NEXT: ; %bb.4:
+; GFX908-NEXT: s_mov_b64 exec, s[6:7]
+; GFX908-NEXT: s_mov_b32 s4, 0x5040100
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_perm_b32 v7, v7, v6, s4
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX908-NEXT: s_movk_i32 s14, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX908-NEXT: s_mov_b32 s15, 0x7060302
-; GFX908-NEXT: .LBB21_3: ; %atomicrmw.start
+; GFX908-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Loop Header: Depth=1
-; GFX908-NEXT: ; Child Loop BB21_4 Depth 2
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: ; Child Loop BB21_6 Depth 2
; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX908-NEXT: v_min_f32_e32 v5, v5, v8
; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
@@ -8821,7 +9610,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX908-NEXT: v_mov_b32_e32 v5, v6
; GFX908-NEXT: s_mov_b64 s[12:13], exec
; GFX908-NEXT: v_mov_b32_e32 v6, v7
-; GFX908-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX908-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
; GFX908-NEXT: v_readfirstlane_b32 s9, v1
@@ -8834,8 +9623,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB21_4
-; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX908-NEXT: s_cbranch_execnz .LBB21_6
+; GFX908-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
@@ -8843,8 +9632,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB21_3
-; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX908-NEXT: s_cbranch_execnz .LBB21_5
+; GFX908-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
@@ -8863,18 +9652,36 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_nop 0
-; GFX8-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
+; GFX8-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB21_1
; GFX8-NEXT: ; %bb.2:
; GFX8-NEXT: s_mov_b64 exec, s[6:7]
+; GFX8-NEXT: s_mov_b64 s[6:7], exec
+; GFX8-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_readfirstlane_b32 s8, v0
+; GFX8-NEXT: v_readfirstlane_b32 s9, v1
+; GFX8-NEXT: v_readfirstlane_b32 s10, v2
+; GFX8-NEXT: v_readfirstlane_b32 s11, v3
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX8-NEXT: s_nop 0
+; GFX8-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1026
+; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB21_3
+; GFX8-NEXT: ; %bb.4:
+; GFX8-NEXT: s_mov_b64 exec, s[6:7]
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX8-NEXT: v_or_b32_e32 v7, v6, v7
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; GFX8-NEXT: .LBB21_3: ; %atomicrmw.start
+; GFX8-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Loop Header: Depth=1
-; GFX8-NEXT: ; Child Loop BB21_4 Depth 2
-; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: ; Child Loop BB21_6 Depth 2
; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX8-NEXT: v_min_f32_e32 v5, v5, v8
; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
@@ -8896,7 +9703,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX8-NEXT: v_mov_b32_e32 v5, v6
; GFX8-NEXT: s_mov_b64 s[12:13], exec
; GFX8-NEXT: v_mov_b32_e32 v6, v7
-; GFX8-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX8-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
; GFX8-NEXT: v_readfirstlane_b32 s9, v1
@@ -8909,8 +9716,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_execnz .LBB21_4
-; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX8-NEXT: s_cbranch_execnz .LBB21_6
+; GFX8-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
@@ -8918,8 +9725,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX8-NEXT: v_mov_b32_e32 v7, v5
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_cbranch_execnz .LBB21_3
-; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX8-NEXT: s_cbranch_execnz .LBB21_5
+; GFX8-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -8939,22 +9746,37 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1024
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB21_1
; GFX7-NEXT: ; %bb.2:
; GFX7-NEXT: s_mov_b64 exec, s[6:7]
+; GFX7-NEXT: s_mov_b64 s[6:7], exec
+; GFX7-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: v_readfirstlane_b32 s8, v0
+; GFX7-NEXT: v_readfirstlane_b32 s9, v1
+; GFX7-NEXT: v_readfirstlane_b32 s10, v2
+; GFX7-NEXT: v_readfirstlane_b32 s11, v3
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX7-NEXT: buffer_load_ushort v8, v4, s[8:11], 0 offen offset:1026
+; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_cbranch_execnz .LBB21_3
+; GFX7-NEXT: ; %bb.4:
+; GFX7-NEXT: s_mov_b64 exec, s[6:7]
; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
+; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
-; GFX7-NEXT: .LBB21_3: ; %atomicrmw.start
+; GFX7-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Loop Header: Depth=1
-; GFX7-NEXT: ; Child Loop BB21_4 Depth 2
+; GFX7-NEXT: ; Child Loop BB21_6 Depth 2
; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v8
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
@@ -8968,7 +9790,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX7-NEXT: v_mov_b32_e32 v8, v6
; GFX7-NEXT: s_mov_b64 s[12:13], exec
; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX7-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX7-NEXT: ; => This Inner Loop Header: Depth=2
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
; GFX7-NEXT: v_readfirstlane_b32 s9, v1
@@ -8981,8 +9803,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v4, s[8:11], 0 offen offset:1024 glc
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB21_4
-; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX7-NEXT: s_cbranch_execnz .LBB21_6
+; GFX7-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
@@ -8991,8 +9813,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX7-NEXT: s_cbranch_execnz .LBB21_3
-; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX7-NEXT: s_cbranch_execnz .LBB21_5
+; GFX7-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v8
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
@@ -9016,24 +9838,40 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: buffer_load_dword v9, v4, s[8:11], 0 offen offset:1024
-; GFX6-NEXT: ; implicit-def: $vgpr4
+; GFX6-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1024
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB21_1
; GFX6-NEXT: ; %bb.2:
; GFX6-NEXT: s_mov_b64 exec, s[6:7]
+; GFX6-NEXT: s_mov_b64 s[6:7], exec
+; GFX6-NEXT: .LBB21_3: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: v_readfirstlane_b32 s8, v0
+; GFX6-NEXT: v_readfirstlane_b32 s9, v1
+; GFX6-NEXT: v_readfirstlane_b32 s10, v2
+; GFX6-NEXT: v_readfirstlane_b32 s11, v3
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
+; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
+; GFX6-NEXT: buffer_load_ushort v9, v4, s[8:11], 0 offen offset:1026
+; GFX6-NEXT: ; implicit-def: $vgpr4
+; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
+; GFX6-NEXT: s_cbranch_execnz .LBB21_3
+; GFX6-NEXT: ; %bb.4:
+; GFX6-NEXT: s_mov_b64 exec, s[6:7]
; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v6
; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v9
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v9
+; GFX6-NEXT: v_lshlrev_b32_e32 v11, 16, v9
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v7
; GFX6-NEXT: s_mov_b64 s[6:7], 0
; GFX6-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
; GFX6-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
-; GFX6-NEXT: .LBB21_3: ; %atomicrmw.start
+; GFX6-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Loop Header: Depth=1
-; GFX6-NEXT: ; Child Loop BB21_4 Depth 2
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v7
+; GFX6-NEXT: ; Child Loop BB21_6 Depth 2
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v11
+; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
; GFX6-NEXT: v_mul_f32_e32 v7, 1.0, v4
; GFX6-NEXT: v_min_f32_e32 v6, v6, v9
@@ -9046,7 +9884,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX6-NEXT: v_mov_b32_e32 v7, v5
; GFX6-NEXT: s_mov_b64 s[12:13], exec
; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
+; GFX6-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX6-NEXT: ; => This Inner Loop Header: Depth=2
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
; GFX6-NEXT: v_readfirstlane_b32 s9, v1
@@ -9059,24 +9897,24 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v8, s[8:11], 0 offen glc
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_cbranch_execnz .LBB21_4
-; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
+; GFX6-NEXT: s_cbranch_execnz .LBB21_6
+; GFX6-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v6
+; GFX6-NEXT: v_and_b32_e32 v11, 0xffff0000, v6
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX6-NEXT: s_cbranch_execnz .LBB21_3
-; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX6-NEXT: s_cbranch_execnz .LBB21_5
+; GFX6-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v7
+; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v11
; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v4
; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 256
%result = atomicrmw fmin ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
index fa1c1854c1d9a..f7a3928e30f9d 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
@@ -3056,19 +3056,17 @@ define [2 x half] @load_a2f16(ptr addrspace(8) inreg %buf) {
; SDAG-LABEL: load_a2f16:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-NEXT: buffer_load_dword v0, off, s[16:19], 0
+; SDAG-NEXT: buffer_load_ushort v0, off, s[16:19], 0
+; SDAG-NEXT: buffer_load_ushort v1, off, s[16:19], 0 offset:2
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-LABEL: load_a2f16:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: buffer_load_dword v0, off, s[16:19], 0
+; GISEL-NEXT: buffer_load_ushort v0, off, s[16:19], 0
+; GISEL-NEXT: buffer_load_ushort v1, off, s[16:19], 0 offset:2
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: v_readfirstlane_b32 s4, v0
-; GISEL-NEXT: s_lshr_b32 s4, s4, 16
-; GISEL-NEXT: v_mov_b32_e32 v1, s4
; GISEL-NEXT: s_setpc_b64 s[30:31]
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
%ret = load [2 x half], ptr addrspace(7) %p
@@ -3079,18 +3077,16 @@ define void @store_a2f16([2 x half] %data, ptr addrspace(8) inreg %buf) {
; SDAG-LABEL: store_a2f16:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-NEXT: s_mov_b32 s4, 0x5040100
-; SDAG-NEXT: v_perm_b32 v0, v1, v0, s4
-; SDAG-NEXT: buffer_store_dword v0, off, s[16:19], 0
+; SDAG-NEXT: buffer_store_short v0, off, s[16:19], 0
+; SDAG-NEXT: buffer_store_short v1, off, s[16:19], 0 offset:2
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-LABEL: store_a2f16:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 0
+; GISEL-NEXT: buffer_store_short v0, off, s[16:19], 0
+; GISEL-NEXT: buffer_store_short v1, off, s[16:19], 0 offset:2
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_setpc_b64 s[30:31]
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -4167,3 +4163,7 @@ define void @store_a4i7([4 x i7] %data, ptr addrspace(8) inreg %buf) {
store [4 x i7] %data, ptr addrspace(7) %p
ret void
}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 7, !"amdgpu.buffer.oob.mode", i32 1}
+
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll
index fe5dd9dec0f54..9e8927014f575 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -check-prefix=SDAG-GFX942 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 < %s | FileCheck -check-prefix=SDAG-GFX1100 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx942 -global-isel=1 < %s | FileCheck -check-prefix=GISEL-GFX942 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -global-isel=1 < %s | FileCheck -check-prefix=GISEL-GFX1100 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx942 -mattr=+unaligned-access-mode < %s | FileCheck -check-prefix=SDAG-GFX942 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+unaligned-access-mode < %s | FileCheck -check-prefix=SDAG-GFX1100 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx942 -mattr=+unaligned-access-mode -global-isel=1 < %s | FileCheck -check-prefix=GISEL-GFX942 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+unaligned-access-mode -global-isel=1 < %s | FileCheck -check-prefix=GISEL-GFX1100 %s
; Note: if you're adding tests here, also add them to
; lower-buffer-fat-pointers-mem-transfer.ll to verify the IR produced by
@@ -1163,20 +1163,20 @@ define amdgpu_kernel void @memcpy_known_small(ptr addrspace(7) %src, ptr addrspa
; GISEL-GFX942-NEXT: s_mov_b32 s8, s1
; GISEL-GFX942-NEXT: s_mov_b32 s9, s2
; GISEL-GFX942-NEXT: s_mov_b32 s10, s3
-; GISEL-GFX942-NEXT: v_mov_b32_e32 v4, s0
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[0:3], v4, s[8:11], 0 offen
+; GISEL-GFX942-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[2:5], v0, s[8:11], 0 offen
; GISEL-GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x44
; GISEL-GFX942-NEXT: s_load_dword s7, s[4:5], 0x54
; GISEL-GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX942-NEXT: s_mov_b32 s4, s1
; GISEL-GFX942-NEXT: s_mov_b32 s5, s2
; GISEL-GFX942-NEXT: s_mov_b32 s6, s3
-; GISEL-GFX942-NEXT: v_mov_b32_e32 v5, s0
+; GISEL-GFX942-NEXT: v_mov_b32_e32 v1, s0
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(0)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[0:3], v5, s[4:7], 0 offen
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[0:3], v4, s[8:11], 0 offen offset:16
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[2:5], v1, s[4:7], 0 offen
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[2:5], v0, s[8:11], 0 offen offset:16
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(0)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[0:3], v5, s[4:7], 0 offen offset:16
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[2:5], v1, s[4:7], 0 offen offset:16
; GISEL-GFX942-NEXT: s_endpgm
;
; GISEL-GFX1100-LABEL: memcpy_known_small:
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index 070c873798647..97cf8eecf3c1f 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -56,6 +56,10 @@
; GCN-O0-NEXT: Dominator Tree Construction
; GCN-O0-NEXT: AMDGPU Lower Kernel Arguments
; GCN-O0-NEXT: Lower buffer fat pointer operations to buffer resources
+; GCN-O0-NEXT: FunctionPass Manager
+; GCN-O0-NEXT: Dominator Tree Construction
+; GCN-O0-NEXT: Natural Loop Information
+; GCN-O0-NEXT: Scalar Evolution Analysis
; GCN-O0-NEXT: AMDGPU lower intrinsics
; GCN-O0-NEXT: FunctionPass Manager
; GCN-O0-NEXT: Lazy Value Information Analysis
@@ -250,6 +254,10 @@
; GCN-O1-NEXT: Block Frequency Analysis
; GCN-O1-NEXT: CodeGen Prepare
; GCN-O1-NEXT: Lower buffer fat pointer operations to buffer resources
+; GCN-O1-NEXT: FunctionPass Manager
+; GCN-O1-NEXT: Dominator Tree Construction
+; GCN-O1-NEXT: Natural Loop Information
+; GCN-O1-NEXT: Scalar Evolution Analysis
; GCN-O1-NEXT: AMDGPU lower intrinsics
; GCN-O1-NEXT: FunctionPass Manager
; GCN-O1-NEXT: Lazy Value Information Analysis
@@ -561,6 +569,10 @@
; GCN-O1-OPTS-NEXT: Scalar Evolution Analysis
; GCN-O1-OPTS-NEXT: GPU Load and Store Vectorizer
; GCN-O1-OPTS-NEXT: Lower buffer fat pointer operations to buffer resources
+; GCN-O1-OPTS-NEXT: FunctionPass Manager
+; GCN-O1-OPTS-NEXT: Dominator Tree Construction
+; GCN-O1-OPTS-NEXT: Natural Loop Information
+; GCN-O1-OPTS-NEXT: Scalar Evolution Analysis
; GCN-O1-OPTS-NEXT: AMDGPU lower intrinsics
; GCN-O1-OPTS-NEXT: FunctionPass Manager
; GCN-O1-OPTS-NEXT: Lazy Value Information Analysis
@@ -882,6 +894,10 @@
; GCN-O2-NEXT: Scalar Evolution Analysis
; GCN-O2-NEXT: GPU Load and Store Vectorizer
; GCN-O2-NEXT: Lower buffer fat pointer operations to buffer resources
+; GCN-O2-NEXT: FunctionPass Manager
+; GCN-O2-NEXT: Dominator Tree Construction
+; GCN-O2-NEXT: Natural Loop Information
+; GCN-O2-NEXT: Scalar Evolution Analysis
; GCN-O2-NEXT: AMDGPU lower intrinsics
; GCN-O2-NEXT: FunctionPass Manager
; GCN-O2-NEXT: Lazy Value Information Analysis
@@ -1218,6 +1234,10 @@
; GCN-O3-NEXT: Scalar Evolution Analysis
; GCN-O3-NEXT: GPU Load and Store Vectorizer
; GCN-O3-NEXT: Lower buffer fat pointer operations to buffer resources
+; GCN-O3-NEXT: FunctionPass Manager
+; GCN-O3-NEXT: Dominator Tree Construction
+; GCN-O3-NEXT: Natural Loop Information
+; GCN-O3-NEXT: Scalar Evolution Analysis
; GCN-O3-NEXT: AMDGPU lower intrinsics
; GCN-O3-NEXT: FunctionPass Manager
; GCN-O3-NEXT: Lazy Value Information Analysis
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-calls.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-calls.ll
index 1288a40483f71..f03c4ec746cfe 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-calls.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-calls.ll
@@ -92,7 +92,7 @@ define void @caller(ptr addrspace(7) noundef nonnull %arg) {
; CHECK-NEXT: [[V_INT_LEGAL:%.*]] = bitcast i160 [[V_INT]] to <5 x i32>
; CHECK-NEXT: [[V_INT_SLICE_0:%.*]] = shufflevector <5 x i32> [[V_INT_LEGAL]], <5 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[V_INT_SLICE_0]], ptr addrspace(8) align 32 [[ARG_RSRC]], i32 [[ARG_OFF]], i32 0, i32 0)
-; CHECK-NEXT: [[ARG_PART_4:%.*]] = add nuw i32 [[ARG_OFF]], 16
+; CHECK-NEXT: [[ARG_PART_4:%.*]] = add i32 [[ARG_OFF]], 16
; CHECK-NEXT: [[V_INT_SLICE_4:%.*]] = extractelement <5 x i32> [[V_INT_LEGAL]], i64 4
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[V_INT_SLICE_4]], ptr addrspace(8) align 16 [[ARG_RSRC]], i32 [[ARG_PART_4]], i32 0, i32 0)
; CHECK-NEXT: ret void
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization-alignment.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization-alignment.ll
new file mode 100644
index 0000000000000..9b99d6fa5bb1d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization-alignment.ll
@@ -0,0 +1,3280 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: sed 's/BUFFER_OOB_MODE/2/' %s | opt -S -mcpu=gfx900 -passes=amdgpu-lower-buffer-fat-pointers | FileCheck %s --check-prefix=STRICT
+;; Note: unaligned-access-mode is default on HSA targets
+; RUN: sed 's/BUFFER_OOB_MODE/2/' %s | opt -S -mcpu=gfx900 -passes=amdgpu-lower-buffer-fat-pointers -mattr=+unaligned-access-mode | FileCheck %s --check-prefix=UNALIGNED_ONLY
+; RUN: sed 's/BUFFER_OOB_MODE/1/' %s | opt -S -mcpu=gfx900 -passes=amdgpu-lower-buffer-fat-pointers | FileCheck %s --check-prefix=RELAXED_OOB_ONLY
+; RUN: sed 's/BUFFER_OOB_MODE/1/' %s | opt -S -mcpu=gfx900 -passes=amdgpu-lower-buffer-fat-pointers -mattr=+unaligned-access-mode | FileCheck %s --check-prefix=BOTH_FLAGS
+
+target triple = "amdgcn--"
+
+define i32 @load_i32_align4(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define i32 @load_i32_align4(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0:[0-9]+]] {
+; STRICT-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: ret i32 [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define i32 @load_i32_align4(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0:[0-9]+]] {
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret i32 [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define i32 @load_i32_align4(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0:[0-9]+]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret i32 [[RET]]
+;
+; BOTH_FLAGS-LABEL: define i32 @load_i32_align4(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0:[0-9]+]] {
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret i32 [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load i32, ptr addrspace(7) %q, align 4
+ ret i32 %ret
+}
+
+define void @store_i32_align4(i32 %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_i32_align4(
+; STRICT-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_i32_align4(
+; UNALIGNED_ONLY-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_i32_align4(
+; RELAXED_OOB_ONLY-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_i32_align4(
+; BOTH_FLAGS-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store i32 %data, ptr addrspace(7) %q, align 4
+ ret void
+}
+
+define i32 @load_i32_align2(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define i32 @load_i32_align2(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i16> poison, i16 [[RET_OFF_0]], i64 0
+; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <2 x i16> [[RET_SLICE_0]], i16 [[RET_OFF_2]], i64 1
+; STRICT-NEXT: [[RET:%.*]] = bitcast <2 x i16> [[RET_SLICE_1]] to i32
+; STRICT-NEXT: ret i32 [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define i32 @load_i32_align2(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret i32 [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define i32 @load_i32_align2(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i16> poison, i16 [[RET_OFF_0]], i64 0
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add nuw i32 [[Q]], 2
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <2 x i16> [[RET_SLICE_0]], i16 [[RET_OFF_2]], i64 1
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast <2 x i16> [[RET_SLICE_1]] to i32
+; RELAXED_OOB_ONLY-NEXT: ret i32 [[RET]]
+;
+; BOTH_FLAGS-LABEL: define i32 @load_i32_align2(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret i32 [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load i32, ptr addrspace(7) %q, align 2
+ ret i32 %ret
+}
+
+define void @store_i32_align2(i32 %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_i32_align2(
+; STRICT-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast i32 [[DATA]] to <2 x i16>
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA_LEGAL]], i64 0
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA_LEGAL]], i64 1
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_i32_align2(
+; UNALIGNED_ONLY-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_i32_align2(
+; RELAXED_OOB_ONLY-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast i32 [[DATA]] to <2 x i16>
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA_LEGAL]], i64 0
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 2
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA_LEGAL]], i64 1
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_i32_align2(
+; BOTH_FLAGS-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store i32 %data, ptr addrspace(7) %q, align 2
+ ret void
+}
+
+define i32 @load_i32_align1(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define i32 @load_i32_align1(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
+; STRICT-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
+; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_3:%.*]] = insertelement <4 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
+; STRICT-NEXT: [[RET:%.*]] = bitcast <4 x i8> [[RET_SLICE_3]] to i32
+; STRICT-NEXT: ret i32 [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define i32 @load_i32_align1(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret i32 [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define i32 @load_i32_align1(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add nuw i32 [[Q]], 1
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add nuw i32 [[Q]], 2
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_3:%.*]] = add nuw i32 [[Q]], 3
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_3:%.*]] = insertelement <4 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast <4 x i8> [[RET_SLICE_3]] to i32
+; RELAXED_OOB_ONLY-NEXT: ret i32 [[RET]]
+;
+; BOTH_FLAGS-LABEL: define i32 @load_i32_align1(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret i32 [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load i32, ptr addrspace(7) %q, align 1
+ ret i32 %ret
+}
+
+define void @store_i32_align1(i32 %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_i32_align1(
+; STRICT-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast i32 [[DATA]] to <4 x i8>
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 0
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 1
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 2
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
+; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 3
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_i32_align1(
+; UNALIGNED_ONLY-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_i32_align1(
+; RELAXED_OOB_ONLY-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast i32 [[DATA]] to <4 x i8>
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 0
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 1
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 1
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_2:%.*]] = add nuw i32 [[Q]], 2
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 2
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_3:%.*]] = add nuw i32 [[Q]], 3
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 3
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_i32_align1(
+; BOTH_FLAGS-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store i32 %data, ptr addrspace(7) %q, align 1
+ ret void
+}
+
+define i64 @load_i64_align4(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define i64 @load_i64_align4(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: ret i64 [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define i64 @load_i64_align4(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret i64 [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define i64 @load_i64_align4(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret i64 [[RET]]
+;
+; BOTH_FLAGS-LABEL: define i64 @load_i64_align4(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret i64 [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load i64, ptr addrspace(7) %q, align 4
+ ret i64 %ret
+}
+
+define void @store_i64_align4(i64 %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_i64_align4(
+; STRICT-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_i64_align4(
+; UNALIGNED_ONLY-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_i64_align4(
+; RELAXED_OOB_ONLY-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_i64_align4(
+; BOTH_FLAGS-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store i64 %data, ptr addrspace(7) %q, align 4
+ ret void
+}
+
+define i64 @load_i64_align1(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define i64 @load_i64_align1(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
+; STRICT-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
+; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
+; STRICT-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
+; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
+; STRICT-NEXT: [[Q_OFF_PTR_5:%.*]] = add i32 [[Q]], 5
+; STRICT-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
+; STRICT-NEXT: [[Q_OFF_PTR_6:%.*]] = add i32 [[Q]], 6
+; STRICT-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
+; STRICT-NEXT: [[Q_OFF_PTR_7:%.*]] = add i32 [[Q]], 7
+; STRICT-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_7:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
+; STRICT-NEXT: [[RET:%.*]] = bitcast <8 x i8> [[RET_SLICE_7]] to i64
+; STRICT-NEXT: ret i64 [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define i64 @load_i64_align1(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret i64 [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define i64 @load_i64_align1(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add nuw i32 [[Q]], 1
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add nuw i32 [[Q]], 2
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_3:%.*]] = add nuw i32 [[Q]], 3
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add nuw i32 [[Q]], 4
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_5:%.*]] = add nuw i32 [[Q]], 5
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_6:%.*]] = add nuw i32 [[Q]], 6
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_7:%.*]] = add nuw i32 [[Q]], 7
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_7:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast <8 x i8> [[RET_SLICE_7]] to i64
+; RELAXED_OOB_ONLY-NEXT: ret i64 [[RET]]
+;
+; BOTH_FLAGS-LABEL: define i64 @load_i64_align1(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret i64 [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load i64, ptr addrspace(7) %q, align 1
+ ret i64 %ret
+}
+
+define void @store_i64_align1(i64 %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_i64_align1(
+; STRICT-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast i64 [[DATA]] to <8 x i8>
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
+; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
+; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
+; STRICT-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
+; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
+; STRICT-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_i64_align1(
+; UNALIGNED_ONLY-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_i64_align1(
+; RELAXED_OOB_ONLY-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast i64 [[DATA]] to <8 x i8>
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 1
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_2:%.*]] = add nuw i32 [[Q]], 2
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_3:%.*]] = add nuw i32 [[Q]], 3
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 4
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_5:%.*]] = add nuw i32 [[Q]], 5
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_6:%.*]] = add nuw i32 [[Q]], 6
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_7:%.*]] = add nuw i32 [[Q]], 7
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_i64_align1(
+; BOTH_FLAGS-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store i64 %data, ptr addrspace(7) %q, align 1
+ ret void
+}
+
+define <2 x i32> @load_v2i32_align4(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define <2 x i32> @load_v2i32_align4(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i32> poison, i32 [[RET_OFF_0]], i64 0
+; STRICT-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
+; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; STRICT-NEXT: [[RET:%.*]] = insertelement <2 x i32> [[RET_SLICE_0]], i32 [[RET_OFF_4]], i64 1
+; STRICT-NEXT: ret <2 x i32> [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define <2 x i32> @load_v2i32_align4(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i32> poison, i32 [[RET_OFF_0]], i64 0
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <2 x i32> [[RET_SLICE_0]], i32 [[RET_OFF_4]], i64 1
+; UNALIGNED_ONLY-NEXT: ret <2 x i32> [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define <2 x i32> @load_v2i32_align4(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret <2 x i32> [[RET]]
+;
+; BOTH_FLAGS-LABEL: define <2 x i32> @load_v2i32_align4(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret <2 x i32> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load <2 x i32>, ptr addrspace(7) %q, align 4
+ ret <2 x i32> %ret
+}
+
+define void @store_v2i32_align4(<2 x i32> %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_v2i32_align4(
+; STRICT-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i32> [[DATA]], i64 0
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 4
+; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i32> [[DATA]], i64 1
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_v2i32_align4(
+; UNALIGNED_ONLY-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i32> [[DATA]], i64 0
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 4
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i32> [[DATA]], i64 1
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_v2i32_align4(
+; RELAXED_OOB_ONLY-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_v2i32_align4(
+; BOTH_FLAGS-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store <2 x i32> %data, ptr addrspace(7) %q, align 4
+ ret void
+}
+
+define <2 x i32> @load_v2i32_align2(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define <2 x i32> @load_v2i32_align2(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i16> poison, i16 [[RET_OFF_0]], i64 0
+; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i16> [[RET_SLICE_0]], i16 [[RET_OFF_2]], i64 1
+; STRICT-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
+; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i16> [[RET_SLICE_1]], i16 [[RET_OFF_4]], i64 2
+; STRICT-NEXT: [[Q_OFF_PTR_6:%.*]] = add i32 [[Q]], 6
+; STRICT-NEXT: [[RET_OFF_6:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_3:%.*]] = insertelement <4 x i16> [[RET_SLICE_2]], i16 [[RET_OFF_6]], i64 3
+; STRICT-NEXT: [[RET:%.*]] = bitcast <4 x i16> [[RET_SLICE_3]] to <2 x i32>
+; STRICT-NEXT: ret <2 x i32> [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define <2 x i32> @load_v2i32_align2(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i32> poison, i32 [[RET_OFF_0]], i64 0
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <2 x i32> [[RET_SLICE_0]], i32 [[RET_OFF_4]], i64 1
+; UNALIGNED_ONLY-NEXT: ret <2 x i32> [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define <2 x i32> @load_v2i32_align2(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i16> poison, i16 [[RET_OFF_0]], i64 0
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add nuw i32 [[Q]], 2
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i16> [[RET_SLICE_0]], i16 [[RET_OFF_2]], i64 1
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add nuw i32 [[Q]], 4
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i16> [[RET_SLICE_1]], i16 [[RET_OFF_4]], i64 2
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_6:%.*]] = add nuw i32 [[Q]], 6
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_3:%.*]] = insertelement <4 x i16> [[RET_SLICE_2]], i16 [[RET_OFF_6]], i64 3
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast <4 x i16> [[RET_SLICE_3]] to <2 x i32>
+; RELAXED_OOB_ONLY-NEXT: ret <2 x i32> [[RET]]
+;
+; BOTH_FLAGS-LABEL: define <2 x i32> @load_v2i32_align2(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret <2 x i32> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load <2 x i32>, ptr addrspace(7) %q, align 2
+ ret <2 x i32> %ret
+}
+
+define void @store_v2i32_align2(<2 x i32> %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_v2i32_align2(
+; STRICT-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i32> [[DATA]] to <4 x i16>
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 0
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 1
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 4
+; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 2
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 6
+; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 3
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_3]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_v2i32_align2(
+; UNALIGNED_ONLY-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i32> [[DATA]], i64 0
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 4
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i32> [[DATA]], i64 1
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_v2i32_align2(
+; RELAXED_OOB_ONLY-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i32> [[DATA]] to <4 x i16>
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 0
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 2
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 1
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_2:%.*]] = add nuw i32 [[Q]], 4
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 2
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_3:%.*]] = add nuw i32 [[Q]], 6
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 3
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_3]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_v2i32_align2(
+; BOTH_FLAGS-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store <2 x i32> %data, ptr addrspace(7) %q, align 2
+ ret void
+}
+
+define <2 x i32> @load_v2i32_align1(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define <2 x i32> @load_v2i32_align1(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
+; STRICT-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
+; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
+; STRICT-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
+; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
+; STRICT-NEXT: [[Q_OFF_PTR_5:%.*]] = add i32 [[Q]], 5
+; STRICT-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
+; STRICT-NEXT: [[Q_OFF_PTR_6:%.*]] = add i32 [[Q]], 6
+; STRICT-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
+; STRICT-NEXT: [[Q_OFF_PTR_7:%.*]] = add i32 [[Q]], 7
+; STRICT-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_7:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
+; STRICT-NEXT: [[RET:%.*]] = bitcast <8 x i8> [[RET_SLICE_7]] to <2 x i32>
+; STRICT-NEXT: ret <2 x i32> [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define <2 x i32> @load_v2i32_align1(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i32> poison, i32 [[RET_OFF_0]], i64 0
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <2 x i32> [[RET_SLICE_0]], i32 [[RET_OFF_4]], i64 1
+; UNALIGNED_ONLY-NEXT: ret <2 x i32> [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define <2 x i32> @load_v2i32_align1(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add nuw i32 [[Q]], 1
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add nuw i32 [[Q]], 2
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_3:%.*]] = add nuw i32 [[Q]], 3
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add nuw i32 [[Q]], 4
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_5:%.*]] = add nuw i32 [[Q]], 5
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_6:%.*]] = add nuw i32 [[Q]], 6
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_7:%.*]] = add nuw i32 [[Q]], 7
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_7:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast <8 x i8> [[RET_SLICE_7]] to <2 x i32>
+; RELAXED_OOB_ONLY-NEXT: ret <2 x i32> [[RET]]
+;
+; BOTH_FLAGS-LABEL: define <2 x i32> @load_v2i32_align1(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret <2 x i32> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load <2 x i32>, ptr addrspace(7) %q, align 1
+ ret <2 x i32> %ret
+}
+
+define void @store_v2i32_align1(<2 x i32> %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_v2i32_align1(
+; STRICT-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i32> [[DATA]] to <8 x i8>
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
+; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
+; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
+; STRICT-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
+; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
+; STRICT-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_v2i32_align1(
+; UNALIGNED_ONLY-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i32> [[DATA]], i64 0
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 4
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i32> [[DATA]], i64 1
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_v2i32_align1(
+; RELAXED_OOB_ONLY-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i32> [[DATA]] to <8 x i8>
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 1
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_2:%.*]] = add nuw i32 [[Q]], 2
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_3:%.*]] = add nuw i32 [[Q]], 3
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 4
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_5:%.*]] = add nuw i32 [[Q]], 5
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_6:%.*]] = add nuw i32 [[Q]], 6
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_7:%.*]] = add nuw i32 [[Q]], 7
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_v2i32_align1(
+; BOTH_FLAGS-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store <2 x i32> %data, ptr addrspace(7) %q, align 1
+ ret void
+}
+
+define <3 x i32> @load_v3i32_align4(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define <3 x i32> @load_v3i32_align4(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <3 x i32> poison, i32 [[RET_OFF_0]], i64 0
+; STRICT-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
+; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <3 x i32> [[RET_SLICE_0]], i32 [[RET_OFF_4]], i64 1
+; STRICT-NEXT: [[Q_OFF_PTR_8:%.*]] = add i32 [[Q]], 8
+; STRICT-NEXT: [[RET_OFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; STRICT-NEXT: [[RET:%.*]] = insertelement <3 x i32> [[RET_SLICE_1]], i32 [[RET_OFF_8]], i64 2
+; STRICT-NEXT: ret <3 x i32> [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define <3 x i32> @load_v3i32_align4(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <3 x i32> poison, i32 [[RET_OFF_0]], i64 0
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <3 x i32> [[RET_SLICE_0]], i32 [[RET_OFF_4]], i64 1
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_8:%.*]] = add i32 [[Q]], 8
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <3 x i32> [[RET_SLICE_1]], i32 [[RET_OFF_8]], i64 2
+; UNALIGNED_ONLY-NEXT: ret <3 x i32> [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define <3 x i32> @load_v3i32_align4(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret <3 x i32> [[RET]]
+;
+; BOTH_FLAGS-LABEL: define <3 x i32> @load_v3i32_align4(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret <3 x i32> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load <3 x i32>, ptr addrspace(7) %q, align 4
+ ret <3 x i32> %ret
+}
+
+define void @store_v3i32_align4(<3 x i32> %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_v3i32_align4(
+; STRICT-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <3 x i32> [[DATA]], i64 0
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 4
+; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <3 x i32> [[DATA]], i64 1
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
+; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <3 x i32> [[DATA]], i64 2
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_v3i32_align4(
+; UNALIGNED_ONLY-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <3 x i32> [[DATA]], i64 0
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 4
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <3 x i32> [[DATA]], i64 1
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <3 x i32> [[DATA]], i64 2
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_v3i32_align4(
+; RELAXED_OOB_ONLY-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_v3i32_align4(
+; BOTH_FLAGS-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store <3 x i32> %data, ptr addrspace(7) %q, align 4
+ ret void
+}
+
+define <3 x i32> @load_v3i32_align8(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define <3 x i32> @load_v3i32_align8(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_EXT_0:%.*]] = shufflevector <2 x i32> [[RET_OFF_0]], <2 x i32> poison, <3 x i32> <i32 0, i32 1, i32 poison>
+; STRICT-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <3 x i32> poison, <3 x i32> [[RET_EXT_0]], <3 x i32> <i32 3, i32 4, i32 2>
+; STRICT-NEXT: [[Q_OFF_PTR_8:%.*]] = add i32 [[Q]], 8
+; STRICT-NEXT: [[RET_OFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; STRICT-NEXT: [[RET:%.*]] = insertelement <3 x i32> [[RET_PARTS_0]], i32 [[RET_OFF_8]], i64 2
+; STRICT-NEXT: ret <3 x i32> [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define <3 x i32> @load_v3i32_align8(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_EXT_0:%.*]] = shufflevector <2 x i32> [[RET_OFF_0]], <2 x i32> poison, <3 x i32> <i32 0, i32 1, i32 poison>
+; UNALIGNED_ONLY-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <3 x i32> poison, <3 x i32> [[RET_EXT_0]], <3 x i32> <i32 3, i32 4, i32 2>
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_8:%.*]] = add i32 [[Q]], 8
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <3 x i32> [[RET_PARTS_0]], i32 [[RET_OFF_8]], i64 2
+; UNALIGNED_ONLY-NEXT: ret <3 x i32> [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define <3 x i32> @load_v3i32_align8(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret <3 x i32> [[RET]]
+;
+; BOTH_FLAGS-LABEL: define <3 x i32> @load_v3i32_align8(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret <3 x i32> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load <3 x i32>, ptr addrspace(7) %q, align 8
+ ret <3 x i32> %ret
+}
+
+define void @store_v3i32_align8(<3 x i32> %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_v3i32_align8(
+; STRICT-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <3 x i32> [[DATA]], <3 x i32> poison, <2 x i32> <i32 0, i32 1>
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
+; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <3 x i32> [[DATA]], i64 2
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_v3i32_align8(
+; UNALIGNED_ONLY-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <3 x i32> [[DATA]], <3 x i32> poison, <2 x i32> <i32 0, i32 1>
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <3 x i32> [[DATA]], i64 2
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_v3i32_align8(
+; RELAXED_OOB_ONLY-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_v3i32_align8(
+; BOTH_FLAGS-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store <3 x i32> %data, ptr addrspace(7) %q, align 8
+ ret void
+}
+
+define <3 x i32> @load_v3i32_align16(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define <3 x i32> @load_v3i32_align16(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: ret <3 x i32> [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define <3 x i32> @load_v3i32_align16(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret <3 x i32> [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define <3 x i32> @load_v3i32_align16(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret <3 x i32> [[RET]]
+;
+; BOTH_FLAGS-LABEL: define <3 x i32> @load_v3i32_align16(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret <3 x i32> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load <3 x i32>, ptr addrspace(7) %q, align 16
+ ret <3 x i32> %ret
+}
+
+define void @store_v3i32_align16(<3 x i32> %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_v3i32_align16(
+; STRICT-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_v3i32_align16(
+; UNALIGNED_ONLY-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_v3i32_align16(
+; RELAXED_OOB_ONLY-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_v3i32_align16(
+; BOTH_FLAGS-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store <3 x i32> %data, ptr addrspace(7) %q, align 16
+ ret void
+}
+
+define <4 x i32> @load_v4i32_align8(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define <4 x i32> @load_v4i32_align8(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_EXT_0:%.*]] = shufflevector <2 x i32> [[RET_OFF_0]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; STRICT-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <4 x i32> poison, <4 x i32> [[RET_EXT_0]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; STRICT-NEXT: [[Q_OFF_PTR_8:%.*]] = add i32 [[Q]], 8
+; STRICT-NEXT: [[RET_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_EXT_2:%.*]] = shufflevector <2 x i32> [[RET_OFF_8]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; STRICT-NEXT: [[RET:%.*]] = shufflevector <4 x i32> [[RET_PARTS_0]], <4 x i32> [[RET_EXT_2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; STRICT-NEXT: ret <4 x i32> [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define <4 x i32> @load_v4i32_align8(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_EXT_0:%.*]] = shufflevector <2 x i32> [[RET_OFF_0]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; UNALIGNED_ONLY-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <4 x i32> poison, <4 x i32> [[RET_EXT_0]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_8:%.*]] = add i32 [[Q]], 8
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_EXT_2:%.*]] = shufflevector <2 x i32> [[RET_OFF_8]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = shufflevector <4 x i32> [[RET_PARTS_0]], <4 x i32> [[RET_EXT_2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; UNALIGNED_ONLY-NEXT: ret <4 x i32> [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define <4 x i32> @load_v4i32_align8(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret <4 x i32> [[RET]]
+;
+; BOTH_FLAGS-LABEL: define <4 x i32> @load_v4i32_align8(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret <4 x i32> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load <4 x i32>, ptr addrspace(7) %q, align 8
+ ret <4 x i32> %ret
+}
+
+define void @store_v4i32_align8(<4 x i32> %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_v4i32_align8(
+; STRICT-SAME: <4 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <4 x i32> [[DATA]], <4 x i32> poison, <2 x i32> <i32 0, i32 1>
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
+; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = shufflevector <4 x i32> [[DATA]], <4 x i32> poison, <2 x i32> <i32 2, i32 3>
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_v4i32_align8(
+; UNALIGNED_ONLY-SAME: <4 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <4 x i32> [[DATA]], <4 x i32> poison, <2 x i32> <i32 0, i32 1>
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = shufflevector <4 x i32> [[DATA]], <4 x i32> poison, <2 x i32> <i32 2, i32 3>
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_v4i32_align8(
+; RELAXED_OOB_ONLY-SAME: <4 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_v4i32_align8(
+; BOTH_FLAGS-SAME: <4 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store <4 x i32> %data, ptr addrspace(7) %q, align 8
+ ret void
+}
+
+define <8 x i32> @load_v8i32_align16(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define <8 x i32> @load_v8i32_align16(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; STRICT-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; STRICT-NEXT: [[Q_OFF_PTR_16:%.*]] = add i32 [[Q]], 16
+; STRICT-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; STRICT-NEXT: [[RET:%.*]] = shufflevector <8 x i32> [[RET_PARTS_0]], <8 x i32> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; STRICT-NEXT: ret <8 x i32> [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define <8 x i32> @load_v8i32_align16(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; UNALIGNED_ONLY-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_16:%.*]] = add i32 [[Q]], 16
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = shufflevector <8 x i32> [[RET_PARTS_0]], <8 x i32> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; UNALIGNED_ONLY-NEXT: ret <8 x i32> [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define <8 x i32> @load_v8i32_align16(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; RELAXED_OOB_ONLY-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_16:%.*]] = add nuw i32 [[Q]], 16
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = shufflevector <8 x i32> [[RET_PARTS_0]], <8 x i32> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; RELAXED_OOB_ONLY-NEXT: ret <8 x i32> [[RET]]
+;
+; BOTH_FLAGS-LABEL: define <8 x i32> @load_v8i32_align16(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; BOTH_FLAGS-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; BOTH_FLAGS-NEXT: [[Q_OFF_PTR_16:%.*]] = add nuw i32 [[Q]], 16
+; BOTH_FLAGS-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = shufflevector <8 x i32> [[RET_PARTS_0]], <8 x i32> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; BOTH_FLAGS-NEXT: ret <8 x i32> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load <8 x i32>, ptr addrspace(7) %q, align 16
+ ret <8 x i32> %ret
+}
+
+define void @store_v8i32_align16(<8 x i32> %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_v8i32_align16(
+; STRICT-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
+; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_v8i32_align16(
+; UNALIGNED_ONLY-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_v8i32_align16(
+; RELAXED_OOB_ONLY-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_v8i32_align16(
+; BOTH_FLAGS-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
+; BOTH_FLAGS-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store <8 x i32> %data, ptr addrspace(7) %q, align 16
+ ret void
+}
+
+define <8 x i32> @load_v8i32_align8(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define <8 x i32> @load_v8i32_align8(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_EXT_0:%.*]] = shufflevector <2 x i32> [[RET_OFF_0]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; STRICT-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; STRICT-NEXT: [[Q_OFF_PTR_8:%.*]] = add i32 [[Q]], 8
+; STRICT-NEXT: [[RET_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_EXT_2:%.*]] = shufflevector <2 x i32> [[RET_OFF_8]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; STRICT-NEXT: [[RET_PARTS_2:%.*]] = shufflevector <8 x i32> [[RET_PARTS_0]], <8 x i32> [[RET_EXT_2]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; STRICT-NEXT: [[Q_OFF_PTR_16:%.*]] = add i32 [[Q]], 16
+; STRICT-NEXT: [[RET_OFF_16:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_EXT_4:%.*]] = shufflevector <2 x i32> [[RET_OFF_16]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; STRICT-NEXT: [[RET_PARTS_4:%.*]] = shufflevector <8 x i32> [[RET_PARTS_2]], <8 x i32> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
+; STRICT-NEXT: [[Q_OFF_PTR_24:%.*]] = add i32 [[Q]], 24
+; STRICT-NEXT: [[RET_OFF_24:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_24]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_EXT_6:%.*]] = shufflevector <2 x i32> [[RET_OFF_24]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; STRICT-NEXT: [[RET:%.*]] = shufflevector <8 x i32> [[RET_PARTS_4]], <8 x i32> [[RET_EXT_6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+; STRICT-NEXT: ret <8 x i32> [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define <8 x i32> @load_v8i32_align8(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_EXT_0:%.*]] = shufflevector <2 x i32> [[RET_OFF_0]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; UNALIGNED_ONLY-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_8:%.*]] = add i32 [[Q]], 8
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_EXT_2:%.*]] = shufflevector <2 x i32> [[RET_OFF_8]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; UNALIGNED_ONLY-NEXT: [[RET_PARTS_2:%.*]] = shufflevector <8 x i32> [[RET_PARTS_0]], <8 x i32> [[RET_EXT_2]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_16:%.*]] = add i32 [[Q]], 16
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_EXT_4:%.*]] = shufflevector <2 x i32> [[RET_OFF_16]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; UNALIGNED_ONLY-NEXT: [[RET_PARTS_4:%.*]] = shufflevector <8 x i32> [[RET_PARTS_2]], <8 x i32> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_24:%.*]] = add i32 [[Q]], 24
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_24:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_24]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_EXT_6:%.*]] = shufflevector <2 x i32> [[RET_OFF_24]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = shufflevector <8 x i32> [[RET_PARTS_4]], <8 x i32> [[RET_EXT_6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+; UNALIGNED_ONLY-NEXT: ret <8 x i32> [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define <8 x i32> @load_v8i32_align8(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; RELAXED_OOB_ONLY-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_16:%.*]] = add nuw i32 [[Q]], 16
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = shufflevector <8 x i32> [[RET_PARTS_0]], <8 x i32> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; RELAXED_OOB_ONLY-NEXT: ret <8 x i32> [[RET]]
+;
+; BOTH_FLAGS-LABEL: define <8 x i32> @load_v8i32_align8(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; BOTH_FLAGS-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; BOTH_FLAGS-NEXT: [[Q_OFF_PTR_16:%.*]] = add nuw i32 [[Q]], 16
+; BOTH_FLAGS-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = shufflevector <8 x i32> [[RET_PARTS_0]], <8 x i32> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; BOTH_FLAGS-NEXT: ret <8 x i32> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load <8 x i32>, ptr addrspace(7) %q, align 8
+ ret <8 x i32> %ret
+}
+
+define void @store_v8i32_align8(<8 x i32> %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_v8i32_align8(
+; STRICT-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <2 x i32> <i32 0, i32 1>
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
+; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <2 x i32> <i32 2, i32 3>
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
+; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <2 x i32> <i32 4, i32 5>
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 24
+; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <2 x i32> <i32 6, i32 7>
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_6]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_v8i32_align8(
+; UNALIGNED_ONLY-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <2 x i32> <i32 0, i32 1>
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <2 x i32> <i32 2, i32 3>
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <2 x i32> <i32 4, i32 5>
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 24
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <2 x i32> <i32 6, i32 7>
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_6]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_v8i32_align8(
+; RELAXED_OOB_ONLY-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_v8i32_align8(
+; BOTH_FLAGS-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
+; BOTH_FLAGS-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store <8 x i32> %data, ptr addrspace(7) %q, align 8
+ ret void
+}
+
+define <8 x i32> @load_v8i32_align4(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define <8 x i32> @load_v8i32_align4(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i32> poison, i32 [[RET_OFF_0]], i64 0
+; STRICT-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
+; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i32> [[RET_SLICE_0]], i32 [[RET_OFF_4]], i64 1
+; STRICT-NEXT: [[Q_OFF_PTR_8:%.*]] = add i32 [[Q]], 8
+; STRICT-NEXT: [[RET_OFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i32> [[RET_SLICE_1]], i32 [[RET_OFF_8]], i64 2
+; STRICT-NEXT: [[Q_OFF_PTR_12:%.*]] = add i32 [[Q]], 12
+; STRICT-NEXT: [[RET_OFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_12]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i32> [[RET_SLICE_2]], i32 [[RET_OFF_12]], i64 3
+; STRICT-NEXT: [[Q_OFF_PTR_16:%.*]] = add i32 [[Q]], 16
+; STRICT-NEXT: [[RET_OFF_16:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i32> [[RET_SLICE_3]], i32 [[RET_OFF_16]], i64 4
+; STRICT-NEXT: [[Q_OFF_PTR_20:%.*]] = add i32 [[Q]], 20
+; STRICT-NEXT: [[RET_OFF_20:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_20]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i32> [[RET_SLICE_4]], i32 [[RET_OFF_20]], i64 5
+; STRICT-NEXT: [[Q_OFF_PTR_24:%.*]] = add i32 [[Q]], 24
+; STRICT-NEXT: [[RET_OFF_24:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_24]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i32> [[RET_SLICE_5]], i32 [[RET_OFF_24]], i64 6
+; STRICT-NEXT: [[Q_OFF_PTR_28:%.*]] = add i32 [[Q]], 28
+; STRICT-NEXT: [[RET_OFF_28:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_28]], i32 0, i32 0)
+; STRICT-NEXT: [[RET:%.*]] = insertelement <8 x i32> [[RET_SLICE_6]], i32 [[RET_OFF_28]], i64 7
+; STRICT-NEXT: ret <8 x i32> [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define <8 x i32> @load_v8i32_align4(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i32> poison, i32 [[RET_OFF_0]], i64 0
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i32> [[RET_SLICE_0]], i32 [[RET_OFF_4]], i64 1
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_8:%.*]] = add i32 [[Q]], 8
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i32> [[RET_SLICE_1]], i32 [[RET_OFF_8]], i64 2
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_12:%.*]] = add i32 [[Q]], 12
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_12]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i32> [[RET_SLICE_2]], i32 [[RET_OFF_12]], i64 3
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_16:%.*]] = add i32 [[Q]], 16
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_16:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i32> [[RET_SLICE_3]], i32 [[RET_OFF_16]], i64 4
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_20:%.*]] = add i32 [[Q]], 20
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_20:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_20]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i32> [[RET_SLICE_4]], i32 [[RET_OFF_20]], i64 5
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_24:%.*]] = add i32 [[Q]], 24
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_24:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_24]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i32> [[RET_SLICE_5]], i32 [[RET_OFF_24]], i64 6
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_28:%.*]] = add i32 [[Q]], 28
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_28:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_28]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <8 x i32> [[RET_SLICE_6]], i32 [[RET_OFF_28]], i64 7
+; UNALIGNED_ONLY-NEXT: ret <8 x i32> [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define <8 x i32> @load_v8i32_align4(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; RELAXED_OOB_ONLY-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_16:%.*]] = add nuw i32 [[Q]], 16
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = shufflevector <8 x i32> [[RET_PARTS_0]], <8 x i32> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; RELAXED_OOB_ONLY-NEXT: ret <8 x i32> [[RET]]
+;
+; BOTH_FLAGS-LABEL: define <8 x i32> @load_v8i32_align4(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; BOTH_FLAGS-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; BOTH_FLAGS-NEXT: [[Q_OFF_PTR_16:%.*]] = add nuw i32 [[Q]], 16
+; BOTH_FLAGS-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = shufflevector <8 x i32> [[RET_PARTS_0]], <8 x i32> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; BOTH_FLAGS-NEXT: ret <8 x i32> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load <8 x i32>, ptr addrspace(7) %q, align 4
+ ret <8 x i32> %ret
+}
+
+define void @store_v8i32_align4(<8 x i32> %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_v8i32_align4(
+; STRICT-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i32> [[DATA]], i64 0
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 4
+; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i32> [[DATA]], i64 1
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
+; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i32> [[DATA]], i64 2
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 12
+; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i32> [[DATA]], i64 3
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_3]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
+; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i32> [[DATA]], i64 4
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 20
+; STRICT-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i32> [[DATA]], i64 5
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_5]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 24
+; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i32> [[DATA]], i64 6
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_6]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 28
+; STRICT-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i32> [[DATA]], i64 7
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_7]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_v8i32_align4(
+; UNALIGNED_ONLY-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i32> [[DATA]], i64 0
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 4
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i32> [[DATA]], i64 1
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_1]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i32> [[DATA]], i64 2
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 12
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i32> [[DATA]], i64 3
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_3]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i32> [[DATA]], i64 4
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 20
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i32> [[DATA]], i64 5
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_5]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 24
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i32> [[DATA]], i64 6
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_6]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 28
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i32> [[DATA]], i64 7
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_7]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_v8i32_align4(
+; RELAXED_OOB_ONLY-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_v8i32_align4(
+; BOTH_FLAGS-SAME: <8 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
+; BOTH_FLAGS-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store <8 x i32> %data, ptr addrspace(7) %q, align 4
+ ret void
+}
+
+define <10 x i32> @load_v10i32_align16(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define <10 x i32> @load_v10i32_align16(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; STRICT-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <10 x i32> poison, <10 x i32> [[RET_EXT_0]], <10 x i32> <i32 10, i32 11, i32 12, i32 13, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
+; STRICT-NEXT: [[Q_OFF_PTR_16:%.*]] = add i32 [[Q]], 16
+; STRICT-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; STRICT-NEXT: [[RET_PARTS_4:%.*]] = shufflevector <10 x i32> [[RET_PARTS_0]], <10 x i32> [[RET_EXT_4]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 10, i32 11, i32 12, i32 13, i32 8, i32 9>
+; STRICT-NEXT: [[Q_OFF_PTR_32:%.*]] = add i32 [[Q]], 32
+; STRICT-NEXT: [[RET_OFF_32:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_32]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_EXT_8:%.*]] = shufflevector <2 x i32> [[RET_OFF_32]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; STRICT-NEXT: [[RET:%.*]] = shufflevector <10 x i32> [[RET_PARTS_4]], <10 x i32> [[RET_EXT_8]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 10, i32 11>
+; STRICT-NEXT: ret <10 x i32> [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define <10 x i32> @load_v10i32_align16(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; UNALIGNED_ONLY-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <10 x i32> poison, <10 x i32> [[RET_EXT_0]], <10 x i32> <i32 10, i32 11, i32 12, i32 13, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_16:%.*]] = add i32 [[Q]], 16
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; UNALIGNED_ONLY-NEXT: [[RET_PARTS_4:%.*]] = shufflevector <10 x i32> [[RET_PARTS_0]], <10 x i32> [[RET_EXT_4]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 10, i32 11, i32 12, i32 13, i32 8, i32 9>
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_32:%.*]] = add i32 [[Q]], 32
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_32:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_32]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_EXT_8:%.*]] = shufflevector <2 x i32> [[RET_OFF_32]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = shufflevector <10 x i32> [[RET_PARTS_4]], <10 x i32> [[RET_EXT_8]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 10, i32 11>
+; UNALIGNED_ONLY-NEXT: ret <10 x i32> [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define <10 x i32> @load_v10i32_align16(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; RELAXED_OOB_ONLY-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <10 x i32> poison, <10 x i32> [[RET_EXT_0]], <10 x i32> <i32 10, i32 11, i32 12, i32 13, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_16:%.*]] = add nuw i32 [[Q]], 16
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; RELAXED_OOB_ONLY-NEXT: [[RET_PARTS_4:%.*]] = shufflevector <10 x i32> [[RET_PARTS_0]], <10 x i32> [[RET_EXT_4]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 10, i32 11, i32 12, i32 13, i32 8, i32 9>
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_32:%.*]] = add nuw i32 [[Q]], 32
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_32:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_32]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_EXT_8:%.*]] = shufflevector <2 x i32> [[RET_OFF_32]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = shufflevector <10 x i32> [[RET_PARTS_4]], <10 x i32> [[RET_EXT_8]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 10, i32 11>
+; RELAXED_OOB_ONLY-NEXT: ret <10 x i32> [[RET]]
+;
+; BOTH_FLAGS-LABEL: define <10 x i32> @load_v10i32_align16(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; BOTH_FLAGS-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <10 x i32> poison, <10 x i32> [[RET_EXT_0]], <10 x i32> <i32 10, i32 11, i32 12, i32 13, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
+; BOTH_FLAGS-NEXT: [[Q_OFF_PTR_16:%.*]] = add nuw i32 [[Q]], 16
+; BOTH_FLAGS-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; BOTH_FLAGS-NEXT: [[RET_PARTS_4:%.*]] = shufflevector <10 x i32> [[RET_PARTS_0]], <10 x i32> [[RET_EXT_4]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 10, i32 11, i32 12, i32 13, i32 8, i32 9>
+; BOTH_FLAGS-NEXT: [[Q_OFF_PTR_32:%.*]] = add nuw i32 [[Q]], 32
+; BOTH_FLAGS-NEXT: [[RET_OFF_32:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 16 [[BUF]], i32 [[Q_OFF_PTR_32]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_EXT_8:%.*]] = shufflevector <2 x i32> [[RET_OFF_32]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = shufflevector <10 x i32> [[RET_PARTS_4]], <10 x i32> [[RET_EXT_8]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 10, i32 11>
+; BOTH_FLAGS-NEXT: ret <10 x i32> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load <10 x i32>, ptr addrspace(7) %q, align 16
+ ret <10 x i32> %ret
+}
+
+define void @store_v10i32_align16(<10 x i32> %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_v10i32_align16(
+; STRICT-SAME: <10 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
+; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_8:%.*]] = add i32 [[Q]], 32
+; STRICT-NEXT: [[DATA_SLICE_8:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 8, i32 9>
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_v10i32_align16(
+; UNALIGNED_ONLY-SAME: <10 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_8:%.*]] = add i32 [[Q]], 32
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_8:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 8, i32 9>
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_v10i32_align16(
+; RELAXED_OOB_ONLY-SAME: <10 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_8:%.*]] = add nuw i32 [[Q]], 32
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_8:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 8, i32 9>
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_v10i32_align16(
+; BOTH_FLAGS-SAME: <10 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
+; BOTH_FLAGS-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[Q_PART_8:%.*]] = add nuw i32 [[Q]], 32
+; BOTH_FLAGS-NEXT: [[DATA_SLICE_8:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 8, i32 9>
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 16 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store <10 x i32> %data, ptr addrspace(7) %q, align 16
+ ret void
+}
+
+define <10 x i32> @load_v10i32_align8(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define <10 x i32> @load_v10i32_align8(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_EXT_0:%.*]] = shufflevector <2 x i32> [[RET_OFF_0]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; STRICT-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <10 x i32> poison, <10 x i32> [[RET_EXT_0]], <10 x i32> <i32 10, i32 11, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
+; STRICT-NEXT: [[Q_OFF_PTR_8:%.*]] = add i32 [[Q]], 8
+; STRICT-NEXT: [[RET_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_EXT_2:%.*]] = shufflevector <2 x i32> [[RET_OFF_8]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; STRICT-NEXT: [[RET_PARTS_2:%.*]] = shufflevector <10 x i32> [[RET_PARTS_0]], <10 x i32> [[RET_EXT_2]], <10 x i32> <i32 0, i32 1, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
+; STRICT-NEXT: [[Q_OFF_PTR_16:%.*]] = add i32 [[Q]], 16
+; STRICT-NEXT: [[RET_OFF_16:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_EXT_4:%.*]] = shufflevector <2 x i32> [[RET_OFF_16]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; STRICT-NEXT: [[RET_PARTS_4:%.*]] = shufflevector <10 x i32> [[RET_PARTS_2]], <10 x i32> [[RET_EXT_4]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 10, i32 11, i32 6, i32 7, i32 8, i32 9>
+; STRICT-NEXT: [[Q_OFF_PTR_24:%.*]] = add i32 [[Q]], 24
+; STRICT-NEXT: [[RET_OFF_24:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_24]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_EXT_6:%.*]] = shufflevector <2 x i32> [[RET_OFF_24]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; STRICT-NEXT: [[RET_PARTS_6:%.*]] = shufflevector <10 x i32> [[RET_PARTS_4]], <10 x i32> [[RET_EXT_6]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 10, i32 11, i32 8, i32 9>
+; STRICT-NEXT: [[Q_OFF_PTR_32:%.*]] = add i32 [[Q]], 32
+; STRICT-NEXT: [[RET_OFF_32:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_32]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_EXT_8:%.*]] = shufflevector <2 x i32> [[RET_OFF_32]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; STRICT-NEXT: [[RET:%.*]] = shufflevector <10 x i32> [[RET_PARTS_6]], <10 x i32> [[RET_EXT_8]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 10, i32 11>
+; STRICT-NEXT: ret <10 x i32> [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define <10 x i32> @load_v10i32_align8(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_EXT_0:%.*]] = shufflevector <2 x i32> [[RET_OFF_0]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; UNALIGNED_ONLY-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <10 x i32> poison, <10 x i32> [[RET_EXT_0]], <10 x i32> <i32 10, i32 11, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_8:%.*]] = add i32 [[Q]], 8
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_EXT_2:%.*]] = shufflevector <2 x i32> [[RET_OFF_8]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; UNALIGNED_ONLY-NEXT: [[RET_PARTS_2:%.*]] = shufflevector <10 x i32> [[RET_PARTS_0]], <10 x i32> [[RET_EXT_2]], <10 x i32> <i32 0, i32 1, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_16:%.*]] = add i32 [[Q]], 16
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_EXT_4:%.*]] = shufflevector <2 x i32> [[RET_OFF_16]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; UNALIGNED_ONLY-NEXT: [[RET_PARTS_4:%.*]] = shufflevector <10 x i32> [[RET_PARTS_2]], <10 x i32> [[RET_EXT_4]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 10, i32 11, i32 6, i32 7, i32 8, i32 9>
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_24:%.*]] = add i32 [[Q]], 24
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_24:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_24]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_EXT_6:%.*]] = shufflevector <2 x i32> [[RET_OFF_24]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; UNALIGNED_ONLY-NEXT: [[RET_PARTS_6:%.*]] = shufflevector <10 x i32> [[RET_PARTS_4]], <10 x i32> [[RET_EXT_6]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 10, i32 11, i32 8, i32 9>
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_32:%.*]] = add i32 [[Q]], 32
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_32:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_32]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_EXT_8:%.*]] = shufflevector <2 x i32> [[RET_OFF_32]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = shufflevector <10 x i32> [[RET_PARTS_6]], <10 x i32> [[RET_EXT_8]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 10, i32 11>
+; UNALIGNED_ONLY-NEXT: ret <10 x i32> [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define <10 x i32> @load_v10i32_align8(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; RELAXED_OOB_ONLY-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <10 x i32> poison, <10 x i32> [[RET_EXT_0]], <10 x i32> <i32 10, i32 11, i32 12, i32 13, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_16:%.*]] = add nuw i32 [[Q]], 16
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; RELAXED_OOB_ONLY-NEXT: [[RET_PARTS_4:%.*]] = shufflevector <10 x i32> [[RET_PARTS_0]], <10 x i32> [[RET_EXT_4]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 10, i32 11, i32 12, i32 13, i32 8, i32 9>
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_32:%.*]] = add nuw i32 [[Q]], 32
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_32:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_32]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_EXT_8:%.*]] = shufflevector <2 x i32> [[RET_OFF_32]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = shufflevector <10 x i32> [[RET_PARTS_4]], <10 x i32> [[RET_EXT_8]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 10, i32 11>
+; RELAXED_OOB_ONLY-NEXT: ret <10 x i32> [[RET]]
+;
+; BOTH_FLAGS-LABEL: define <10 x i32> @load_v10i32_align8(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; BOTH_FLAGS-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <10 x i32> poison, <10 x i32> [[RET_EXT_0]], <10 x i32> <i32 10, i32 11, i32 12, i32 13, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
+; BOTH_FLAGS-NEXT: [[Q_OFF_PTR_16:%.*]] = add nuw i32 [[Q]], 16
+; BOTH_FLAGS-NEXT: [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_16]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; BOTH_FLAGS-NEXT: [[RET_PARTS_4:%.*]] = shufflevector <10 x i32> [[RET_PARTS_0]], <10 x i32> [[RET_EXT_4]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 10, i32 11, i32 12, i32 13, i32 8, i32 9>
+; BOTH_FLAGS-NEXT: [[Q_OFF_PTR_32:%.*]] = add nuw i32 [[Q]], 32
+; BOTH_FLAGS-NEXT: [[RET_OFF_32:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q_OFF_PTR_32]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET_EXT_8:%.*]] = shufflevector <2 x i32> [[RET_OFF_32]], <2 x i32> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = shufflevector <10 x i32> [[RET_PARTS_4]], <10 x i32> [[RET_EXT_8]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 10, i32 11>
+; BOTH_FLAGS-NEXT: ret <10 x i32> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load <10 x i32>, ptr addrspace(7) %q, align 8
+ ret <10 x i32> %ret
+}
+
+define void @store_v10i32_align8(<10 x i32> %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_v10i32_align8(
+; STRICT-SAME: <10 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 0, i32 1>
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
+; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 2, i32 3>
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
+; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 4, i32 5>
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 24
+; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 6, i32 7>
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_6]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_8:%.*]] = add i32 [[Q]], 32
+; STRICT-NEXT: [[DATA_SLICE_8:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 8, i32 9>
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_v10i32_align8(
+; UNALIGNED_ONLY-SAME: <10 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 0, i32 1>
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 8
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 2, i32 3>
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_2]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 16
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 4, i32 5>
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 24
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 6, i32 7>
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_6]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_8:%.*]] = add i32 [[Q]], 32
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_8:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 8, i32 9>
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_v10i32_align8(
+; RELAXED_OOB_ONLY-SAME: <10 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_8:%.*]] = add nuw i32 [[Q]], 32
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_8:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 8, i32 9>
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_v10i32_align8(
+; BOTH_FLAGS-SAME: <10 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 16
+; BOTH_FLAGS-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[Q_PART_8:%.*]] = add nuw i32 [[Q]], 32
+; BOTH_FLAGS-NEXT: [[DATA_SLICE_8:%.*]] = shufflevector <10 x i32> [[DATA]], <10 x i32> poison, <2 x i32> <i32 8, i32 9>
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_8]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q_PART_8]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store <10 x i32> %data, ptr addrspace(7) %q, align 8
+ ret void
+}
+
+define i16 @load_i16_align1(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define i16 @load_i16_align1(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <2 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; STRICT-NEXT: [[RET:%.*]] = bitcast <2 x i8> [[RET_SLICE_1]] to i16
+; STRICT-NEXT: ret i16 [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define i16 @load_i16_align1(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret i16 [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define i16 @load_i16_align1(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add nuw i32 [[Q]], 1
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <2 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast <2 x i8> [[RET_SLICE_1]] to i16
+; RELAXED_OOB_ONLY-NEXT: ret i16 [[RET]]
+;
+; BOTH_FLAGS-LABEL: define i16 @load_i16_align1(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret i16 [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load i16, ptr addrspace(7) %q, align 1
+ ret i16 %ret
+}
+
+define void @store_i16_align1(i16 %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_i16_align1(
+; STRICT-SAME: i16 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast i16 [[DATA]] to <2 x i8>
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i8> [[DATA_LEGAL]], i64 0
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i8> [[DATA_LEGAL]], i64 1
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_i16_align1(
+; UNALIGNED_ONLY-SAME: i16 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_i16_align1(
+; RELAXED_OOB_ONLY-SAME: i16 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast i16 [[DATA]] to <2 x i8>
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i8> [[DATA_LEGAL]], i64 0
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 1
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i8> [[DATA_LEGAL]], i64 1
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_i16_align1(
+; BOTH_FLAGS-SAME: i16 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store i16 %data, ptr addrspace(7) %q, align 1
+ ret void
+}
+
+define <2 x i16> @load_v2i16_align4(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define <2 x i16> @load_v2i16_align4(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i16> poison, i16 [[RET_OFF_0]], i64 0
+; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET:%.*]] = insertelement <2 x i16> [[RET_SLICE_0]], i16 [[RET_OFF_2]], i64 1
+; STRICT-NEXT: ret <2 x i16> [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define <2 x i16> @load_v2i16_align4(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i16> poison, i16 [[RET_OFF_0]], i64 0
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <2 x i16> [[RET_SLICE_0]], i16 [[RET_OFF_2]], i64 1
+; UNALIGNED_ONLY-NEXT: ret <2 x i16> [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define <2 x i16> @load_v2i16_align4(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret <2 x i16> [[RET]]
+;
+; BOTH_FLAGS-LABEL: define <2 x i16> @load_v2i16_align4(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret <2 x i16> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load <2 x i16>, ptr addrspace(7) %q, align 4
+ ret <2 x i16> %ret
+}
+
+define void @store_v2i16_align4(<2 x i16> %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_v2i16_align4(
+; STRICT-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA]], i64 0
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA]], i64 1
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_v2i16_align4(
+; UNALIGNED_ONLY-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA]], i64 0
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 2
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA]], i64 1
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_v2i16_align4(
+; RELAXED_OOB_ONLY-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_v2i16_align4(
+; BOTH_FLAGS-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store <2 x i16> %data, ptr addrspace(7) %q, align 4
+ ret void
+}
+
+define <2 x i16> @load_v2i16_align2(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define <2 x i16> @load_v2i16_align2(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i16> poison, i16 [[RET_OFF_0]], i64 0
+; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET:%.*]] = insertelement <2 x i16> [[RET_SLICE_0]], i16 [[RET_OFF_2]], i64 1
+; STRICT-NEXT: ret <2 x i16> [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define <2 x i16> @load_v2i16_align2(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i16> poison, i16 [[RET_OFF_0]], i64 0
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <2 x i16> [[RET_SLICE_0]], i16 [[RET_OFF_2]], i64 1
+; UNALIGNED_ONLY-NEXT: ret <2 x i16> [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define <2 x i16> @load_v2i16_align2(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i16> poison, i16 [[RET_OFF_0]], i64 0
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add nuw i32 [[Q]], 2
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = insertelement <2 x i16> [[RET_SLICE_0]], i16 [[RET_OFF_2]], i64 1
+; RELAXED_OOB_ONLY-NEXT: ret <2 x i16> [[RET]]
+;
+; BOTH_FLAGS-LABEL: define <2 x i16> @load_v2i16_align2(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret <2 x i16> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load <2 x i16>, ptr addrspace(7) %q, align 2
+ ret <2 x i16> %ret
+}
+
+define void @store_v2i16_align2(<2 x i16> %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_v2i16_align2(
+; STRICT-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA]], i64 0
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA]], i64 1
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_v2i16_align2(
+; UNALIGNED_ONLY-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA]], i64 0
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 2
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA]], i64 1
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_v2i16_align2(
+; RELAXED_OOB_ONLY-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA]], i64 0
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 2
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA]], i64 1
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_v2i16_align2(
+; BOTH_FLAGS-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store <2 x i16> %data, ptr addrspace(7) %q, align 2
+ ret void
+}
+
+define <2 x i16> @load_v2i16_align1(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define <2 x i16> @load_v2i16_align1(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
+; STRICT-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
+; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_3:%.*]] = insertelement <4 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
+; STRICT-NEXT: [[RET:%.*]] = bitcast <4 x i8> [[RET_SLICE_3]] to <2 x i16>
+; STRICT-NEXT: ret <2 x i16> [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define <2 x i16> @load_v2i16_align1(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i16> poison, i16 [[RET_OFF_0]], i64 0
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <2 x i16> [[RET_SLICE_0]], i16 [[RET_OFF_2]], i64 1
+; UNALIGNED_ONLY-NEXT: ret <2 x i16> [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define <2 x i16> @load_v2i16_align1(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add nuw i32 [[Q]], 1
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add nuw i32 [[Q]], 2
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_3:%.*]] = add nuw i32 [[Q]], 3
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_3:%.*]] = insertelement <4 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast <4 x i8> [[RET_SLICE_3]] to <2 x i16>
+; RELAXED_OOB_ONLY-NEXT: ret <2 x i16> [[RET]]
+;
+; BOTH_FLAGS-LABEL: define <2 x i16> @load_v2i16_align1(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret <2 x i16> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load <2 x i16>, ptr addrspace(7) %q, align 1
+ ret <2 x i16> %ret
+}
+
+define void @store_v2i16_align1(<2 x i16> %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_v2i16_align1(
+; STRICT-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i16> [[DATA]] to <4 x i8>
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 0
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 1
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 2
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
+; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 3
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_v2i16_align1(
+; UNALIGNED_ONLY-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA]], i64 0
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 2
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA]], i64 1
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_v2i16_align1(
+; RELAXED_OOB_ONLY-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i16> [[DATA]] to <4 x i8>
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 0
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 1
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 1
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_2:%.*]] = add nuw i32 [[Q]], 2
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 2
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_3:%.*]] = add nuw i32 [[Q]], 3
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA_LEGAL]], i64 3
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_v2i16_align1(
+; BOTH_FLAGS-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store <2 x i16> %data, ptr addrspace(7) %q, align 1
+ ret void
+}
+
+define <2 x i8> @load_v2i8_align2(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define <2 x i8> @load_v2i8_align2(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET:%.*]] = insertelement <2 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; STRICT-NEXT: ret <2 x i8> [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define <2 x i8> @load_v2i8_align2(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <2 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; UNALIGNED_ONLY-NEXT: ret <2 x i8> [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define <2 x i8> @load_v2i8_align2(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET_LOADABLE:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast i16 [[RET_LOADABLE]] to <2 x i8>
+; RELAXED_OOB_ONLY-NEXT: ret <2 x i8> [[RET]]
+;
+; BOTH_FLAGS-LABEL: define <2 x i8> @load_v2i8_align2(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = bitcast i16 [[RET_LOADABLE]] to <2 x i8>
+; BOTH_FLAGS-NEXT: ret <2 x i8> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load <2 x i8>, ptr addrspace(7) %q, align 2
+ ret <2 x i8> %ret
+}
+
+define void @store_v2i8_align2(<2 x i8> %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_v2i8_align2(
+; STRICT-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i8> [[DATA]], i64 0
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i8> [[DATA]], i64 1
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_v2i8_align2(
+; UNALIGNED_ONLY-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i8> [[DATA]], i64 0
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i8> [[DATA]], i64 1
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_v2i8_align2(
+; RELAXED_OOB_ONLY-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i8> [[DATA]] to i16
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_v2i8_align2(
+; BOTH_FLAGS-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i8> [[DATA]] to i16
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store <2 x i8> %data, ptr addrspace(7) %q, align 2
+ ret void
+}
+
+define <2 x i8> @load_v2i8_align1(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define <2 x i8> @load_v2i8_align1(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET:%.*]] = insertelement <2 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; STRICT-NEXT: ret <2 x i8> [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define <2 x i8> @load_v2i8_align1(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <2 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; UNALIGNED_ONLY-NEXT: ret <2 x i8> [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define <2 x i8> @load_v2i8_align1(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add nuw i32 [[Q]], 1
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = insertelement <2 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; RELAXED_OOB_ONLY-NEXT: ret <2 x i8> [[RET]]
+;
+; BOTH_FLAGS-LABEL: define <2 x i8> @load_v2i8_align1(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = bitcast i16 [[RET_LOADABLE]] to <2 x i8>
+; BOTH_FLAGS-NEXT: ret <2 x i8> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load <2 x i8>, ptr addrspace(7) %q, align 1
+ ret <2 x i8> %ret
+}
+
+define void @store_v2i8_align1(<2 x i8> %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_v2i8_align1(
+; STRICT-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i8> [[DATA]], i64 0
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i8> [[DATA]], i64 1
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_v2i8_align1(
+; UNALIGNED_ONLY-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i8> [[DATA]], i64 0
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i8> [[DATA]], i64 1
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_v2i8_align1(
+; RELAXED_OOB_ONLY-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i8> [[DATA]], i64 0
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 1
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i8> [[DATA]], i64 1
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_v2i8_align1(
+; BOTH_FLAGS-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <2 x i8> [[DATA]] to i16
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store <2 x i8> %data, ptr addrspace(7) %q, align 1
+ ret void
+}
+
+define <4 x i8> @load_v4i8_align4(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define <4 x i8> @load_v4i8_align4(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
+; STRICT-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
+; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; STRICT-NEXT: [[RET:%.*]] = insertelement <4 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
+; STRICT-NEXT: ret <4 x i8> [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define <4 x i8> @load_v4i8_align4(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <4 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
+; UNALIGNED_ONLY-NEXT: ret <4 x i8> [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define <4 x i8> @load_v4i8_align4(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast i32 [[RET_LOADABLE]] to <4 x i8>
+; RELAXED_OOB_ONLY-NEXT: ret <4 x i8> [[RET]]
+;
+; BOTH_FLAGS-LABEL: define <4 x i8> @load_v4i8_align4(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = bitcast i32 [[RET_LOADABLE]] to <4 x i8>
+; BOTH_FLAGS-NEXT: ret <4 x i8> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load <4 x i8>, ptr addrspace(7) %q, align 4
+ ret <4 x i8> %ret
+}
+
+define void @store_v4i8_align4(<4 x i8> %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_v4i8_align4(
+; STRICT-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA]], i64 0
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA]], i64 1
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA]], i64 2
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
+; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA]], i64 3
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_v4i8_align4(
+; UNALIGNED_ONLY-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA]], i64 0
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA]], i64 1
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA]], i64 2
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA]], i64 3
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_v4i8_align4(
+; RELAXED_OOB_ONLY-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <4 x i8> [[DATA]] to i32
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_v4i8_align4(
+; BOTH_FLAGS-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <4 x i8> [[DATA]] to i32
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store <4 x i8> %data, ptr addrspace(7) %q, align 4
+ ret void
+}
+
+define <4 x i8> @load_v4i8_align2(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define <4 x i8> @load_v4i8_align2(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
+; STRICT-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
+; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; STRICT-NEXT: [[RET:%.*]] = insertelement <4 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
+; STRICT-NEXT: ret <4 x i8> [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define <4 x i8> @load_v4i8_align2(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <4 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
+; UNALIGNED_ONLY-NEXT: ret <4 x i8> [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define <4 x i8> @load_v4i8_align2(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x i16> poison, i16 [[RET_OFF_0]], i64 0
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add nuw i32 [[Q]], 2
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <2 x i16> [[RET_SLICE_0]], i16 [[RET_OFF_2]], i64 1
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast <2 x i16> [[RET_SLICE_1]] to <4 x i8>
+; RELAXED_OOB_ONLY-NEXT: ret <4 x i8> [[RET]]
+;
+; BOTH_FLAGS-LABEL: define <4 x i8> @load_v4i8_align2(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = bitcast i32 [[RET_LOADABLE]] to <4 x i8>
+; BOTH_FLAGS-NEXT: ret <4 x i8> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load <4 x i8>, ptr addrspace(7) %q, align 2
+ ret <4 x i8> %ret
+}
+
+define void @store_v4i8_align2(<4 x i8> %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_v4i8_align2(
+; STRICT-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA]], i64 0
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA]], i64 1
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA]], i64 2
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
+; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA]], i64 3
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_v4i8_align2(
+; UNALIGNED_ONLY-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA]], i64 0
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA]], i64 1
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA]], i64 2
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA]], i64 3
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_v4i8_align2(
+; RELAXED_OOB_ONLY-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <4 x i8> [[DATA]] to <2 x i16>
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x i16> [[DATA_LEGAL]], i64 0
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 2
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x i16> [[DATA_LEGAL]], i64 1
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_v4i8_align2(
+; BOTH_FLAGS-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <4 x i8> [[DATA]] to i32
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store <4 x i8> %data, ptr addrspace(7) %q, align 2
+ ret void
+}
+
+define <4 x i8> @load_v4i8_align1(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define <4 x i8> @load_v4i8_align1(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
+; STRICT-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
+; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; STRICT-NEXT: [[RET:%.*]] = insertelement <4 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
+; STRICT-NEXT: ret <4 x i8> [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define <4 x i8> @load_v4i8_align1(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = insertelement <4 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
+; UNALIGNED_ONLY-NEXT: ret <4 x i8> [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define <4 x i8> @load_v4i8_align1(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add nuw i32 [[Q]], 1
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add nuw i32 [[Q]], 2
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_3:%.*]] = add nuw i32 [[Q]], 3
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = insertelement <4 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
+; RELAXED_OOB_ONLY-NEXT: ret <4 x i8> [[RET]]
+;
+; BOTH_FLAGS-LABEL: define <4 x i8> @load_v4i8_align1(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = bitcast i32 [[RET_LOADABLE]] to <4 x i8>
+; BOTH_FLAGS-NEXT: ret <4 x i8> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load <4 x i8>, ptr addrspace(7) %q, align 1
+ ret <4 x i8> %ret
+}
+
+define void @store_v4i8_align1(<4 x i8> %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_v4i8_align1(
+; STRICT-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA]], i64 0
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA]], i64 1
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA]], i64 2
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
+; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA]], i64 3
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_v4i8_align1(
+; UNALIGNED_ONLY-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA]], i64 0
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA]], i64 1
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA]], i64 2
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA]], i64 3
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_v4i8_align1(
+; RELAXED_OOB_ONLY-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i8> [[DATA]], i64 0
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 1
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i8> [[DATA]], i64 1
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_2:%.*]] = add nuw i32 [[Q]], 2
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i8> [[DATA]], i64 2
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_3:%.*]] = add nuw i32 [[Q]], 3
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i8> [[DATA]], i64 3
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_v4i8_align1(
+; BOTH_FLAGS-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <4 x i8> [[DATA]] to i32
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store <4 x i8> %data, ptr addrspace(7) %q, align 1
+ ret void
+}
+
+define <16 x i4> @load_v16i4_align8(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define <16 x i4> @load_v16i4_align8(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
+; STRICT-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
+; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
+; STRICT-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
+; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
+; STRICT-NEXT: [[Q_OFF_PTR_5:%.*]] = add i32 [[Q]], 5
+; STRICT-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
+; STRICT-NEXT: [[Q_OFF_PTR_6:%.*]] = add i32 [[Q]], 6
+; STRICT-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
+; STRICT-NEXT: [[Q_OFF_PTR_7:%.*]] = add i32 [[Q]], 7
+; STRICT-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_7:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
+; STRICT-NEXT: [[RET:%.*]] = bitcast <8 x i8> [[RET_SLICE_7]] to <16 x i4>
+; STRICT-NEXT: ret <16 x i4> [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define <16 x i4> @load_v16i4_align8(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_5:%.*]] = add i32 [[Q]], 5
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_6:%.*]] = add i32 [[Q]], 6
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_7:%.*]] = add i32 [[Q]], 7
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_7:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = bitcast <8 x i8> [[RET_SLICE_7]] to <16 x i4>
+; UNALIGNED_ONLY-NEXT: ret <16 x i4> [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define <16 x i4> @load_v16i4_align8(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast <2 x i32> [[RET_LOADABLE]] to <16 x i4>
+; RELAXED_OOB_ONLY-NEXT: ret <16 x i4> [[RET]]
+;
+; BOTH_FLAGS-LABEL: define <16 x i4> @load_v16i4_align8(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = bitcast <2 x i32> [[RET_LOADABLE]] to <16 x i4>
+; BOTH_FLAGS-NEXT: ret <16 x i4> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load <16 x i4>, ptr addrspace(7) %q, align 8
+ ret <16 x i4> %ret
+}
+
+define void @store_v16i4_align8(<16 x i4> %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_v16i4_align8(
+; STRICT-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
+; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
+; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
+; STRICT-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
+; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
+; STRICT-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_v16i4_align8(
+; UNALIGNED_ONLY-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_v16i4_align8(
+; RELAXED_OOB_ONLY-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <2 x i32>
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_v16i4_align8(
+; BOTH_FLAGS-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <2 x i32>
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store <16 x i4> %data, ptr addrspace(7) %q, align 8
+ ret void
+}
+
+define <16 x i4> @load_v16i4_align4(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define <16 x i4> @load_v16i4_align4(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
+; STRICT-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
+; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
+; STRICT-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
+; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
+; STRICT-NEXT: [[Q_OFF_PTR_5:%.*]] = add i32 [[Q]], 5
+; STRICT-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
+; STRICT-NEXT: [[Q_OFF_PTR_6:%.*]] = add i32 [[Q]], 6
+; STRICT-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
+; STRICT-NEXT: [[Q_OFF_PTR_7:%.*]] = add i32 [[Q]], 7
+; STRICT-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_7:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
+; STRICT-NEXT: [[RET:%.*]] = bitcast <8 x i8> [[RET_SLICE_7]] to <16 x i4>
+; STRICT-NEXT: ret <16 x i4> [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define <16 x i4> @load_v16i4_align4(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_5:%.*]] = add i32 [[Q]], 5
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_6:%.*]] = add i32 [[Q]], 6
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_7:%.*]] = add i32 [[Q]], 7
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_7:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = bitcast <8 x i8> [[RET_SLICE_7]] to <16 x i4>
+; UNALIGNED_ONLY-NEXT: ret <16 x i4> [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define <16 x i4> @load_v16i4_align4(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast <2 x i32> [[RET_LOADABLE]] to <16 x i4>
+; RELAXED_OOB_ONLY-NEXT: ret <16 x i4> [[RET]]
+;
+; BOTH_FLAGS-LABEL: define <16 x i4> @load_v16i4_align4(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = bitcast <2 x i32> [[RET_LOADABLE]] to <16 x i4>
+; BOTH_FLAGS-NEXT: ret <16 x i4> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load <16 x i4>, ptr addrspace(7) %q, align 4
+ ret <16 x i4> %ret
+}
+
+define void @store_v16i4_align4(<16 x i4> %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_v16i4_align4(
+; STRICT-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
+; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
+; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
+; STRICT-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
+; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
+; STRICT-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_v16i4_align4(
+; UNALIGNED_ONLY-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_v16i4_align4(
+; RELAXED_OOB_ONLY-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <2 x i32>
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_v16i4_align4(
+; BOTH_FLAGS-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <2 x i32>
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store <16 x i4> %data, ptr addrspace(7) %q, align 4
+ ret void
+}
+
+define <16 x i4> @load_v16i4_align2(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define <16 x i4> @load_v16i4_align2(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
+; STRICT-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
+; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
+; STRICT-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
+; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
+; STRICT-NEXT: [[Q_OFF_PTR_5:%.*]] = add i32 [[Q]], 5
+; STRICT-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
+; STRICT-NEXT: [[Q_OFF_PTR_6:%.*]] = add i32 [[Q]], 6
+; STRICT-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
+; STRICT-NEXT: [[Q_OFF_PTR_7:%.*]] = add i32 [[Q]], 7
+; STRICT-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_7:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
+; STRICT-NEXT: [[RET:%.*]] = bitcast <8 x i8> [[RET_SLICE_7]] to <16 x i4>
+; STRICT-NEXT: ret <16 x i4> [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define <16 x i4> @load_v16i4_align2(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_5:%.*]] = add i32 [[Q]], 5
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_6:%.*]] = add i32 [[Q]], 6
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_7:%.*]] = add i32 [[Q]], 7
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_7:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = bitcast <8 x i8> [[RET_SLICE_7]] to <16 x i4>
+; UNALIGNED_ONLY-NEXT: ret <16 x i4> [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define <16 x i4> @load_v16i4_align2(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <4 x i16> poison, i16 [[RET_OFF_0]], i64 0
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add nuw i32 [[Q]], 2
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <4 x i16> [[RET_SLICE_0]], i16 [[RET_OFF_2]], i64 1
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add nuw i32 [[Q]], 4
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <4 x i16> [[RET_SLICE_1]], i16 [[RET_OFF_4]], i64 2
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_6:%.*]] = add nuw i32 [[Q]], 6
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_3:%.*]] = insertelement <4 x i16> [[RET_SLICE_2]], i16 [[RET_OFF_6]], i64 3
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast <4 x i16> [[RET_SLICE_3]] to <16 x i4>
+; RELAXED_OOB_ONLY-NEXT: ret <16 x i4> [[RET]]
+;
+; BOTH_FLAGS-LABEL: define <16 x i4> @load_v16i4_align2(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = bitcast <2 x i32> [[RET_LOADABLE]] to <16 x i4>
+; BOTH_FLAGS-NEXT: ret <16 x i4> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load <16 x i4>, ptr addrspace(7) %q, align 2
+ ret <16 x i4> %ret
+}
+
+define void @store_v16i4_align2(<16 x i4> %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_v16i4_align2(
+; STRICT-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
+; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
+; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
+; STRICT-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
+; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
+; STRICT-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_v16i4_align2(
+; UNALIGNED_ONLY-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_v16i4_align2(
+; RELAXED_OOB_ONLY-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <4 x i16>
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 0
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 2
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 1
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_2:%.*]] = add nuw i32 [[Q]], 4
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 2
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_3:%.*]] = add nuw i32 [[Q]], 6
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <4 x i16> [[DATA_LEGAL]], i64 3
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_3]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_v16i4_align2(
+; BOTH_FLAGS-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <2 x i32>
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store <16 x i4> %data, ptr addrspace(7) %q, align 2
+ ret void
+}
+
+define <16 x i4> @load_v16i4_align1(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define <16 x i4> @load_v16i4_align1(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; STRICT-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
+; STRICT-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
+; STRICT-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
+; STRICT-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
+; STRICT-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
+; STRICT-NEXT: [[Q_OFF_PTR_5:%.*]] = add i32 [[Q]], 5
+; STRICT-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
+; STRICT-NEXT: [[Q_OFF_PTR_6:%.*]] = add i32 [[Q]], 6
+; STRICT-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
+; STRICT-NEXT: [[Q_OFF_PTR_7:%.*]] = add i32 [[Q]], 7
+; STRICT-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_7:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
+; STRICT-NEXT: [[RET:%.*]] = bitcast <8 x i8> [[RET_SLICE_7]] to <16 x i4>
+; STRICT-NEXT: ret <16 x i4> [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define <16 x i4> @load_v16i4_align1(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add i32 [[Q]], 1
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_3:%.*]] = add i32 [[Q]], 3
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_5:%.*]] = add i32 [[Q]], 5
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_6:%.*]] = add i32 [[Q]], 6
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
+; UNALIGNED_ONLY-NEXT: [[Q_OFF_PTR_7:%.*]] = add i32 [[Q]], 7
+; UNALIGNED_ONLY-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[RET_SLICE_7:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = bitcast <8 x i8> [[RET_SLICE_7]] to <16 x i4>
+; UNALIGNED_ONLY-NEXT: ret <16 x i4> [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define <16 x i4> @load_v16i4_align1(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_1:%.*]] = add nuw i32 [[Q]], 1
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add nuw i32 [[Q]], 2
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_3:%.*]] = add nuw i32 [[Q]], 3
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add nuw i32 [[Q]], 4
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_5:%.*]] = add nuw i32 [[Q]], 5
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_5]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_6:%.*]] = add nuw i32 [[Q]], 6
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_7:%.*]] = add nuw i32 [[Q]], 7
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 [[Q_OFF_PTR_7]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_7:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = bitcast <8 x i8> [[RET_SLICE_7]] to <16 x i4>
+; RELAXED_OOB_ONLY-NEXT: ret <16 x i4> [[RET]]
+;
+; BOTH_FLAGS-LABEL: define <16 x i4> @load_v16i4_align1(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = bitcast <2 x i32> [[RET_LOADABLE]] to <16 x i4>
+; BOTH_FLAGS-NEXT: ret <16 x i4> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ %ret = load <16 x i4>, ptr addrspace(7) %q, align 1
+ ret <16 x i4> %ret
+}
+
+define void @store_v16i4_align1(<16 x i4> %data, ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define void @store_v16i4_align1(
+; STRICT-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
+; STRICT-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
+; STRICT-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
+; STRICT-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
+; STRICT-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
+; STRICT-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
+; STRICT-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; STRICT-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
+; STRICT-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
+; STRICT-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; STRICT-NEXT: ret void
+;
+; UNALIGNED_ONLY-LABEL: define void @store_v16i4_align1(
+; UNALIGNED_ONLY-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_1:%.*]] = add i32 [[Q]], 1
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_2:%.*]] = add i32 [[Q]], 2
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_3:%.*]] = add i32 [[Q]], 3
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_4:%.*]] = add i32 [[Q]], 4
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_5:%.*]] = add i32 [[Q]], 5
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_6:%.*]] = add i32 [[Q]], 6
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: [[Q_PART_7:%.*]] = add i32 [[Q]], 7
+; UNALIGNED_ONLY-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
+; UNALIGNED_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret void
+;
+; RELAXED_OOB_ONLY-LABEL: define void @store_v16i4_align1(
+; RELAXED_OOB_ONLY-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <8 x i8>
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 0
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_1:%.*]] = add nuw i32 [[Q]], 1
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 1
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_1]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_2:%.*]] = add nuw i32 [[Q]], 2
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 2
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_3:%.*]] = add nuw i32 [[Q]], 3
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 3
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_3]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_4:%.*]] = add nuw i32 [[Q]], 4
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 4
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_5:%.*]] = add nuw i32 [[Q]], 5
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 5
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_5]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_6:%.*]] = add nuw i32 [[Q]], 6
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 6
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_6]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[Q_PART_7:%.*]] = add nuw i32 [[Q]], 7
+; RELAXED_OOB_ONLY-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA_LEGAL]], i64 7
+; RELAXED_OOB_ONLY-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q_PART_7]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret void
+;
+; BOTH_FLAGS-LABEL: define void @store_v16i4_align1(
+; BOTH_FLAGS-SAME: <16 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]], i32 [[Q:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[DATA_LEGAL:%.*]] = bitcast <16 x i4> [[DATA]] to <2 x i32>
+; BOTH_FLAGS-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off
+ store <16 x i4> %data, ptr addrspace(7) %q, align 1
+ ret void
+}
+
+define <4 x i32> @load_v4i32_align8_nneg(ptr addrspace(8) inreg %buf, i32 %off) {
+; STRICT-LABEL: define <4 x i32> @load_v4i32_align8_nneg(
+; STRICT-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[OFF:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[Q:%.*]] = call i32 @llvm.umin.i32(i32 [[OFF]], i32 2147483640)
+; STRICT-NEXT: [[RET:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: ret <4 x i32> [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define <4 x i32> @load_v4i32_align8_nneg(
+; UNALIGNED_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[OFF:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[Q:%.*]] = call i32 @llvm.umin.i32(i32 [[OFF]], i32 2147483640)
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret <4 x i32> [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define <4 x i32> @load_v4i32_align8_nneg(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[OFF:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[Q:%.*]] = call i32 @llvm.umin.i32(i32 [[OFF]], i32 2147483640)
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: ret <4 x i32> [[RET]]
+;
+; BOTH_FLAGS-LABEL: define <4 x i32> @load_v4i32_align8_nneg(
+; BOTH_FLAGS-SAME: ptr addrspace(8) inreg [[BUF:%.*]], i32 [[OFF:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[Q:%.*]] = call i32 @llvm.umin.i32(i32 [[OFF]], i32 2147483640)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret <4 x i32> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %off.clamped = call i32 @llvm.umin.i32(i32 %off, i32 2147483640)
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off.clamped
+ %ret = load <4 x i32>, ptr addrspace(7) %q, align 8
+ ret <4 x i32> %ret
+}
+
+define <8 x half> @load_v8f16_align2_not_oob(ptr addrspace(1) inreg %ptr, i32 %off) {
+; STRICT-LABEL: define <8 x half> @load_v8f16_align2_not_oob(
+; STRICT-SAME: ptr addrspace(1) inreg [[PTR:%.*]], i32 [[OFF:%.*]]) #[[ATTR0]] {
+; STRICT-NEXT: [[BUF:%.*]] = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1(ptr addrspace(1) [[PTR]], i16 0, i64 8192, i32 159744)
+; STRICT-NEXT: [[Q:%.*]] = call i32 @llvm.umin.i32(i32 [[OFF]], i32 1024)
+; STRICT-NEXT: [[RET_OFF_0:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x half> poison, half [[RET_OFF_0]], i64 0
+; STRICT-NEXT: [[Q_OFF_PTR_2:%.*]] = add i32 [[Q]], 2
+; STRICT-NEXT: [[RET_OFF_2:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x half> [[RET_SLICE_0]], half [[RET_OFF_2]], i64 1
+; STRICT-NEXT: [[Q_OFF_PTR_4:%.*]] = add i32 [[Q]], 4
+; STRICT-NEXT: [[RET_OFF_4:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x half> [[RET_SLICE_1]], half [[RET_OFF_4]], i64 2
+; STRICT-NEXT: [[Q_OFF_PTR_6:%.*]] = add i32 [[Q]], 6
+; STRICT-NEXT: [[RET_OFF_6:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x half> [[RET_SLICE_2]], half [[RET_OFF_6]], i64 3
+; STRICT-NEXT: [[Q_OFF_PTR_8:%.*]] = add i32 [[Q]], 8
+; STRICT-NEXT: [[RET_OFF_8:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x half> [[RET_SLICE_3]], half [[RET_OFF_8]], i64 4
+; STRICT-NEXT: [[Q_OFF_PTR_10:%.*]] = add i32 [[Q]], 10
+; STRICT-NEXT: [[RET_OFF_10:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_10]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x half> [[RET_SLICE_4]], half [[RET_OFF_10]], i64 5
+; STRICT-NEXT: [[Q_OFF_PTR_12:%.*]] = add i32 [[Q]], 12
+; STRICT-NEXT: [[RET_OFF_12:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_12]], i32 0, i32 0)
+; STRICT-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x half> [[RET_SLICE_5]], half [[RET_OFF_12]], i64 6
+; STRICT-NEXT: [[Q_OFF_PTR_14:%.*]] = add i32 [[Q]], 14
+; STRICT-NEXT: [[RET_OFF_14:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_14]], i32 0, i32 0)
+; STRICT-NEXT: [[RET:%.*]] = insertelement <8 x half> [[RET_SLICE_6]], half [[RET_OFF_14]], i64 7
+; STRICT-NEXT: ret <8 x half> [[RET]]
+;
+; UNALIGNED_ONLY-LABEL: define <8 x half> @load_v8f16_align2_not_oob(
+; UNALIGNED_ONLY-SAME: ptr addrspace(1) inreg [[PTR:%.*]], i32 [[OFF:%.*]]) #[[ATTR0]] {
+; UNALIGNED_ONLY-NEXT: [[BUF:%.*]] = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1(ptr addrspace(1) [[PTR]], i16 0, i64 8192, i32 159744)
+; UNALIGNED_ONLY-NEXT: [[Q:%.*]] = call i32 @llvm.umin.i32(i32 [[OFF]], i32 1024)
+; UNALIGNED_ONLY-NEXT: [[RET:%.*]] = call <8 x half> @llvm.amdgcn.raw.ptr.buffer.load.v8f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; UNALIGNED_ONLY-NEXT: ret <8 x half> [[RET]]
+;
+; RELAXED_OOB_ONLY-LABEL: define <8 x half> @load_v8f16_align2_not_oob(
+; RELAXED_OOB_ONLY-SAME: ptr addrspace(1) inreg [[PTR:%.*]], i32 [[OFF:%.*]]) #[[ATTR0]] {
+; RELAXED_OOB_ONLY-NEXT: [[BUF:%.*]] = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1(ptr addrspace(1) [[PTR]], i16 0, i64 8192, i32 159744)
+; RELAXED_OOB_ONLY-NEXT: [[Q:%.*]] = call i32 @llvm.umin.i32(i32 [[OFF]], i32 1024)
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_0:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x half> poison, half [[RET_OFF_0]], i64 0
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_2:%.*]] = add nuw i32 [[Q]], 2
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_2:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_2]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x half> [[RET_SLICE_0]], half [[RET_OFF_2]], i64 1
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_4:%.*]] = add nuw i32 [[Q]], 4
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_4:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_4]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x half> [[RET_SLICE_1]], half [[RET_OFF_4]], i64 2
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_6:%.*]] = add nuw i32 [[Q]], 6
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_6:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_6]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x half> [[RET_SLICE_2]], half [[RET_OFF_6]], i64 3
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_8:%.*]] = add nuw i32 [[Q]], 8
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_8:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_8]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x half> [[RET_SLICE_3]], half [[RET_OFF_8]], i64 4
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_10:%.*]] = add nuw i32 [[Q]], 10
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_10:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_10]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x half> [[RET_SLICE_4]], half [[RET_OFF_10]], i64 5
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_12:%.*]] = add nuw i32 [[Q]], 12
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_12:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_12]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x half> [[RET_SLICE_5]], half [[RET_OFF_12]], i64 6
+; RELAXED_OOB_ONLY-NEXT: [[Q_OFF_PTR_14:%.*]] = add nuw i32 [[Q]], 14
+; RELAXED_OOB_ONLY-NEXT: [[RET_OFF_14:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q_OFF_PTR_14]], i32 0, i32 0)
+; RELAXED_OOB_ONLY-NEXT: [[RET:%.*]] = insertelement <8 x half> [[RET_SLICE_6]], half [[RET_OFF_14]], i64 7
+; RELAXED_OOB_ONLY-NEXT: ret <8 x half> [[RET]]
+;
+; BOTH_FLAGS-LABEL: define <8 x half> @load_v8f16_align2_not_oob(
+; BOTH_FLAGS-SAME: ptr addrspace(1) inreg [[PTR:%.*]], i32 [[OFF:%.*]]) #[[ATTR0]] {
+; BOTH_FLAGS-NEXT: [[BUF:%.*]] = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1(ptr addrspace(1) [[PTR]], i16 0, i64 8192, i32 159744)
+; BOTH_FLAGS-NEXT: [[Q:%.*]] = call i32 @llvm.umin.i32(i32 [[OFF]], i32 1024)
+; BOTH_FLAGS-NEXT: [[RET:%.*]] = call <8 x half> @llvm.amdgcn.raw.ptr.buffer.load.v8f16(ptr addrspace(8) align 2 [[BUF]], i32 [[Q]], i32 0, i32 0)
+; BOTH_FLAGS-NEXT: ret <8 x half> [[RET]]
+;
+ %buf = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1(ptr addrspace(1) %ptr, i16 0, i64 8192, i32 159744)
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %off.clamped = call i32 @llvm.umin.i32(i32 %off, i32 1024)
+
+ %q = getelementptr i8, ptr addrspace(7) %p, i32 %off.clamped
+ %ret = load <8 x half>, ptr addrspace(7) %q, align 2
+ ret <8 x half> %ret
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 7, !"amdgpu.buffer.oob.mode", i32 BUFFER_OOB_MODE}
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization.ll
index 47efd8ac07410..0f1bda254e206 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization.ll
@@ -363,6 +363,34 @@ define void @store_v2f16(<2 x half> %data, ptr addrspace(8) inreg %buf) {
ret void
}
+define <2 x half> @load_v2f16_align2(ptr addrspace(8) inreg %buf) {
+; CHECK-LABEL: define <2 x half> @load_v2f16_align2(
+; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x half> poison, half [[RET_OFF_0]], i64 0
+; CHECK-NEXT: [[RET_OFF_2:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 2, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = insertelement <2 x half> [[RET_SLICE_0]], half [[RET_OFF_2]], i64 1
+; CHECK-NEXT: ret <2 x half> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %ret = load <2 x half>, ptr addrspace(7) %p, align 2
+ ret <2 x half> %ret
+}
+
+define void @store_v2f16_align2(<2 x half> %data, ptr addrspace(8) inreg %buf) {
+; CHECK-LABEL: define void @store_v2f16_align2(
+; CHECK-SAME: <2 x half> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <2 x half> [[DATA]], i64 0
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half [[DATA_SLICE_0]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <2 x half> [[DATA]], i64 1
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half [[DATA_SLICE_1]], ptr addrspace(8) align 2 [[BUF]], i32 2, i32 0, i32 0)
+; CHECK-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ store <2 x half> %data, ptr addrspace(7) %p, align 2
+ ret void
+}
+
define <4 x bfloat> @load_v4bf16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <4 x bfloat> @load_v4bf16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
@@ -1400,6 +1428,58 @@ define void @store_v8i8(<8 x i8> %data, ptr addrspace(8) inreg %buf) {
ret void
}
+define <8 x i8> @load_v8i8_align1(ptr addrspace(8) inreg %buf) {
+; CHECK-LABEL: define <8 x i8> @load_v8i8_align1(
+; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_SLICE_0:%.*]] = insertelement <8 x i8> poison, i8 [[RET_OFF_0]], i64 0
+; CHECK-NEXT: [[RET_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 1, i32 0, i32 0)
+; CHECK-NEXT: [[RET_SLICE_1:%.*]] = insertelement <8 x i8> [[RET_SLICE_0]], i8 [[RET_OFF_1]], i64 1
+; CHECK-NEXT: [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 2, i32 0, i32 0)
+; CHECK-NEXT: [[RET_SLICE_2:%.*]] = insertelement <8 x i8> [[RET_SLICE_1]], i8 [[RET_OFF_2]], i64 2
+; CHECK-NEXT: [[RET_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 3, i32 0, i32 0)
+; CHECK-NEXT: [[RET_SLICE_3:%.*]] = insertelement <8 x i8> [[RET_SLICE_2]], i8 [[RET_OFF_3]], i64 3
+; CHECK-NEXT: [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 4, i32 0, i32 0)
+; CHECK-NEXT: [[RET_SLICE_4:%.*]] = insertelement <8 x i8> [[RET_SLICE_3]], i8 [[RET_OFF_4]], i64 4
+; CHECK-NEXT: [[RET_OFF_5:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 5, i32 0, i32 0)
+; CHECK-NEXT: [[RET_SLICE_5:%.*]] = insertelement <8 x i8> [[RET_SLICE_4]], i8 [[RET_OFF_5]], i64 5
+; CHECK-NEXT: [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 6, i32 0, i32 0)
+; CHECK-NEXT: [[RET_SLICE_6:%.*]] = insertelement <8 x i8> [[RET_SLICE_5]], i8 [[RET_OFF_6]], i64 6
+; CHECK-NEXT: [[RET_OFF_7:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 7, i32 0, i32 0)
+; CHECK-NEXT: [[RET:%.*]] = insertelement <8 x i8> [[RET_SLICE_6]], i8 [[RET_OFF_7]], i64 7
+; CHECK-NEXT: ret <8 x i8> [[RET]]
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ %ret = load <8 x i8>, ptr addrspace(7) %p, align 1
+ ret <8 x i8> %ret
+}
+
+define void @store_v8i8_align1(<8 x i8> %data, ptr addrspace(8) inreg %buf) {
+; CHECK-LABEL: define void @store_v8i8_align1(
+; CHECK-SAME: <8 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = extractelement <8 x i8> [[DATA]], i64 0
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_0]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[DATA_SLICE_1:%.*]] = extractelement <8 x i8> [[DATA]], i64 1
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_1]], ptr addrspace(8) align 1 [[BUF]], i32 1, i32 0, i32 0)
+; CHECK-NEXT: [[DATA_SLICE_2:%.*]] = extractelement <8 x i8> [[DATA]], i64 2
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 1 [[BUF]], i32 2, i32 0, i32 0)
+; CHECK-NEXT: [[DATA_SLICE_3:%.*]] = extractelement <8 x i8> [[DATA]], i64 3
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_3]], ptr addrspace(8) align 1 [[BUF]], i32 3, i32 0, i32 0)
+; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = extractelement <8 x i8> [[DATA]], i64 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 1 [[BUF]], i32 4, i32 0, i32 0)
+; CHECK-NEXT: [[DATA_SLICE_5:%.*]] = extractelement <8 x i8> [[DATA]], i64 5
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_5]], ptr addrspace(8) align 1 [[BUF]], i32 5, i32 0, i32 0)
+; CHECK-NEXT: [[DATA_SLICE_6:%.*]] = extractelement <8 x i8> [[DATA]], i64 6
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 1 [[BUF]], i32 6, i32 0, i32 0)
+; CHECK-NEXT: [[DATA_SLICE_7:%.*]] = extractelement <8 x i8> [[DATA]], i64 7
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_7]], ptr addrspace(8) align 1 [[BUF]], i32 7, i32 0, i32 0)
+; CHECK-NEXT: ret void
+;
+ %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
+ store <8 x i8> %data, ptr addrspace(7) %p, align 1
+ ret void
+}
+
define <12 x i8> @load_v12i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <12 x i8> @load_v12i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
@@ -1514,10 +1594,10 @@ define [2 x i32] @load_a2i32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define [2 x i32] @load_a2i32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
-; CHECK-NEXT: [[RET_ELEM_0:%.*]] = extractelement <2 x i32> [[RET_LOADABLE]], i64 0
-; CHECK-NEXT: [[RET_AS_ARRAY_0:%.*]] = insertvalue [2 x i32] poison, i32 [[RET_ELEM_0]], 0
-; CHECK-NEXT: [[RET_ELEM_1:%.*]] = extractelement <2 x i32> [[RET_LOADABLE]], i64 1
-; CHECK-NEXT: [[RET:%.*]] = insertvalue [2 x i32] [[RET_AS_ARRAY_0]], i32 [[RET_ELEM_1]], 1
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = extractelement <2 x i32> [[RET_LOADABLE]], i64 0
+; CHECK-NEXT: [[RET_AS_ARRAY_0:%.*]] = insertvalue [2 x i32] poison, i32 [[RET_OFF_0]], 0
+; CHECK-NEXT: [[RET_OFF_4:%.*]] = extractelement <2 x i32> [[RET_LOADABLE]], i64 1
+; CHECK-NEXT: [[RET:%.*]] = insertvalue [2 x i32] [[RET_AS_ARRAY_0]], i32 [[RET_OFF_4]], 1
; CHECK-NEXT: ret [2 x i32] [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1543,11 +1623,12 @@ define void @store_a2i32([2 x i32] %data, ptr addrspace(8) inreg %buf) {
define [2 x half] @load_a2f16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define [2 x half] @load_a2f16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call <2 x half> @llvm.amdgcn.raw.ptr.buffer.load.v2f16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
-; CHECK-NEXT: [[RET_ELEM_0:%.*]] = extractelement <2 x half> [[RET_LOADABLE]], i64 0
-; CHECK-NEXT: [[RET_AS_ARRAY_0:%.*]] = insertvalue [2 x half] poison, half [[RET_ELEM_0]], 0
-; CHECK-NEXT: [[RET_ELEM_1:%.*]] = extractelement <2 x half> [[RET_LOADABLE]], i64 1
-; CHECK-NEXT: [[RET:%.*]] = insertvalue [2 x half] [[RET_AS_ARRAY_0]], half [[RET_ELEM_1]], 1
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_SLICE_0:%.*]] = insertelement <2 x half> poison, half [[RET_OFF_0]], i64 0
+; CHECK-NEXT: [[RET_OFF_2:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 2, i32 0, i32 0)
+; CHECK-NEXT: [[RET_LOADABLE:%.*]] = insertelement <2 x half> [[RET_SLICE_0]], half [[RET_OFF_2]], i64 1
+; CHECK-NEXT: [[RET_AS_ARRAY_0:%.*]] = insertvalue [2 x half] poison, half [[RET_OFF_0]], 0
+; CHECK-NEXT: [[RET:%.*]] = insertvalue [2 x half] [[RET_AS_ARRAY_0]], half [[RET_OFF_2]], 1
; CHECK-NEXT: ret [2 x half] [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1562,7 +1643,8 @@ define void @store_a2f16([2 x half] %data, ptr addrspace(8) inreg %buf) {
; CHECK-NEXT: [[DATA_AS_VEC_0:%.*]] = insertelement <2 x half> poison, half [[DATA_ELEM_0]], i64 0
; CHECK-NEXT: [[DATA_ELEM_1:%.*]] = extractvalue [2 x half] [[DATA]], 1
; CHECK-NEXT: [[DATA_AS_VEC_1:%.*]] = insertelement <2 x half> [[DATA_AS_VEC_0]], half [[DATA_ELEM_1]], i64 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2f16(<2 x half> [[DATA_AS_VEC_1]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half [[DATA_ELEM_0]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half [[DATA_ELEM_1]], ptr addrspace(8) align 2 [[BUF]], i32 2, i32 0, i32 0)
; CHECK-NEXT: ret void
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1574,10 +1656,10 @@ define [2 x ptr addrspace(1)] @load_a2p1(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define [2 x ptr addrspace(1)] @load_a2p1(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[RET_LOADABLE:%.*]] = call <2 x ptr addrspace(1)> @llvm.amdgcn.raw.ptr.buffer.load.v2p1(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
-; CHECK-NEXT: [[RET_ELEM_0:%.*]] = extractelement <2 x ptr addrspace(1)> [[RET_LOADABLE]], i64 0
-; CHECK-NEXT: [[RET_AS_ARRAY_0:%.*]] = insertvalue [2 x ptr addrspace(1)] poison, ptr addrspace(1) [[RET_ELEM_0]], 0
-; CHECK-NEXT: [[RET_ELEM_1:%.*]] = extractelement <2 x ptr addrspace(1)> [[RET_LOADABLE]], i64 1
-; CHECK-NEXT: [[RET:%.*]] = insertvalue [2 x ptr addrspace(1)] [[RET_AS_ARRAY_0]], ptr addrspace(1) [[RET_ELEM_1]], 1
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = extractelement <2 x ptr addrspace(1)> [[RET_LOADABLE]], i64 0
+; CHECK-NEXT: [[RET_AS_ARRAY_0:%.*]] = insertvalue [2 x ptr addrspace(1)] poison, ptr addrspace(1) [[RET_OFF_0]], 0
+; CHECK-NEXT: [[RET_OFF_8:%.*]] = extractelement <2 x ptr addrspace(1)> [[RET_LOADABLE]], i64 1
+; CHECK-NEXT: [[RET:%.*]] = insertvalue [2 x ptr addrspace(1)] [[RET_AS_ARRAY_0]], ptr addrspace(1) [[RET_OFF_8]], 1
; CHECK-NEXT: ret [2 x ptr addrspace(1)] [[RET]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -1878,10 +1960,10 @@ define void @store_v2i6(<2 x i6> %data, ptr addrspace(8) inreg %buf) {
define <6 x i32> @load_v32i6(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <6 x i32> @load_v32i6(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT: [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison>
; CHECK-NEXT: [[RET_PARTS_0:%.*]] = shufflevector <6 x i32> poison, <6 x i32> [[RET_EXT_0]], <6 x i32> <i32 6, i32 7, i32 8, i32 9, i32 4, i32 5>
-; CHECK-NEXT: [[RET_OFF_16:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: [[RET_OFF_16:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
; CHECK-NEXT: [[RET_EXT_4:%.*]] = shufflevector <2 x i32> [[RET_OFF_16]], <2 x i32> poison, <6 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[RET_PARTS_4:%.*]] = shufflevector <6 x i32> [[RET_PARTS_0]], <6 x i32> [[RET_EXT_4]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 7>
; CHECK-NEXT: [[RET:%.*]] = bitcast <6 x i32> [[RET_PARTS_4]] to <32 x i6>
@@ -1889,7 +1971,7 @@ define <6 x i32> @load_v32i6(ptr addrspace(8) inreg %buf) {
; CHECK-NEXT: ret <6 x i32> [[RET_CAST]]
;
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
- %ret = load <32 x i6>, ptr addrspace(7) %p
+ %ret = load <32 x i6>, ptr addrspace(7) %p, align 4
%ret.cast = bitcast <32 x i6> %ret to <6 x i32>
ret <6 x i32> %ret.cast
}
@@ -1899,14 +1981,14 @@ define void @store_v32i6(<6 x i32> %data.abi, ptr addrspace(8) inreg %buf) {
; CHECK-SAME: <6 x i32> [[DATA_ABI:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[DATA:%.*]] = bitcast <6 x i32> [[DATA_ABI]] to <32 x i6>
; CHECK-NEXT: [[DATA_SLICE_0:%.*]] = shufflevector <6 x i32> [[DATA_ABI]], <6 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT: [[DATA_SLICE_4:%.*]] = shufflevector <6 x i32> [[DATA_ABI]], <6 x i32> poison, <2 x i32> <i32 4, i32 5>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0)
; CHECK-NEXT: ret void
;
%data = bitcast <6 x i32> %data.abi to <32 x i6>
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
- store <32 x i6> %data, ptr addrspace(7) %p
+ store <32 x i6> %data, ptr addrspace(7) %p, align 4
ret void
}
@@ -1971,16 +2053,16 @@ define [2 x [2 x i32]] @load_a2a2i32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define [2 x [2 x i32]] @load_a2a2i32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[RET0_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
-; CHECK-NEXT: [[RET0_ELEM_0:%.*]] = extractelement <2 x i32> [[RET0_OFF_0]], i64 0
-; CHECK-NEXT: [[RET0_AS_ARRAY_0:%.*]] = insertvalue [2 x i32] poison, i32 [[RET0_ELEM_0]], 0
-; CHECK-NEXT: [[RET0_ELEM_1:%.*]] = extractelement <2 x i32> [[RET0_OFF_0]], i64 1
-; CHECK-NEXT: [[RET0_AS_ARRAY_1:%.*]] = insertvalue [2 x i32] [[RET0_AS_ARRAY_0]], i32 [[RET0_ELEM_1]], 1
+; CHECK-NEXT: [[RET0_OFF_1:%.*]] = extractelement <2 x i32> [[RET0_OFF_0]], i64 0
+; CHECK-NEXT: [[RET0_AS_ARRAY_0:%.*]] = insertvalue [2 x i32] poison, i32 [[RET0_OFF_1]], 0
+; CHECK-NEXT: [[RET0_OFF_4:%.*]] = extractelement <2 x i32> [[RET0_OFF_0]], i64 1
+; CHECK-NEXT: [[RET0_AS_ARRAY_1:%.*]] = insertvalue [2 x i32] [[RET0_AS_ARRAY_0]], i32 [[RET0_OFF_4]], 1
; CHECK-NEXT: [[RET0:%.*]] = insertvalue [2 x [2 x i32]] poison, [2 x i32] [[RET0_AS_ARRAY_1]], 0
; CHECK-NEXT: [[RET1_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 8, i32 0, i32 0)
-; CHECK-NEXT: [[RET1_ELEM_0:%.*]] = extractelement <2 x i32> [[RET1_OFF_8]], i64 0
-; CHECK-NEXT: [[RET1_AS_ARRAY_0:%.*]] = insertvalue [2 x i32] poison, i32 [[RET1_ELEM_0]], 0
-; CHECK-NEXT: [[RET1_ELEM_1:%.*]] = extractelement <2 x i32> [[RET1_OFF_8]], i64 1
-; CHECK-NEXT: [[RET1_AS_ARRAY_1:%.*]] = insertvalue [2 x i32] [[RET1_AS_ARRAY_0]], i32 [[RET1_ELEM_1]], 1
+; CHECK-NEXT: [[RET1_OFF_9:%.*]] = extractelement <2 x i32> [[RET1_OFF_8]], i64 0
+; CHECK-NEXT: [[RET1_AS_ARRAY_0:%.*]] = insertvalue [2 x i32] poison, i32 [[RET1_OFF_9]], 0
+; CHECK-NEXT: [[RET1_OFF_12:%.*]] = extractelement <2 x i32> [[RET1_OFF_8]], i64 1
+; CHECK-NEXT: [[RET1_AS_ARRAY_1:%.*]] = insertvalue [2 x i32] [[RET1_AS_ARRAY_0]], i32 [[RET1_OFF_12]], 1
; CHECK-NEXT: [[RET:%.*]] = insertvalue [2 x [2 x i32]] [[RET0]], [2 x i32] [[RET1_AS_ARRAY_1]], 1
; CHECK-NEXT: ret [2 x [2 x i32]] [[RET]]
;
@@ -2234,3 +2316,6 @@ define void @store_nxv2i32(<vscale x 2 x i32> %data, ptr addrspace(8) inreg %buf
store <vscale x 2 x i32> %data, ptr addrspace(7) %p
ret void
}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 7, !"amdgpu.buffer.oob.mode", i32 1}
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-mem-transfer.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-mem-transfer.ll
index 09710860743d0..297174f41e75e 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-mem-transfer.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-mem-transfer.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -S -mcpu=gfx900 -amdgpu-lower-buffer-fat-pointers < %s | FileCheck %s
-; RUN: opt -S -mcpu=gfx900 -passes=amdgpu-lower-buffer-fat-pointers < %s | FileCheck %s
+; RUN: opt -S -mcpu=gfx900 -mattr=+unaligned-access-mode -amdgpu-lower-buffer-fat-pointers < %s | FileCheck %s
+; RUN: opt -S -mcpu=gfx900 -mattr=+unaligned-access-mode -passes=amdgpu-lower-buffer-fat-pointers < %s | FileCheck %s
target triple = "amdgcn--"
@@ -26,112 +26,112 @@ define void @memcpy_known(ptr addrspace(7) inreg %src, ptr addrspace(7) inreg %d
; CHECK-NEXT: [[DOTOFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[TMP1]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_0:%.*]] = shufflevector <4 x i32> [[DOTOFF_0]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_0:%.*]] = shufflevector <64 x i32> poison, <64 x i32> [[DOTEXT_0]], <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_16:%.*]] = add nuw i32 [[TMP1]], 16
+; CHECK-NEXT: [[DOTOFF_PTR_16:%.*]] = add i32 [[TMP1]], 16
; CHECK-NEXT: [[DOTOFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_16]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_4:%.*]] = shufflevector <4 x i32> [[DOTOFF_16]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_4:%.*]] = shufflevector <64 x i32> [[DOTPARTS_0]], <64 x i32> [[DOTEXT_4]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 64, i32 65, i32 66, i32 67, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_32:%.*]] = add nuw i32 [[TMP1]], 32
+; CHECK-NEXT: [[DOTOFF_PTR_32:%.*]] = add i32 [[TMP1]], 32
; CHECK-NEXT: [[DOTOFF_32:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_32]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_8:%.*]] = shufflevector <4 x i32> [[DOTOFF_32]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_8:%.*]] = shufflevector <64 x i32> [[DOTPARTS_4]], <64 x i32> [[DOTEXT_8]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 64, i32 65, i32 66, i32 67, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_48:%.*]] = add nuw i32 [[TMP1]], 48
+; CHECK-NEXT: [[DOTOFF_PTR_48:%.*]] = add i32 [[TMP1]], 48
; CHECK-NEXT: [[DOTOFF_48:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_48]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_12:%.*]] = shufflevector <4 x i32> [[DOTOFF_48]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_12:%.*]] = shufflevector <64 x i32> [[DOTPARTS_8]], <64 x i32> [[DOTEXT_12]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 64, i32 65, i32 66, i32 67, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_64:%.*]] = add nuw i32 [[TMP1]], 64
+; CHECK-NEXT: [[DOTOFF_PTR_64:%.*]] = add i32 [[TMP1]], 64
; CHECK-NEXT: [[DOTOFF_64:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_64]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_16:%.*]] = shufflevector <4 x i32> [[DOTOFF_64]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_16:%.*]] = shufflevector <64 x i32> [[DOTPARTS_12]], <64 x i32> [[DOTEXT_16]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_80:%.*]] = add nuw i32 [[TMP1]], 80
+; CHECK-NEXT: [[DOTOFF_PTR_80:%.*]] = add i32 [[TMP1]], 80
; CHECK-NEXT: [[DOTOFF_80:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_80]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_20:%.*]] = shufflevector <4 x i32> [[DOTOFF_80]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_20:%.*]] = shufflevector <64 x i32> [[DOTPARTS_16]], <64 x i32> [[DOTEXT_20]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 64, i32 65, i32 66, i32 67, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_96:%.*]] = add nuw i32 [[TMP1]], 96
+; CHECK-NEXT: [[DOTOFF_PTR_96:%.*]] = add i32 [[TMP1]], 96
; CHECK-NEXT: [[DOTOFF_96:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_96]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_24:%.*]] = shufflevector <4 x i32> [[DOTOFF_96]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_24:%.*]] = shufflevector <64 x i32> [[DOTPARTS_20]], <64 x i32> [[DOTEXT_24]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 64, i32 65, i32 66, i32 67, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_112:%.*]] = add nuw i32 [[TMP1]], 112
+; CHECK-NEXT: [[DOTOFF_PTR_112:%.*]] = add i32 [[TMP1]], 112
; CHECK-NEXT: [[DOTOFF_112:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_112]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_28:%.*]] = shufflevector <4 x i32> [[DOTOFF_112]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_28:%.*]] = shufflevector <64 x i32> [[DOTPARTS_24]], <64 x i32> [[DOTEXT_28]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 64, i32 65, i32 66, i32 67, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_128:%.*]] = add nuw i32 [[TMP1]], 128
+; CHECK-NEXT: [[DOTOFF_PTR_128:%.*]] = add i32 [[TMP1]], 128
; CHECK-NEXT: [[DOTOFF_128:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_128]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_32:%.*]] = shufflevector <4 x i32> [[DOTOFF_128]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_32:%.*]] = shufflevector <64 x i32> [[DOTPARTS_28]], <64 x i32> [[DOTEXT_32]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 64, i32 65, i32 66, i32 67, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_144:%.*]] = add nuw i32 [[TMP1]], 144
+; CHECK-NEXT: [[DOTOFF_PTR_144:%.*]] = add i32 [[TMP1]], 144
; CHECK-NEXT: [[DOTOFF_144:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_144]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_36:%.*]] = shufflevector <4 x i32> [[DOTOFF_144]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_36:%.*]] = shufflevector <64 x i32> [[DOTPARTS_32]], <64 x i32> [[DOTEXT_36]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 64, i32 65, i32 66, i32 67, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_160:%.*]] = add nuw i32 [[TMP1]], 160
+; CHECK-NEXT: [[DOTOFF_PTR_160:%.*]] = add i32 [[TMP1]], 160
; CHECK-NEXT: [[DOTOFF_160:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_160]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_40:%.*]] = shufflevector <4 x i32> [[DOTOFF_160]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_40:%.*]] = shufflevector <64 x i32> [[DOTPARTS_36]], <64 x i32> [[DOTEXT_40]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 64, i32 65, i32 66, i32 67, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_176:%.*]] = add nuw i32 [[TMP1]], 176
+; CHECK-NEXT: [[DOTOFF_PTR_176:%.*]] = add i32 [[TMP1]], 176
; CHECK-NEXT: [[DOTOFF_176:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_176]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_44:%.*]] = shufflevector <4 x i32> [[DOTOFF_176]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_44:%.*]] = shufflevector <64 x i32> [[DOTPARTS_40]], <64 x i32> [[DOTEXT_44]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 64, i32 65, i32 66, i32 67, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_192:%.*]] = add nuw i32 [[TMP1]], 192
+; CHECK-NEXT: [[DOTOFF_PTR_192:%.*]] = add i32 [[TMP1]], 192
; CHECK-NEXT: [[DOTOFF_192:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_192]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_48:%.*]] = shufflevector <4 x i32> [[DOTOFF_192]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_48:%.*]] = shufflevector <64 x i32> [[DOTPARTS_44]], <64 x i32> [[DOTEXT_48]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 64, i32 65, i32 66, i32 67, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_208:%.*]] = add nuw i32 [[TMP1]], 208
+; CHECK-NEXT: [[DOTOFF_PTR_208:%.*]] = add i32 [[TMP1]], 208
; CHECK-NEXT: [[DOTOFF_208:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_208]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_52:%.*]] = shufflevector <4 x i32> [[DOTOFF_208]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_52:%.*]] = shufflevector <64 x i32> [[DOTPARTS_48]], <64 x i32> [[DOTEXT_52]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 64, i32 65, i32 66, i32 67, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_224:%.*]] = add nuw i32 [[TMP1]], 224
+; CHECK-NEXT: [[DOTOFF_PTR_224:%.*]] = add i32 [[TMP1]], 224
; CHECK-NEXT: [[DOTOFF_224:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_224]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_56:%.*]] = shufflevector <4 x i32> [[DOTOFF_224]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_56:%.*]] = shufflevector <64 x i32> [[DOTPARTS_52]], <64 x i32> [[DOTEXT_56]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 64, i32 65, i32 66, i32 67, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_240:%.*]] = add nuw i32 [[TMP1]], 240
+; CHECK-NEXT: [[DOTOFF_PTR_240:%.*]] = add i32 [[TMP1]], 240
; CHECK-NEXT: [[DOTOFF_240:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_240]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_60:%.*]] = shufflevector <4 x i32> [[DOTOFF_240]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <64 x i32> [[DOTPARTS_56]], <64 x i32> [[DOTEXT_60]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 64, i32 65, i32 66, i32 67>
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX]]
; CHECK-NEXT: [[DOTSLICE_0:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_0]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_4:%.*]] = add nuw i32 [[TMP3]], 16
+; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP3]], 16
; CHECK-NEXT: [[DOTSLICE_4:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_4]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_8:%.*]] = add nuw i32 [[TMP3]], 32
+; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP3]], 32
; CHECK-NEXT: [[DOTSLICE_8:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_8]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_12:%.*]] = add nuw i32 [[TMP3]], 48
+; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP3]], 48
; CHECK-NEXT: [[DOTSLICE_12:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_12]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_16:%.*]] = add nuw i32 [[TMP3]], 64
+; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP3]], 64
; CHECK-NEXT: [[DOTSLICE_16:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_16]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_20:%.*]] = add nuw i32 [[TMP3]], 80
+; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP3]], 80
; CHECK-NEXT: [[DOTSLICE_20:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_20]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_24:%.*]] = add nuw i32 [[TMP3]], 96
+; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP3]], 96
; CHECK-NEXT: [[DOTSLICE_24:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_24]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_28:%.*]] = add nuw i32 [[TMP3]], 112
+; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP3]], 112
; CHECK-NEXT: [[DOTSLICE_28:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_28]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_32:%.*]] = add nuw i32 [[TMP3]], 128
+; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP3]], 128
; CHECK-NEXT: [[DOTSLICE_32:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_32]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_36:%.*]] = add nuw i32 [[TMP3]], 144
+; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP3]], 144
; CHECK-NEXT: [[DOTSLICE_36:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 36, i32 37, i32 38, i32 39>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_36]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_40:%.*]] = add nuw i32 [[TMP3]], 160
+; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP3]], 160
; CHECK-NEXT: [[DOTSLICE_40:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 40, i32 41, i32 42, i32 43>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_40]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_44:%.*]] = add nuw i32 [[TMP3]], 176
+; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP3]], 176
; CHECK-NEXT: [[DOTSLICE_44:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 44, i32 45, i32 46, i32 47>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_44]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_48:%.*]] = add nuw i32 [[TMP3]], 192
+; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP3]], 192
; CHECK-NEXT: [[DOTSLICE_48:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 48, i32 49, i32 50, i32 51>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_48]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_52:%.*]] = add nuw i32 [[TMP3]], 208
+; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP3]], 208
; CHECK-NEXT: [[DOTSLICE_52:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 52, i32 53, i32 54, i32 55>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_52]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_56:%.*]] = add nuw i32 [[TMP3]], 224
+; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP3]], 224
; CHECK-NEXT: [[DOTSLICE_56:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 56, i32 57, i32 58, i32 59>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_56]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_60:%.*]] = add nuw i32 [[TMP3]], 240
+; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP3]], 240
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTOFF_240]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
; CHECK-NEXT: [[TMP4]] = add i32 [[LOOP_INDEX]], 256
; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i32 [[TMP4]], 8192
@@ -150,12 +150,44 @@ define void @memcpy_known_small(ptr addrspace(7) inreg %src, ptr addrspace(7) in
; CHECK-NEXT: [[DST_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[DST]], 1
; CHECK-NEXT: [[SRC_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 0
; CHECK-NEXT: [[SRC_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 1
-; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0)
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[TMP1]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_0:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_0]], i64 0
+; CHECK-NEXT: [[DOTOFF_PTR_4:%.*]] = add i32 [[SRC_OFF]], 4
+; CHECK-NEXT: [[DOTOFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_4]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_1:%.*]] = insertelement <4 x i32> [[DOTSLICE_0]], i32 [[DOTOFF_4]], i64 1
+; CHECK-NEXT: [[DOTOFF_PTR_8:%.*]] = add i32 [[SRC_OFF]], 8
+; CHECK-NEXT: [[DOTOFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_8]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_2:%.*]] = insertelement <4 x i32> [[DOTSLICE_1]], i32 [[DOTOFF_8]], i64 2
+; CHECK-NEXT: [[DOTOFF_PTR_12:%.*]] = add i32 [[SRC_OFF]], 12
+; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[DST_OFF]], 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[DST_OFF]], 8
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[DST_OFF]], 12
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[SRC_OFF]], 16
-; CHECK-NEXT: [[TMP3:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_01:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_02:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_01]], i64 0
+; CHECK-NEXT: [[DOTOFF_PTR_43:%.*]] = add i32 [[TMP2]], 4
+; CHECK-NEXT: [[DOTOFF_44:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_43]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_15:%.*]] = insertelement <4 x i32> [[DOTSLICE_02]], i32 [[DOTOFF_44]], i64 1
+; CHECK-NEXT: [[DOTOFF_PTR_86:%.*]] = add i32 [[TMP2]], 8
+; CHECK-NEXT: [[DOTOFF_87:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_86]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_28:%.*]] = insertelement <4 x i32> [[DOTSLICE_15]], i32 [[DOTOFF_87]], i64 2
+; CHECK-NEXT: [[DOTOFF_PTR_129:%.*]] = add i32 [[TMP2]], 12
+; CHECK-NEXT: [[DOTOFF_1210:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_129]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[DOTSLICE_28]], i32 [[DOTOFF_1210]], i64 3
; CHECK-NEXT: [[TMP4:%.*]] = add nuw i32 [[DST_OFF]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[TMP3]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_01]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_111:%.*]] = add i32 [[TMP4]], 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_111]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_212:%.*]] = add i32 [[TMP4]], 8
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_87]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_212]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_313:%.*]] = add i32 [[TMP4]], 12
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_1210]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_313]], i32 0, i32 0)
; CHECK-NEXT: ret void
;
call void @llvm.memcpy.p7.p7.i32(ptr addrspace(7) %dst, ptr addrspace(7) %src, i32 32, i1 false)
@@ -216,117 +248,326 @@ define void @memcpy_known_i64(ptr addrspace(7) inreg %src, ptr addrspace(7) inre
; CHECK-NEXT: [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], %[[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[LOOP_INDEX_C:%.*]] = trunc i64 [[LOOP_INDEX]] to i32
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[SRC_OFF]], [[LOOP_INDEX_C]]
-; CHECK-NEXT: [[DOTOFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP1]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_0:%.*]] = shufflevector <4 x i32> [[DOTOFF_0]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_0:%.*]] = shufflevector <64 x i32> poison, <64 x i32> [[DOTEXT_0]], <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_16:%.*]] = add nuw i32 [[TMP1]], 16
-; CHECK-NEXT: [[DOTOFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_16]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_4:%.*]] = shufflevector <4 x i32> [[DOTOFF_16]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_4:%.*]] = shufflevector <64 x i32> [[DOTPARTS_0]], <64 x i32> [[DOTEXT_4]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 64, i32 65, i32 66, i32 67, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_32:%.*]] = add nuw i32 [[TMP1]], 32
-; CHECK-NEXT: [[DOTOFF_32:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_32]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_8:%.*]] = shufflevector <4 x i32> [[DOTOFF_32]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_8:%.*]] = shufflevector <64 x i32> [[DOTPARTS_4]], <64 x i32> [[DOTEXT_8]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 64, i32 65, i32 66, i32 67, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_48:%.*]] = add nuw i32 [[TMP1]], 48
-; CHECK-NEXT: [[DOTOFF_48:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_48]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_12:%.*]] = shufflevector <4 x i32> [[DOTOFF_48]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_12:%.*]] = shufflevector <64 x i32> [[DOTPARTS_8]], <64 x i32> [[DOTEXT_12]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 64, i32 65, i32 66, i32 67, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_64:%.*]] = add nuw i32 [[TMP1]], 64
-; CHECK-NEXT: [[DOTOFF_64:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_64]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_16:%.*]] = shufflevector <4 x i32> [[DOTOFF_64]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_16:%.*]] = shufflevector <64 x i32> [[DOTPARTS_12]], <64 x i32> [[DOTEXT_16]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_80:%.*]] = add nuw i32 [[TMP1]], 80
-; CHECK-NEXT: [[DOTOFF_80:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_80]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_20:%.*]] = shufflevector <4 x i32> [[DOTOFF_80]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_20:%.*]] = shufflevector <64 x i32> [[DOTPARTS_16]], <64 x i32> [[DOTEXT_20]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 64, i32 65, i32 66, i32 67, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_96:%.*]] = add nuw i32 [[TMP1]], 96
-; CHECK-NEXT: [[DOTOFF_96:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_96]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_24:%.*]] = shufflevector <4 x i32> [[DOTOFF_96]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_24:%.*]] = shufflevector <64 x i32> [[DOTPARTS_20]], <64 x i32> [[DOTEXT_24]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 64, i32 65, i32 66, i32 67, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_112:%.*]] = add nuw i32 [[TMP1]], 112
-; CHECK-NEXT: [[DOTOFF_112:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_112]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_28:%.*]] = shufflevector <4 x i32> [[DOTOFF_112]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_28:%.*]] = shufflevector <64 x i32> [[DOTPARTS_24]], <64 x i32> [[DOTEXT_28]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 64, i32 65, i32 66, i32 67, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_128:%.*]] = add nuw i32 [[TMP1]], 128
-; CHECK-NEXT: [[DOTOFF_128:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_128]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_32:%.*]] = shufflevector <4 x i32> [[DOTOFF_128]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_32:%.*]] = shufflevector <64 x i32> [[DOTPARTS_28]], <64 x i32> [[DOTEXT_32]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 64, i32 65, i32 66, i32 67, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_144:%.*]] = add nuw i32 [[TMP1]], 144
-; CHECK-NEXT: [[DOTOFF_144:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_144]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_36:%.*]] = shufflevector <4 x i32> [[DOTOFF_144]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_36:%.*]] = shufflevector <64 x i32> [[DOTPARTS_32]], <64 x i32> [[DOTEXT_36]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 64, i32 65, i32 66, i32 67, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_160:%.*]] = add nuw i32 [[TMP1]], 160
-; CHECK-NEXT: [[DOTOFF_160:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_160]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_40:%.*]] = shufflevector <4 x i32> [[DOTOFF_160]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_40:%.*]] = shufflevector <64 x i32> [[DOTPARTS_36]], <64 x i32> [[DOTEXT_40]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 64, i32 65, i32 66, i32 67, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_176:%.*]] = add nuw i32 [[TMP1]], 176
-; CHECK-NEXT: [[DOTOFF_176:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_176]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_44:%.*]] = shufflevector <4 x i32> [[DOTOFF_176]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_44:%.*]] = shufflevector <64 x i32> [[DOTPARTS_40]], <64 x i32> [[DOTEXT_44]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 64, i32 65, i32 66, i32 67, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_192:%.*]] = add nuw i32 [[TMP1]], 192
-; CHECK-NEXT: [[DOTOFF_192:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_192]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_48:%.*]] = shufflevector <4 x i32> [[DOTOFF_192]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_48:%.*]] = shufflevector <64 x i32> [[DOTPARTS_44]], <64 x i32> [[DOTEXT_48]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 64, i32 65, i32 66, i32 67, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_208:%.*]] = add nuw i32 [[TMP1]], 208
-; CHECK-NEXT: [[DOTOFF_208:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_208]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_52:%.*]] = shufflevector <4 x i32> [[DOTOFF_208]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_52:%.*]] = shufflevector <64 x i32> [[DOTPARTS_48]], <64 x i32> [[DOTEXT_52]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 64, i32 65, i32 66, i32 67, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_224:%.*]] = add nuw i32 [[TMP1]], 224
-; CHECK-NEXT: [[DOTOFF_224:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_224]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_56:%.*]] = shufflevector <4 x i32> [[DOTOFF_224]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_56:%.*]] = shufflevector <64 x i32> [[DOTPARTS_52]], <64 x i32> [[DOTEXT_56]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 64, i32 65, i32 66, i32 67, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_240:%.*]] = add nuw i32 [[TMP1]], 240
-; CHECK-NEXT: [[DOTOFF_240:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_240]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_60:%.*]] = shufflevector <4 x i32> [[DOTOFF_240]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <64 x i32> [[DOTPARTS_56]], <64 x i32> [[DOTEXT_60]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 64, i32 65, i32 66, i32 67>
+; CHECK-NEXT: [[DOTOFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP1]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_0:%.*]] = insertelement <64 x i32> poison, i32 [[DOTOFF_0]], i64 0
+; CHECK-NEXT: [[DOTOFF_PTR_4:%.*]] = add i32 [[TMP1]], 4
+; CHECK-NEXT: [[DOTOFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_4]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_1:%.*]] = insertelement <64 x i32> [[DOTSLICE_0]], i32 [[DOTOFF_4]], i64 1
+; CHECK-NEXT: [[DOTOFF_PTR_8:%.*]] = add i32 [[TMP1]], 8
+; CHECK-NEXT: [[DOTOFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_8]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_2:%.*]] = insertelement <64 x i32> [[DOTSLICE_1]], i32 [[DOTOFF_8]], i64 2
+; CHECK-NEXT: [[DOTOFF_PTR_12:%.*]] = add i32 [[TMP1]], 12
+; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_3:%.*]] = insertelement <64 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
+; CHECK-NEXT: [[DOTOFF_PTR_16:%.*]] = add i32 [[TMP1]], 16
+; CHECK-NEXT: [[DOTOFF_16:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_16]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_4:%.*]] = insertelement <64 x i32> [[DOTSLICE_3]], i32 [[DOTOFF_16]], i64 4
+; CHECK-NEXT: [[DOTOFF_PTR_20:%.*]] = add i32 [[TMP1]], 20
+; CHECK-NEXT: [[DOTOFF_20:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_20]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_5:%.*]] = insertelement <64 x i32> [[DOTSLICE_4]], i32 [[DOTOFF_20]], i64 5
+; CHECK-NEXT: [[DOTOFF_PTR_24:%.*]] = add i32 [[TMP1]], 24
+; CHECK-NEXT: [[DOTOFF_24:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_24]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_6:%.*]] = insertelement <64 x i32> [[DOTSLICE_5]], i32 [[DOTOFF_24]], i64 6
+; CHECK-NEXT: [[DOTOFF_PTR_28:%.*]] = add i32 [[TMP1]], 28
+; CHECK-NEXT: [[DOTOFF_28:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_28]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_7:%.*]] = insertelement <64 x i32> [[DOTSLICE_6]], i32 [[DOTOFF_28]], i64 7
+; CHECK-NEXT: [[DOTOFF_PTR_32:%.*]] = add i32 [[TMP1]], 32
+; CHECK-NEXT: [[DOTOFF_32:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_32]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_8:%.*]] = insertelement <64 x i32> [[DOTSLICE_7]], i32 [[DOTOFF_32]], i64 8
+; CHECK-NEXT: [[DOTOFF_PTR_36:%.*]] = add i32 [[TMP1]], 36
+; CHECK-NEXT: [[DOTOFF_36:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_36]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_9:%.*]] = insertelement <64 x i32> [[DOTSLICE_8]], i32 [[DOTOFF_36]], i64 9
+; CHECK-NEXT: [[DOTOFF_PTR_40:%.*]] = add i32 [[TMP1]], 40
+; CHECK-NEXT: [[DOTOFF_40:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_40]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_10:%.*]] = insertelement <64 x i32> [[DOTSLICE_9]], i32 [[DOTOFF_40]], i64 10
+; CHECK-NEXT: [[DOTOFF_PTR_44:%.*]] = add i32 [[TMP1]], 44
+; CHECK-NEXT: [[DOTOFF_44:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_44]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_11:%.*]] = insertelement <64 x i32> [[DOTSLICE_10]], i32 [[DOTOFF_44]], i64 11
+; CHECK-NEXT: [[DOTOFF_PTR_48:%.*]] = add i32 [[TMP1]], 48
+; CHECK-NEXT: [[DOTOFF_48:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_48]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_12:%.*]] = insertelement <64 x i32> [[DOTSLICE_11]], i32 [[DOTOFF_48]], i64 12
+; CHECK-NEXT: [[DOTOFF_PTR_52:%.*]] = add i32 [[TMP1]], 52
+; CHECK-NEXT: [[DOTOFF_52:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_52]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_13:%.*]] = insertelement <64 x i32> [[DOTSLICE_12]], i32 [[DOTOFF_52]], i64 13
+; CHECK-NEXT: [[DOTOFF_PTR_56:%.*]] = add i32 [[TMP1]], 56
+; CHECK-NEXT: [[DOTOFF_56:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_56]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_14:%.*]] = insertelement <64 x i32> [[DOTSLICE_13]], i32 [[DOTOFF_56]], i64 14
+; CHECK-NEXT: [[DOTOFF_PTR_60:%.*]] = add i32 [[TMP1]], 60
+; CHECK-NEXT: [[DOTOFF_60:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_60]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_15:%.*]] = insertelement <64 x i32> [[DOTSLICE_14]], i32 [[DOTOFF_60]], i64 15
+; CHECK-NEXT: [[DOTOFF_PTR_64:%.*]] = add i32 [[TMP1]], 64
+; CHECK-NEXT: [[DOTOFF_64:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_64]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_16:%.*]] = insertelement <64 x i32> [[DOTSLICE_15]], i32 [[DOTOFF_64]], i64 16
+; CHECK-NEXT: [[DOTOFF_PTR_68:%.*]] = add i32 [[TMP1]], 68
+; CHECK-NEXT: [[DOTOFF_68:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_68]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_17:%.*]] = insertelement <64 x i32> [[DOTSLICE_16]], i32 [[DOTOFF_68]], i64 17
+; CHECK-NEXT: [[DOTOFF_PTR_72:%.*]] = add i32 [[TMP1]], 72
+; CHECK-NEXT: [[DOTOFF_72:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_72]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_18:%.*]] = insertelement <64 x i32> [[DOTSLICE_17]], i32 [[DOTOFF_72]], i64 18
+; CHECK-NEXT: [[DOTOFF_PTR_76:%.*]] = add i32 [[TMP1]], 76
+; CHECK-NEXT: [[DOTOFF_76:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_76]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_19:%.*]] = insertelement <64 x i32> [[DOTSLICE_18]], i32 [[DOTOFF_76]], i64 19
+; CHECK-NEXT: [[DOTOFF_PTR_80:%.*]] = add i32 [[TMP1]], 80
+; CHECK-NEXT: [[DOTOFF_80:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_80]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_20:%.*]] = insertelement <64 x i32> [[DOTSLICE_19]], i32 [[DOTOFF_80]], i64 20
+; CHECK-NEXT: [[DOTOFF_PTR_84:%.*]] = add i32 [[TMP1]], 84
+; CHECK-NEXT: [[DOTOFF_84:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_84]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_21:%.*]] = insertelement <64 x i32> [[DOTSLICE_20]], i32 [[DOTOFF_84]], i64 21
+; CHECK-NEXT: [[DOTOFF_PTR_88:%.*]] = add i32 [[TMP1]], 88
+; CHECK-NEXT: [[DOTOFF_88:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_88]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_22:%.*]] = insertelement <64 x i32> [[DOTSLICE_21]], i32 [[DOTOFF_88]], i64 22
+; CHECK-NEXT: [[DOTOFF_PTR_92:%.*]] = add i32 [[TMP1]], 92
+; CHECK-NEXT: [[DOTOFF_92:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_92]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_23:%.*]] = insertelement <64 x i32> [[DOTSLICE_22]], i32 [[DOTOFF_92]], i64 23
+; CHECK-NEXT: [[DOTOFF_PTR_96:%.*]] = add i32 [[TMP1]], 96
+; CHECK-NEXT: [[DOTOFF_96:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_96]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_24:%.*]] = insertelement <64 x i32> [[DOTSLICE_23]], i32 [[DOTOFF_96]], i64 24
+; CHECK-NEXT: [[DOTOFF_PTR_100:%.*]] = add i32 [[TMP1]], 100
+; CHECK-NEXT: [[DOTOFF_100:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_100]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_25:%.*]] = insertelement <64 x i32> [[DOTSLICE_24]], i32 [[DOTOFF_100]], i64 25
+; CHECK-NEXT: [[DOTOFF_PTR_104:%.*]] = add i32 [[TMP1]], 104
+; CHECK-NEXT: [[DOTOFF_104:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_104]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_26:%.*]] = insertelement <64 x i32> [[DOTSLICE_25]], i32 [[DOTOFF_104]], i64 26
+; CHECK-NEXT: [[DOTOFF_PTR_108:%.*]] = add i32 [[TMP1]], 108
+; CHECK-NEXT: [[DOTOFF_108:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_108]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_27:%.*]] = insertelement <64 x i32> [[DOTSLICE_26]], i32 [[DOTOFF_108]], i64 27
+; CHECK-NEXT: [[DOTOFF_PTR_112:%.*]] = add i32 [[TMP1]], 112
+; CHECK-NEXT: [[DOTOFF_112:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_112]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_28:%.*]] = insertelement <64 x i32> [[DOTSLICE_27]], i32 [[DOTOFF_112]], i64 28
+; CHECK-NEXT: [[DOTOFF_PTR_116:%.*]] = add i32 [[TMP1]], 116
+; CHECK-NEXT: [[DOTOFF_116:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_116]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_29:%.*]] = insertelement <64 x i32> [[DOTSLICE_28]], i32 [[DOTOFF_116]], i64 29
+; CHECK-NEXT: [[DOTOFF_PTR_120:%.*]] = add i32 [[TMP1]], 120
+; CHECK-NEXT: [[DOTOFF_120:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_120]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_30:%.*]] = insertelement <64 x i32> [[DOTSLICE_29]], i32 [[DOTOFF_120]], i64 30
+; CHECK-NEXT: [[DOTOFF_PTR_124:%.*]] = add i32 [[TMP1]], 124
+; CHECK-NEXT: [[DOTOFF_124:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_124]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_31:%.*]] = insertelement <64 x i32> [[DOTSLICE_30]], i32 [[DOTOFF_124]], i64 31
+; CHECK-NEXT: [[DOTOFF_PTR_128:%.*]] = add i32 [[TMP1]], 128
+; CHECK-NEXT: [[DOTOFF_128:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_128]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_32:%.*]] = insertelement <64 x i32> [[DOTSLICE_31]], i32 [[DOTOFF_128]], i64 32
+; CHECK-NEXT: [[DOTOFF_PTR_132:%.*]] = add i32 [[TMP1]], 132
+; CHECK-NEXT: [[DOTOFF_132:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_132]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_33:%.*]] = insertelement <64 x i32> [[DOTSLICE_32]], i32 [[DOTOFF_132]], i64 33
+; CHECK-NEXT: [[DOTOFF_PTR_136:%.*]] = add i32 [[TMP1]], 136
+; CHECK-NEXT: [[DOTOFF_136:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_136]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_34:%.*]] = insertelement <64 x i32> [[DOTSLICE_33]], i32 [[DOTOFF_136]], i64 34
+; CHECK-NEXT: [[DOTOFF_PTR_140:%.*]] = add i32 [[TMP1]], 140
+; CHECK-NEXT: [[DOTOFF_140:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_140]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_35:%.*]] = insertelement <64 x i32> [[DOTSLICE_34]], i32 [[DOTOFF_140]], i64 35
+; CHECK-NEXT: [[DOTOFF_PTR_144:%.*]] = add i32 [[TMP1]], 144
+; CHECK-NEXT: [[DOTOFF_144:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_144]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_36:%.*]] = insertelement <64 x i32> [[DOTSLICE_35]], i32 [[DOTOFF_144]], i64 36
+; CHECK-NEXT: [[DOTOFF_PTR_148:%.*]] = add i32 [[TMP1]], 148
+; CHECK-NEXT: [[DOTOFF_148:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_148]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_37:%.*]] = insertelement <64 x i32> [[DOTSLICE_36]], i32 [[DOTOFF_148]], i64 37
+; CHECK-NEXT: [[DOTOFF_PTR_152:%.*]] = add i32 [[TMP1]], 152
+; CHECK-NEXT: [[DOTOFF_152:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_152]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_38:%.*]] = insertelement <64 x i32> [[DOTSLICE_37]], i32 [[DOTOFF_152]], i64 38
+; CHECK-NEXT: [[DOTOFF_PTR_156:%.*]] = add i32 [[TMP1]], 156
+; CHECK-NEXT: [[DOTOFF_156:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_156]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_39:%.*]] = insertelement <64 x i32> [[DOTSLICE_38]], i32 [[DOTOFF_156]], i64 39
+; CHECK-NEXT: [[DOTOFF_PTR_160:%.*]] = add i32 [[TMP1]], 160
+; CHECK-NEXT: [[DOTOFF_160:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_160]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_40:%.*]] = insertelement <64 x i32> [[DOTSLICE_39]], i32 [[DOTOFF_160]], i64 40
+; CHECK-NEXT: [[DOTOFF_PTR_164:%.*]] = add i32 [[TMP1]], 164
+; CHECK-NEXT: [[DOTOFF_164:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_164]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_41:%.*]] = insertelement <64 x i32> [[DOTSLICE_40]], i32 [[DOTOFF_164]], i64 41
+; CHECK-NEXT: [[DOTOFF_PTR_168:%.*]] = add i32 [[TMP1]], 168
+; CHECK-NEXT: [[DOTOFF_168:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_168]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_42:%.*]] = insertelement <64 x i32> [[DOTSLICE_41]], i32 [[DOTOFF_168]], i64 42
+; CHECK-NEXT: [[DOTOFF_PTR_172:%.*]] = add i32 [[TMP1]], 172
+; CHECK-NEXT: [[DOTOFF_172:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_172]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_43:%.*]] = insertelement <64 x i32> [[DOTSLICE_42]], i32 [[DOTOFF_172]], i64 43
+; CHECK-NEXT: [[DOTOFF_PTR_176:%.*]] = add i32 [[TMP1]], 176
+; CHECK-NEXT: [[DOTOFF_176:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_176]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_44:%.*]] = insertelement <64 x i32> [[DOTSLICE_43]], i32 [[DOTOFF_176]], i64 44
+; CHECK-NEXT: [[DOTOFF_PTR_180:%.*]] = add i32 [[TMP1]], 180
+; CHECK-NEXT: [[DOTOFF_180:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_180]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_45:%.*]] = insertelement <64 x i32> [[DOTSLICE_44]], i32 [[DOTOFF_180]], i64 45
+; CHECK-NEXT: [[DOTOFF_PTR_184:%.*]] = add i32 [[TMP1]], 184
+; CHECK-NEXT: [[DOTOFF_184:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_184]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_46:%.*]] = insertelement <64 x i32> [[DOTSLICE_45]], i32 [[DOTOFF_184]], i64 46
+; CHECK-NEXT: [[DOTOFF_PTR_188:%.*]] = add i32 [[TMP1]], 188
+; CHECK-NEXT: [[DOTOFF_188:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_188]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_47:%.*]] = insertelement <64 x i32> [[DOTSLICE_46]], i32 [[DOTOFF_188]], i64 47
+; CHECK-NEXT: [[DOTOFF_PTR_192:%.*]] = add i32 [[TMP1]], 192
+; CHECK-NEXT: [[DOTOFF_192:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_192]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_48:%.*]] = insertelement <64 x i32> [[DOTSLICE_47]], i32 [[DOTOFF_192]], i64 48
+; CHECK-NEXT: [[DOTOFF_PTR_196:%.*]] = add i32 [[TMP1]], 196
+; CHECK-NEXT: [[DOTOFF_196:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_196]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_49:%.*]] = insertelement <64 x i32> [[DOTSLICE_48]], i32 [[DOTOFF_196]], i64 49
+; CHECK-NEXT: [[DOTOFF_PTR_200:%.*]] = add i32 [[TMP1]], 200
+; CHECK-NEXT: [[DOTOFF_200:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_200]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_50:%.*]] = insertelement <64 x i32> [[DOTSLICE_49]], i32 [[DOTOFF_200]], i64 50
+; CHECK-NEXT: [[DOTOFF_PTR_204:%.*]] = add i32 [[TMP1]], 204
+; CHECK-NEXT: [[DOTOFF_204:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_204]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_51:%.*]] = insertelement <64 x i32> [[DOTSLICE_50]], i32 [[DOTOFF_204]], i64 51
+; CHECK-NEXT: [[DOTOFF_PTR_208:%.*]] = add i32 [[TMP1]], 208
+; CHECK-NEXT: [[DOTOFF_208:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_208]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_52:%.*]] = insertelement <64 x i32> [[DOTSLICE_51]], i32 [[DOTOFF_208]], i64 52
+; CHECK-NEXT: [[DOTOFF_PTR_212:%.*]] = add i32 [[TMP1]], 212
+; CHECK-NEXT: [[DOTOFF_212:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_212]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_53:%.*]] = insertelement <64 x i32> [[DOTSLICE_52]], i32 [[DOTOFF_212]], i64 53
+; CHECK-NEXT: [[DOTOFF_PTR_216:%.*]] = add i32 [[TMP1]], 216
+; CHECK-NEXT: [[DOTOFF_216:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_216]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_54:%.*]] = insertelement <64 x i32> [[DOTSLICE_53]], i32 [[DOTOFF_216]], i64 54
+; CHECK-NEXT: [[DOTOFF_PTR_220:%.*]] = add i32 [[TMP1]], 220
+; CHECK-NEXT: [[DOTOFF_220:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_220]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_55:%.*]] = insertelement <64 x i32> [[DOTSLICE_54]], i32 [[DOTOFF_220]], i64 55
+; CHECK-NEXT: [[DOTOFF_PTR_224:%.*]] = add i32 [[TMP1]], 224
+; CHECK-NEXT: [[DOTOFF_224:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_224]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_56:%.*]] = insertelement <64 x i32> [[DOTSLICE_55]], i32 [[DOTOFF_224]], i64 56
+; CHECK-NEXT: [[DOTOFF_PTR_228:%.*]] = add i32 [[TMP1]], 228
+; CHECK-NEXT: [[DOTOFF_228:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_228]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_57:%.*]] = insertelement <64 x i32> [[DOTSLICE_56]], i32 [[DOTOFF_228]], i64 57
+; CHECK-NEXT: [[DOTOFF_PTR_232:%.*]] = add i32 [[TMP1]], 232
+; CHECK-NEXT: [[DOTOFF_232:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_232]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_58:%.*]] = insertelement <64 x i32> [[DOTSLICE_57]], i32 [[DOTOFF_232]], i64 58
+; CHECK-NEXT: [[DOTOFF_PTR_236:%.*]] = add i32 [[TMP1]], 236
+; CHECK-NEXT: [[DOTOFF_236:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_236]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_59:%.*]] = insertelement <64 x i32> [[DOTSLICE_58]], i32 [[DOTOFF_236]], i64 59
+; CHECK-NEXT: [[DOTOFF_PTR_240:%.*]] = add i32 [[TMP1]], 240
+; CHECK-NEXT: [[DOTOFF_240:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_240]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_60:%.*]] = insertelement <64 x i32> [[DOTSLICE_59]], i32 [[DOTOFF_240]], i64 60
+; CHECK-NEXT: [[DOTOFF_PTR_244:%.*]] = add i32 [[TMP1]], 244
+; CHECK-NEXT: [[DOTOFF_244:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_244]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_61:%.*]] = insertelement <64 x i32> [[DOTSLICE_60]], i32 [[DOTOFF_244]], i64 61
+; CHECK-NEXT: [[DOTOFF_PTR_248:%.*]] = add i32 [[TMP1]], 248
+; CHECK-NEXT: [[DOTOFF_248:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_248]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_62:%.*]] = insertelement <64 x i32> [[DOTSLICE_61]], i32 [[DOTOFF_248]], i64 62
+; CHECK-NEXT: [[DOTOFF_PTR_252:%.*]] = add i32 [[TMP1]], 252
+; CHECK-NEXT: [[DOTOFF_252:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_252]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <64 x i32> [[DOTSLICE_62]], i32 [[DOTOFF_252]], i64 63
; CHECK-NEXT: [[LOOP_INDEX_C1:%.*]] = trunc i64 [[LOOP_INDEX]] to i32
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX_C1]]
-; CHECK-NEXT: [[DOTSLICE_0:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_4:%.*]] = add nuw i32 [[TMP3]], 16
-; CHECK-NEXT: [[DOTSLICE_4:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_8:%.*]] = add nuw i32 [[TMP3]], 32
-; CHECK-NEXT: [[DOTSLICE_8:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_12:%.*]] = add nuw i32 [[TMP3]], 48
-; CHECK-NEXT: [[DOTSLICE_12:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_16:%.*]] = add nuw i32 [[TMP3]], 64
-; CHECK-NEXT: [[DOTSLICE_16:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_16]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_20:%.*]] = add nuw i32 [[TMP3]], 80
-; CHECK-NEXT: [[DOTSLICE_20:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_20]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_24:%.*]] = add nuw i32 [[TMP3]], 96
-; CHECK-NEXT: [[DOTSLICE_24:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_24]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_28:%.*]] = add nuw i32 [[TMP3]], 112
-; CHECK-NEXT: [[DOTSLICE_28:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_28]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_32:%.*]] = add nuw i32 [[TMP3]], 128
-; CHECK-NEXT: [[DOTSLICE_32:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_32]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_36:%.*]] = add nuw i32 [[TMP3]], 144
-; CHECK-NEXT: [[DOTSLICE_36:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 36, i32 37, i32 38, i32 39>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_36]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_40:%.*]] = add nuw i32 [[TMP3]], 160
-; CHECK-NEXT: [[DOTSLICE_40:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 40, i32 41, i32 42, i32 43>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_40]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_44:%.*]] = add nuw i32 [[TMP3]], 176
-; CHECK-NEXT: [[DOTSLICE_44:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 44, i32 45, i32 46, i32 47>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_48:%.*]] = add nuw i32 [[TMP3]], 192
-; CHECK-NEXT: [[DOTSLICE_48:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 48, i32 49, i32 50, i32 51>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_48]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_52:%.*]] = add nuw i32 [[TMP3]], 208
-; CHECK-NEXT: [[DOTSLICE_52:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 52, i32 53, i32 54, i32 55>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_52]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_56:%.*]] = add nuw i32 [[TMP3]], 224
-; CHECK-NEXT: [[DOTSLICE_56:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 56, i32 57, i32 58, i32 59>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_56]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_60:%.*]] = add nuw i32 [[TMP3]], 240
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTOFF_240]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP3]], 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP3]], 8
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP3]], 12
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP3]], 16
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_16]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_5:%.*]] = add i32 [[TMP3]], 20
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_20]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_5]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_6:%.*]] = add i32 [[TMP3]], 24
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_24]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_6]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_7:%.*]] = add i32 [[TMP3]], 28
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_28]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_7]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP3]], 32
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_32]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_9:%.*]] = add i32 [[TMP3]], 36
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_36]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_9]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_10:%.*]] = add i32 [[TMP3]], 40
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_40]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_10]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP3]], 44
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP3]], 48
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_48]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_13:%.*]] = add i32 [[TMP3]], 52
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_52]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_13]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_14:%.*]] = add i32 [[TMP3]], 56
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_56]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_14]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_15:%.*]] = add i32 [[TMP3]], 60
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_60]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_15]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP3]], 64
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_64]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_17:%.*]] = add i32 [[TMP3]], 68
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_68]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_17]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_18:%.*]] = add i32 [[TMP3]], 72
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_72]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_18]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_19:%.*]] = add i32 [[TMP3]], 76
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_76]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_19]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP3]], 80
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_80]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_21:%.*]] = add i32 [[TMP3]], 84
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_84]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_21]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP3]], 88
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_88]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_23:%.*]] = add i32 [[TMP3]], 92
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_92]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_23]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP3]], 96
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_96]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_25:%.*]] = add i32 [[TMP3]], 100
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_100]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_25]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_26:%.*]] = add i32 [[TMP3]], 104
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_104]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_26]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_27:%.*]] = add i32 [[TMP3]], 108
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_108]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_27]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP3]], 112
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_112]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_29:%.*]] = add i32 [[TMP3]], 116
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_116]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_29]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_30:%.*]] = add i32 [[TMP3]], 120
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_120]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_30]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_31:%.*]] = add i32 [[TMP3]], 124
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_124]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_31]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP3]], 128
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_128]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP3]], 132
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_132]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_34:%.*]] = add i32 [[TMP3]], 136
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_136]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_34]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_35:%.*]] = add i32 [[TMP3]], 140
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_140]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_35]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP3]], 144
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_144]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_37:%.*]] = add i32 [[TMP3]], 148
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_148]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_37]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_38:%.*]] = add i32 [[TMP3]], 152
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_152]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_38]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_39:%.*]] = add i32 [[TMP3]], 156
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_156]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_39]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP3]], 160
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_160]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_41:%.*]] = add i32 [[TMP3]], 164
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_164]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_41]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_42:%.*]] = add i32 [[TMP3]], 168
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_168]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_42]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_43:%.*]] = add i32 [[TMP3]], 172
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_172]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_43]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP3]], 176
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_176]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_45:%.*]] = add i32 [[TMP3]], 180
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_180]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_45]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_46:%.*]] = add i32 [[TMP3]], 184
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_184]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_46]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_47:%.*]] = add i32 [[TMP3]], 188
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_188]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_47]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP3]], 192
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_192]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_49:%.*]] = add i32 [[TMP3]], 196
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_196]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_49]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_50:%.*]] = add i32 [[TMP3]], 200
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_200]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_50]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_51:%.*]] = add i32 [[TMP3]], 204
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_204]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_51]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP3]], 208
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_208]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_53:%.*]] = add i32 [[TMP3]], 212
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_212]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_53]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_54:%.*]] = add i32 [[TMP3]], 216
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_216]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_54]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_55:%.*]] = add i32 [[TMP3]], 220
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_220]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_55]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP3]], 224
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_224]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_57:%.*]] = add i32 [[TMP3]], 228
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_228]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_57]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_58:%.*]] = add i32 [[TMP3]], 232
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_232]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_58]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_59:%.*]] = add i32 [[TMP3]], 236
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_236]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_59]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP3]], 240
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_240]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_61:%.*]] = add i32 [[TMP3]], 244
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_244]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_61]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_62:%.*]] = add i32 [[TMP3]], 248
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_248]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_62]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_63:%.*]] = add i32 [[TMP3]], 252
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_252]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_63]], i32 0, i32 0)
; CHECK-NEXT: [[TMP4]] = add i64 [[LOOP_INDEX]], 256
; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 8192
; CHECK-NEXT: br i1 [[TMP5]], label %[[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]], label %[[STATIC_MEMCPY_POST_LOOP_EXPANSION:.*]]
@@ -344,12 +585,44 @@ define void @memcpy_known_i32_volatile(ptr addrspace(7) inreg %src, ptr addrspac
; CHECK-NEXT: [[DST_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[DST]], 1
; CHECK-NEXT: [[SRC_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 0
; CHECK-NEXT: [[SRC_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 1
-; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 -2147483648)
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[TMP1]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTOFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTSLICE_0:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_0]], i64 0
+; CHECK-NEXT: [[DOTOFF_PTR_4:%.*]] = add i32 [[SRC_OFF]], 4
+; CHECK-NEXT: [[DOTOFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_4]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTSLICE_1:%.*]] = insertelement <4 x i32> [[DOTSLICE_0]], i32 [[DOTOFF_4]], i64 1
+; CHECK-NEXT: [[DOTOFF_PTR_8:%.*]] = add i32 [[SRC_OFF]], 8
+; CHECK-NEXT: [[DOTOFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_8]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTSLICE_2:%.*]] = insertelement <4 x i32> [[DOTSLICE_1]], i32 [[DOTOFF_8]], i64 2
+; CHECK-NEXT: [[DOTOFF_PTR_12:%.*]] = add i32 [[SRC_OFF]], 12
+; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[DST_OFF]], 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[DST_OFF]], 8
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[DST_OFF]], 12
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 -2147483648)
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[SRC_OFF]], 16
-; CHECK-NEXT: [[TMP3:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTOFF_01:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTSLICE_02:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_01]], i64 0
+; CHECK-NEXT: [[DOTOFF_PTR_43:%.*]] = add i32 [[TMP2]], 4
+; CHECK-NEXT: [[DOTOFF_44:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_43]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTSLICE_15:%.*]] = insertelement <4 x i32> [[DOTSLICE_02]], i32 [[DOTOFF_44]], i64 1
+; CHECK-NEXT: [[DOTOFF_PTR_86:%.*]] = add i32 [[TMP2]], 8
+; CHECK-NEXT: [[DOTOFF_87:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_86]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTSLICE_28:%.*]] = insertelement <4 x i32> [[DOTSLICE_15]], i32 [[DOTOFF_87]], i64 2
+; CHECK-NEXT: [[DOTOFF_PTR_129:%.*]] = add i32 [[TMP2]], 12
+; CHECK-NEXT: [[DOTOFF_1210:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_129]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[DOTSLICE_28]], i32 [[DOTOFF_1210]], i64 3
; CHECK-NEXT: [[TMP4:%.*]] = add nuw i32 [[DST_OFF]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[TMP3]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_01]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTPART_111:%.*]] = add i32 [[TMP4]], 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_111]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTPART_212:%.*]] = add i32 [[TMP4]], 8
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_87]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_212]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTPART_313:%.*]] = add i32 [[TMP4]], 12
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_1210]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_313]], i32 0, i32 -2147483648)
; CHECK-NEXT: ret void
;
call void @llvm.memcpy.p7.p7.i32(ptr addrspace(7) %dst, ptr addrspace(7) %src, i32 32, i1 true)
@@ -370,9 +643,25 @@ define void @memcpy_unknown(ptr addrspace(7) inreg %src, ptr addrspace(7) inreg
; CHECK: [[DYNAMIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]]:
; CHECK-NEXT: [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP7:%.*]], %[[DYNAMIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[SRC_OFF]], [[LOOP_INDEX]]
-; CHECK-NEXT: [[TMP5:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP4]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP4]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_0:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_0]], i64 0
+; CHECK-NEXT: [[DOTOFF_PTR_4:%.*]] = add i32 [[TMP4]], 4
+; CHECK-NEXT: [[DOTOFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_4]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_1:%.*]] = insertelement <4 x i32> [[DOTSLICE_0]], i32 [[DOTOFF_4]], i64 1
+; CHECK-NEXT: [[DOTOFF_PTR_8:%.*]] = add i32 [[TMP4]], 8
+; CHECK-NEXT: [[DOTOFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_8]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_2:%.*]] = insertelement <4 x i32> [[DOTSLICE_1]], i32 [[DOTOFF_8]], i64 2
+; CHECK-NEXT: [[DOTOFF_PTR_12:%.*]] = add i32 [[TMP4]], 12
+; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[TMP5]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP6]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP6]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP6]], 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP6]], 8
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP6]], 12
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
; CHECK-NEXT: [[TMP7]] = add i32 [[LOOP_INDEX]], 16
; CHECK-NEXT: [[TMP8:%.*]] = icmp ult i32 [[TMP7]], [[TMP2]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[DYNAMIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]], label %[[DYNAMIC_MEMCPY_LOOP_EXPANSION_RESIDUAL_COND]]
@@ -409,49 +698,49 @@ define void @memcpy_known_p1_to_p7(ptr addrspace(1) inreg %src, ptr addrspace(7)
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX]]
; CHECK-NEXT: [[DOTSLICE_0:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_0]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_4:%.*]] = add nuw i32 [[TMP3]], 16
+; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP3]], 16
; CHECK-NEXT: [[DOTSLICE_4:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_4]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_8:%.*]] = add nuw i32 [[TMP3]], 32
+; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP3]], 32
; CHECK-NEXT: [[DOTSLICE_8:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_8]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_12:%.*]] = add nuw i32 [[TMP3]], 48
+; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP3]], 48
; CHECK-NEXT: [[DOTSLICE_12:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_12]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_16:%.*]] = add nuw i32 [[TMP3]], 64
+; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP3]], 64
; CHECK-NEXT: [[DOTSLICE_16:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_16]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_20:%.*]] = add nuw i32 [[TMP3]], 80
+; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP3]], 80
; CHECK-NEXT: [[DOTSLICE_20:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_20]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_24:%.*]] = add nuw i32 [[TMP3]], 96
+; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP3]], 96
; CHECK-NEXT: [[DOTSLICE_24:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_24]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_28:%.*]] = add nuw i32 [[TMP3]], 112
+; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP3]], 112
; CHECK-NEXT: [[DOTSLICE_28:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_28]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_32:%.*]] = add nuw i32 [[TMP3]], 128
+; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP3]], 128
; CHECK-NEXT: [[DOTSLICE_32:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_32]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_36:%.*]] = add nuw i32 [[TMP3]], 144
+; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP3]], 144
; CHECK-NEXT: [[DOTSLICE_36:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 36, i32 37, i32 38, i32 39>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_36]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_40:%.*]] = add nuw i32 [[TMP3]], 160
+; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP3]], 160
; CHECK-NEXT: [[DOTSLICE_40:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 40, i32 41, i32 42, i32 43>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_40]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_44:%.*]] = add nuw i32 [[TMP3]], 176
+; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP3]], 176
; CHECK-NEXT: [[DOTSLICE_44:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 44, i32 45, i32 46, i32 47>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_44]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_48:%.*]] = add nuw i32 [[TMP3]], 192
+; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP3]], 192
; CHECK-NEXT: [[DOTSLICE_48:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 48, i32 49, i32 50, i32 51>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_48]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_52:%.*]] = add nuw i32 [[TMP3]], 208
+; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP3]], 208
; CHECK-NEXT: [[DOTSLICE_52:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 52, i32 53, i32 54, i32 55>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_52]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_56:%.*]] = add nuw i32 [[TMP3]], 224
+; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP3]], 224
; CHECK-NEXT: [[DOTSLICE_56:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 56, i32 57, i32 58, i32 59>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_56]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_60:%.*]] = add nuw i32 [[TMP3]], 240
+; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP3]], 240
; CHECK-NEXT: [[DOTSLICE_60:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_60]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
; CHECK-NEXT: [[TMP4]] = add i32 [[LOOP_INDEX]], 256
@@ -476,63 +765,63 @@ define void @memcpy_known_p7_to_p1(ptr addrspace(7) inreg %src, ptr addrspace(1)
; CHECK-NEXT: [[DOTOFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[TMP1]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_0:%.*]] = shufflevector <4 x i32> [[DOTOFF_0]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_0:%.*]] = shufflevector <64 x i32> poison, <64 x i32> [[DOTEXT_0]], <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_16:%.*]] = add nuw i32 [[TMP1]], 16
+; CHECK-NEXT: [[DOTOFF_PTR_16:%.*]] = add i32 [[TMP1]], 16
; CHECK-NEXT: [[DOTOFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_16]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_4:%.*]] = shufflevector <4 x i32> [[DOTOFF_16]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_4:%.*]] = shufflevector <64 x i32> [[DOTPARTS_0]], <64 x i32> [[DOTEXT_4]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 64, i32 65, i32 66, i32 67, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_32:%.*]] = add nuw i32 [[TMP1]], 32
+; CHECK-NEXT: [[DOTOFF_PTR_32:%.*]] = add i32 [[TMP1]], 32
; CHECK-NEXT: [[DOTOFF_32:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_32]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_8:%.*]] = shufflevector <4 x i32> [[DOTOFF_32]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_8:%.*]] = shufflevector <64 x i32> [[DOTPARTS_4]], <64 x i32> [[DOTEXT_8]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 64, i32 65, i32 66, i32 67, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_48:%.*]] = add nuw i32 [[TMP1]], 48
+; CHECK-NEXT: [[DOTOFF_PTR_48:%.*]] = add i32 [[TMP1]], 48
; CHECK-NEXT: [[DOTOFF_48:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_48]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_12:%.*]] = shufflevector <4 x i32> [[DOTOFF_48]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_12:%.*]] = shufflevector <64 x i32> [[DOTPARTS_8]], <64 x i32> [[DOTEXT_12]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 64, i32 65, i32 66, i32 67, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_64:%.*]] = add nuw i32 [[TMP1]], 64
+; CHECK-NEXT: [[DOTOFF_PTR_64:%.*]] = add i32 [[TMP1]], 64
; CHECK-NEXT: [[DOTOFF_64:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_64]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_16:%.*]] = shufflevector <4 x i32> [[DOTOFF_64]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_16:%.*]] = shufflevector <64 x i32> [[DOTPARTS_12]], <64 x i32> [[DOTEXT_16]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_80:%.*]] = add nuw i32 [[TMP1]], 80
+; CHECK-NEXT: [[DOTOFF_PTR_80:%.*]] = add i32 [[TMP1]], 80
; CHECK-NEXT: [[DOTOFF_80:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_80]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_20:%.*]] = shufflevector <4 x i32> [[DOTOFF_80]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_20:%.*]] = shufflevector <64 x i32> [[DOTPARTS_16]], <64 x i32> [[DOTEXT_20]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 64, i32 65, i32 66, i32 67, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_96:%.*]] = add nuw i32 [[TMP1]], 96
+; CHECK-NEXT: [[DOTOFF_PTR_96:%.*]] = add i32 [[TMP1]], 96
; CHECK-NEXT: [[DOTOFF_96:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_96]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_24:%.*]] = shufflevector <4 x i32> [[DOTOFF_96]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_24:%.*]] = shufflevector <64 x i32> [[DOTPARTS_20]], <64 x i32> [[DOTEXT_24]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 64, i32 65, i32 66, i32 67, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_112:%.*]] = add nuw i32 [[TMP1]], 112
+; CHECK-NEXT: [[DOTOFF_PTR_112:%.*]] = add i32 [[TMP1]], 112
; CHECK-NEXT: [[DOTOFF_112:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_112]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_28:%.*]] = shufflevector <4 x i32> [[DOTOFF_112]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_28:%.*]] = shufflevector <64 x i32> [[DOTPARTS_24]], <64 x i32> [[DOTEXT_28]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 64, i32 65, i32 66, i32 67, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_128:%.*]] = add nuw i32 [[TMP1]], 128
+; CHECK-NEXT: [[DOTOFF_PTR_128:%.*]] = add i32 [[TMP1]], 128
; CHECK-NEXT: [[DOTOFF_128:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_128]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_32:%.*]] = shufflevector <4 x i32> [[DOTOFF_128]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_32:%.*]] = shufflevector <64 x i32> [[DOTPARTS_28]], <64 x i32> [[DOTEXT_32]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 64, i32 65, i32 66, i32 67, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_144:%.*]] = add nuw i32 [[TMP1]], 144
+; CHECK-NEXT: [[DOTOFF_PTR_144:%.*]] = add i32 [[TMP1]], 144
; CHECK-NEXT: [[DOTOFF_144:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_144]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_36:%.*]] = shufflevector <4 x i32> [[DOTOFF_144]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_36:%.*]] = shufflevector <64 x i32> [[DOTPARTS_32]], <64 x i32> [[DOTEXT_36]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 64, i32 65, i32 66, i32 67, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_160:%.*]] = add nuw i32 [[TMP1]], 160
+; CHECK-NEXT: [[DOTOFF_PTR_160:%.*]] = add i32 [[TMP1]], 160
; CHECK-NEXT: [[DOTOFF_160:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_160]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_40:%.*]] = shufflevector <4 x i32> [[DOTOFF_160]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_40:%.*]] = shufflevector <64 x i32> [[DOTPARTS_36]], <64 x i32> [[DOTEXT_40]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 64, i32 65, i32 66, i32 67, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_176:%.*]] = add nuw i32 [[TMP1]], 176
+; CHECK-NEXT: [[DOTOFF_PTR_176:%.*]] = add i32 [[TMP1]], 176
; CHECK-NEXT: [[DOTOFF_176:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_176]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_44:%.*]] = shufflevector <4 x i32> [[DOTOFF_176]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_44:%.*]] = shufflevector <64 x i32> [[DOTPARTS_40]], <64 x i32> [[DOTEXT_44]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 64, i32 65, i32 66, i32 67, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_192:%.*]] = add nuw i32 [[TMP1]], 192
+; CHECK-NEXT: [[DOTOFF_PTR_192:%.*]] = add i32 [[TMP1]], 192
; CHECK-NEXT: [[DOTOFF_192:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_192]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_48:%.*]] = shufflevector <4 x i32> [[DOTOFF_192]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_48:%.*]] = shufflevector <64 x i32> [[DOTPARTS_44]], <64 x i32> [[DOTEXT_48]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 64, i32 65, i32 66, i32 67, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_208:%.*]] = add nuw i32 [[TMP1]], 208
+; CHECK-NEXT: [[DOTOFF_PTR_208:%.*]] = add i32 [[TMP1]], 208
; CHECK-NEXT: [[DOTOFF_208:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_208]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_52:%.*]] = shufflevector <4 x i32> [[DOTOFF_208]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_52:%.*]] = shufflevector <64 x i32> [[DOTPARTS_48]], <64 x i32> [[DOTEXT_52]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 64, i32 65, i32 66, i32 67, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_224:%.*]] = add nuw i32 [[TMP1]], 224
+; CHECK-NEXT: [[DOTOFF_PTR_224:%.*]] = add i32 [[TMP1]], 224
; CHECK-NEXT: [[DOTOFF_224:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_224]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_56:%.*]] = shufflevector <4 x i32> [[DOTOFF_224]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_56:%.*]] = shufflevector <64 x i32> [[DOTPARTS_52]], <64 x i32> [[DOTEXT_56]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 64, i32 65, i32 66, i32 67, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_240:%.*]] = add nuw i32 [[TMP1]], 240
+; CHECK-NEXT: [[DOTOFF_PTR_240:%.*]] = add i32 [[TMP1]], 240
; CHECK-NEXT: [[DOTOFF_240:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_240]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_60:%.*]] = shufflevector <4 x i32> [[DOTOFF_240]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <64 x i32> [[DOTPARTS_56]], <64 x i32> [[DOTEXT_60]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 64, i32 65, i32 66, i32 67>
@@ -589,63 +878,63 @@ define void @memcpy_known_p7_to_p3_long(ptr addrspace(7) inreg %src, ptr addrspa
; CHECK-NEXT: [[DOTOFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[TMP1]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_0:%.*]] = shufflevector <4 x i32> [[DOTOFF_0]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_0:%.*]] = shufflevector <64 x i32> poison, <64 x i32> [[DOTEXT_0]], <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_16:%.*]] = add nuw i32 [[TMP1]], 16
+; CHECK-NEXT: [[DOTOFF_PTR_16:%.*]] = add i32 [[TMP1]], 16
; CHECK-NEXT: [[DOTOFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_16]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_4:%.*]] = shufflevector <4 x i32> [[DOTOFF_16]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_4:%.*]] = shufflevector <64 x i32> [[DOTPARTS_0]], <64 x i32> [[DOTEXT_4]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 64, i32 65, i32 66, i32 67, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_32:%.*]] = add nuw i32 [[TMP1]], 32
+; CHECK-NEXT: [[DOTOFF_PTR_32:%.*]] = add i32 [[TMP1]], 32
; CHECK-NEXT: [[DOTOFF_32:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_32]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_8:%.*]] = shufflevector <4 x i32> [[DOTOFF_32]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_8:%.*]] = shufflevector <64 x i32> [[DOTPARTS_4]], <64 x i32> [[DOTEXT_8]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 64, i32 65, i32 66, i32 67, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_48:%.*]] = add nuw i32 [[TMP1]], 48
+; CHECK-NEXT: [[DOTOFF_PTR_48:%.*]] = add i32 [[TMP1]], 48
; CHECK-NEXT: [[DOTOFF_48:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_48]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_12:%.*]] = shufflevector <4 x i32> [[DOTOFF_48]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_12:%.*]] = shufflevector <64 x i32> [[DOTPARTS_8]], <64 x i32> [[DOTEXT_12]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 64, i32 65, i32 66, i32 67, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_64:%.*]] = add nuw i32 [[TMP1]], 64
+; CHECK-NEXT: [[DOTOFF_PTR_64:%.*]] = add i32 [[TMP1]], 64
; CHECK-NEXT: [[DOTOFF_64:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_64]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_16:%.*]] = shufflevector <4 x i32> [[DOTOFF_64]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_16:%.*]] = shufflevector <64 x i32> [[DOTPARTS_12]], <64 x i32> [[DOTEXT_16]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_80:%.*]] = add nuw i32 [[TMP1]], 80
+; CHECK-NEXT: [[DOTOFF_PTR_80:%.*]] = add i32 [[TMP1]], 80
; CHECK-NEXT: [[DOTOFF_80:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_80]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_20:%.*]] = shufflevector <4 x i32> [[DOTOFF_80]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_20:%.*]] = shufflevector <64 x i32> [[DOTPARTS_16]], <64 x i32> [[DOTEXT_20]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 64, i32 65, i32 66, i32 67, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_96:%.*]] = add nuw i32 [[TMP1]], 96
+; CHECK-NEXT: [[DOTOFF_PTR_96:%.*]] = add i32 [[TMP1]], 96
; CHECK-NEXT: [[DOTOFF_96:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_96]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_24:%.*]] = shufflevector <4 x i32> [[DOTOFF_96]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_24:%.*]] = shufflevector <64 x i32> [[DOTPARTS_20]], <64 x i32> [[DOTEXT_24]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 64, i32 65, i32 66, i32 67, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_112:%.*]] = add nuw i32 [[TMP1]], 112
+; CHECK-NEXT: [[DOTOFF_PTR_112:%.*]] = add i32 [[TMP1]], 112
; CHECK-NEXT: [[DOTOFF_112:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_112]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_28:%.*]] = shufflevector <4 x i32> [[DOTOFF_112]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_28:%.*]] = shufflevector <64 x i32> [[DOTPARTS_24]], <64 x i32> [[DOTEXT_28]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 64, i32 65, i32 66, i32 67, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_128:%.*]] = add nuw i32 [[TMP1]], 128
+; CHECK-NEXT: [[DOTOFF_PTR_128:%.*]] = add i32 [[TMP1]], 128
; CHECK-NEXT: [[DOTOFF_128:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_128]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_32:%.*]] = shufflevector <4 x i32> [[DOTOFF_128]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_32:%.*]] = shufflevector <64 x i32> [[DOTPARTS_28]], <64 x i32> [[DOTEXT_32]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 64, i32 65, i32 66, i32 67, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_144:%.*]] = add nuw i32 [[TMP1]], 144
+; CHECK-NEXT: [[DOTOFF_PTR_144:%.*]] = add i32 [[TMP1]], 144
; CHECK-NEXT: [[DOTOFF_144:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_144]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_36:%.*]] = shufflevector <4 x i32> [[DOTOFF_144]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_36:%.*]] = shufflevector <64 x i32> [[DOTPARTS_32]], <64 x i32> [[DOTEXT_36]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 64, i32 65, i32 66, i32 67, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_160:%.*]] = add nuw i32 [[TMP1]], 160
+; CHECK-NEXT: [[DOTOFF_PTR_160:%.*]] = add i32 [[TMP1]], 160
; CHECK-NEXT: [[DOTOFF_160:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_160]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_40:%.*]] = shufflevector <4 x i32> [[DOTOFF_160]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_40:%.*]] = shufflevector <64 x i32> [[DOTPARTS_36]], <64 x i32> [[DOTEXT_40]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 64, i32 65, i32 66, i32 67, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_176:%.*]] = add nuw i32 [[TMP1]], 176
+; CHECK-NEXT: [[DOTOFF_PTR_176:%.*]] = add i32 [[TMP1]], 176
; CHECK-NEXT: [[DOTOFF_176:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_176]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_44:%.*]] = shufflevector <4 x i32> [[DOTOFF_176]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_44:%.*]] = shufflevector <64 x i32> [[DOTPARTS_40]], <64 x i32> [[DOTEXT_44]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 64, i32 65, i32 66, i32 67, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_192:%.*]] = add nuw i32 [[TMP1]], 192
+; CHECK-NEXT: [[DOTOFF_PTR_192:%.*]] = add i32 [[TMP1]], 192
; CHECK-NEXT: [[DOTOFF_192:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_192]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_48:%.*]] = shufflevector <4 x i32> [[DOTOFF_192]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_48:%.*]] = shufflevector <64 x i32> [[DOTPARTS_44]], <64 x i32> [[DOTEXT_48]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 64, i32 65, i32 66, i32 67, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_208:%.*]] = add nuw i32 [[TMP1]], 208
+; CHECK-NEXT: [[DOTOFF_PTR_208:%.*]] = add i32 [[TMP1]], 208
; CHECK-NEXT: [[DOTOFF_208:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_208]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_52:%.*]] = shufflevector <4 x i32> [[DOTOFF_208]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_52:%.*]] = shufflevector <64 x i32> [[DOTPARTS_48]], <64 x i32> [[DOTEXT_52]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 64, i32 65, i32 66, i32 67, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_224:%.*]] = add nuw i32 [[TMP1]], 224
+; CHECK-NEXT: [[DOTOFF_PTR_224:%.*]] = add i32 [[TMP1]], 224
; CHECK-NEXT: [[DOTOFF_224:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_224]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_56:%.*]] = shufflevector <4 x i32> [[DOTOFF_224]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_56:%.*]] = shufflevector <64 x i32> [[DOTPARTS_52]], <64 x i32> [[DOTEXT_56]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 64, i32 65, i32 66, i32 67, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_240:%.*]] = add nuw i32 [[TMP1]], 240
+; CHECK-NEXT: [[DOTOFF_PTR_240:%.*]] = add i32 [[TMP1]], 240
; CHECK-NEXT: [[DOTOFF_240:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_240]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_60:%.*]] = shufflevector <4 x i32> [[DOTOFF_240]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <64 x i32> [[DOTPARTS_56]], <64 x i32> [[DOTEXT_60]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 64, i32 65, i32 66, i32 67>
@@ -683,112 +972,112 @@ define void @memcpy.inline_known(ptr addrspace(7) inreg %src, ptr addrspace(7) i
; CHECK-NEXT: [[DOTOFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[TMP1]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_0:%.*]] = shufflevector <4 x i32> [[DOTOFF_0]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_0:%.*]] = shufflevector <64 x i32> poison, <64 x i32> [[DOTEXT_0]], <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_16:%.*]] = add nuw i32 [[TMP1]], 16
+; CHECK-NEXT: [[DOTOFF_PTR_16:%.*]] = add i32 [[TMP1]], 16
; CHECK-NEXT: [[DOTOFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_16]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_4:%.*]] = shufflevector <4 x i32> [[DOTOFF_16]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_4:%.*]] = shufflevector <64 x i32> [[DOTPARTS_0]], <64 x i32> [[DOTEXT_4]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 64, i32 65, i32 66, i32 67, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_32:%.*]] = add nuw i32 [[TMP1]], 32
+; CHECK-NEXT: [[DOTOFF_PTR_32:%.*]] = add i32 [[TMP1]], 32
; CHECK-NEXT: [[DOTOFF_32:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_32]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_8:%.*]] = shufflevector <4 x i32> [[DOTOFF_32]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_8:%.*]] = shufflevector <64 x i32> [[DOTPARTS_4]], <64 x i32> [[DOTEXT_8]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 64, i32 65, i32 66, i32 67, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_48:%.*]] = add nuw i32 [[TMP1]], 48
+; CHECK-NEXT: [[DOTOFF_PTR_48:%.*]] = add i32 [[TMP1]], 48
; CHECK-NEXT: [[DOTOFF_48:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_48]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_12:%.*]] = shufflevector <4 x i32> [[DOTOFF_48]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_12:%.*]] = shufflevector <64 x i32> [[DOTPARTS_8]], <64 x i32> [[DOTEXT_12]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 64, i32 65, i32 66, i32 67, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_64:%.*]] = add nuw i32 [[TMP1]], 64
+; CHECK-NEXT: [[DOTOFF_PTR_64:%.*]] = add i32 [[TMP1]], 64
; CHECK-NEXT: [[DOTOFF_64:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_64]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_16:%.*]] = shufflevector <4 x i32> [[DOTOFF_64]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_16:%.*]] = shufflevector <64 x i32> [[DOTPARTS_12]], <64 x i32> [[DOTEXT_16]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_80:%.*]] = add nuw i32 [[TMP1]], 80
+; CHECK-NEXT: [[DOTOFF_PTR_80:%.*]] = add i32 [[TMP1]], 80
; CHECK-NEXT: [[DOTOFF_80:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_80]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_20:%.*]] = shufflevector <4 x i32> [[DOTOFF_80]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_20:%.*]] = shufflevector <64 x i32> [[DOTPARTS_16]], <64 x i32> [[DOTEXT_20]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 64, i32 65, i32 66, i32 67, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_96:%.*]] = add nuw i32 [[TMP1]], 96
+; CHECK-NEXT: [[DOTOFF_PTR_96:%.*]] = add i32 [[TMP1]], 96
; CHECK-NEXT: [[DOTOFF_96:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_96]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_24:%.*]] = shufflevector <4 x i32> [[DOTOFF_96]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_24:%.*]] = shufflevector <64 x i32> [[DOTPARTS_20]], <64 x i32> [[DOTEXT_24]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 64, i32 65, i32 66, i32 67, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_112:%.*]] = add nuw i32 [[TMP1]], 112
+; CHECK-NEXT: [[DOTOFF_PTR_112:%.*]] = add i32 [[TMP1]], 112
; CHECK-NEXT: [[DOTOFF_112:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_112]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_28:%.*]] = shufflevector <4 x i32> [[DOTOFF_112]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_28:%.*]] = shufflevector <64 x i32> [[DOTPARTS_24]], <64 x i32> [[DOTEXT_28]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 64, i32 65, i32 66, i32 67, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_128:%.*]] = add nuw i32 [[TMP1]], 128
+; CHECK-NEXT: [[DOTOFF_PTR_128:%.*]] = add i32 [[TMP1]], 128
; CHECK-NEXT: [[DOTOFF_128:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_128]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_32:%.*]] = shufflevector <4 x i32> [[DOTOFF_128]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_32:%.*]] = shufflevector <64 x i32> [[DOTPARTS_28]], <64 x i32> [[DOTEXT_32]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 64, i32 65, i32 66, i32 67, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_144:%.*]] = add nuw i32 [[TMP1]], 144
+; CHECK-NEXT: [[DOTOFF_PTR_144:%.*]] = add i32 [[TMP1]], 144
; CHECK-NEXT: [[DOTOFF_144:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_144]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_36:%.*]] = shufflevector <4 x i32> [[DOTOFF_144]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_36:%.*]] = shufflevector <64 x i32> [[DOTPARTS_32]], <64 x i32> [[DOTEXT_36]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 64, i32 65, i32 66, i32 67, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_160:%.*]] = add nuw i32 [[TMP1]], 160
+; CHECK-NEXT: [[DOTOFF_PTR_160:%.*]] = add i32 [[TMP1]], 160
; CHECK-NEXT: [[DOTOFF_160:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_160]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_40:%.*]] = shufflevector <4 x i32> [[DOTOFF_160]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_40:%.*]] = shufflevector <64 x i32> [[DOTPARTS_36]], <64 x i32> [[DOTEXT_40]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 64, i32 65, i32 66, i32 67, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_176:%.*]] = add nuw i32 [[TMP1]], 176
+; CHECK-NEXT: [[DOTOFF_PTR_176:%.*]] = add i32 [[TMP1]], 176
; CHECK-NEXT: [[DOTOFF_176:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_176]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_44:%.*]] = shufflevector <4 x i32> [[DOTOFF_176]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_44:%.*]] = shufflevector <64 x i32> [[DOTPARTS_40]], <64 x i32> [[DOTEXT_44]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 64, i32 65, i32 66, i32 67, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_192:%.*]] = add nuw i32 [[TMP1]], 192
+; CHECK-NEXT: [[DOTOFF_PTR_192:%.*]] = add i32 [[TMP1]], 192
; CHECK-NEXT: [[DOTOFF_192:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_192]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_48:%.*]] = shufflevector <4 x i32> [[DOTOFF_192]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_48:%.*]] = shufflevector <64 x i32> [[DOTPARTS_44]], <64 x i32> [[DOTEXT_48]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 64, i32 65, i32 66, i32 67, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_208:%.*]] = add nuw i32 [[TMP1]], 208
+; CHECK-NEXT: [[DOTOFF_PTR_208:%.*]] = add i32 [[TMP1]], 208
; CHECK-NEXT: [[DOTOFF_208:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_208]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_52:%.*]] = shufflevector <4 x i32> [[DOTOFF_208]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_52:%.*]] = shufflevector <64 x i32> [[DOTPARTS_48]], <64 x i32> [[DOTEXT_52]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 64, i32 65, i32 66, i32 67, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_224:%.*]] = add nuw i32 [[TMP1]], 224
+; CHECK-NEXT: [[DOTOFF_PTR_224:%.*]] = add i32 [[TMP1]], 224
; CHECK-NEXT: [[DOTOFF_224:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_224]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_56:%.*]] = shufflevector <4 x i32> [[DOTOFF_224]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_56:%.*]] = shufflevector <64 x i32> [[DOTPARTS_52]], <64 x i32> [[DOTEXT_56]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 64, i32 65, i32 66, i32 67, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_240:%.*]] = add nuw i32 [[TMP1]], 240
+; CHECK-NEXT: [[DOTOFF_PTR_240:%.*]] = add i32 [[TMP1]], 240
; CHECK-NEXT: [[DOTOFF_240:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_240]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_60:%.*]] = shufflevector <4 x i32> [[DOTOFF_240]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <64 x i32> [[DOTPARTS_56]], <64 x i32> [[DOTEXT_60]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 64, i32 65, i32 66, i32 67>
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX]]
; CHECK-NEXT: [[DOTSLICE_0:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_0]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_4:%.*]] = add nuw i32 [[TMP3]], 16
+; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP3]], 16
; CHECK-NEXT: [[DOTSLICE_4:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_4]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_8:%.*]] = add nuw i32 [[TMP3]], 32
+; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP3]], 32
; CHECK-NEXT: [[DOTSLICE_8:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_8]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_12:%.*]] = add nuw i32 [[TMP3]], 48
+; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP3]], 48
; CHECK-NEXT: [[DOTSLICE_12:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_12]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_16:%.*]] = add nuw i32 [[TMP3]], 64
+; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP3]], 64
; CHECK-NEXT: [[DOTSLICE_16:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_16]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_20:%.*]] = add nuw i32 [[TMP3]], 80
+; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP3]], 80
; CHECK-NEXT: [[DOTSLICE_20:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_20]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_24:%.*]] = add nuw i32 [[TMP3]], 96
+; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP3]], 96
; CHECK-NEXT: [[DOTSLICE_24:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_24]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_28:%.*]] = add nuw i32 [[TMP3]], 112
+; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP3]], 112
; CHECK-NEXT: [[DOTSLICE_28:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_28]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_32:%.*]] = add nuw i32 [[TMP3]], 128
+; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP3]], 128
; CHECK-NEXT: [[DOTSLICE_32:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_32]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_36:%.*]] = add nuw i32 [[TMP3]], 144
+; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP3]], 144
; CHECK-NEXT: [[DOTSLICE_36:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 36, i32 37, i32 38, i32 39>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_36]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_40:%.*]] = add nuw i32 [[TMP3]], 160
+; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP3]], 160
; CHECK-NEXT: [[DOTSLICE_40:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 40, i32 41, i32 42, i32 43>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_40]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_44:%.*]] = add nuw i32 [[TMP3]], 176
+; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP3]], 176
; CHECK-NEXT: [[DOTSLICE_44:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 44, i32 45, i32 46, i32 47>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_44]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_48:%.*]] = add nuw i32 [[TMP3]], 192
+; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP3]], 192
; CHECK-NEXT: [[DOTSLICE_48:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 48, i32 49, i32 50, i32 51>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_48]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_52:%.*]] = add nuw i32 [[TMP3]], 208
+; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP3]], 208
; CHECK-NEXT: [[DOTSLICE_52:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 52, i32 53, i32 54, i32 55>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_52]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_56:%.*]] = add nuw i32 [[TMP3]], 224
+; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP3]], 224
; CHECK-NEXT: [[DOTSLICE_56:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 56, i32 57, i32 58, i32 59>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_56]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_60:%.*]] = add nuw i32 [[TMP3]], 240
+; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP3]], 240
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTOFF_240]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
; CHECK-NEXT: [[TMP4]] = add i32 [[LOOP_INDEX]], 256
; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i32 [[TMP4]], 8192
@@ -807,12 +1096,44 @@ define void @memcpy.inline_known_small(ptr addrspace(7) inreg %src, ptr addrspac
; CHECK-NEXT: [[DST_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[DST]], 1
; CHECK-NEXT: [[SRC_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 0
; CHECK-NEXT: [[SRC_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 1
-; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0)
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[TMP1]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_0:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_0]], i64 0
+; CHECK-NEXT: [[DOTOFF_PTR_4:%.*]] = add i32 [[SRC_OFF]], 4
+; CHECK-NEXT: [[DOTOFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_4]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_1:%.*]] = insertelement <4 x i32> [[DOTSLICE_0]], i32 [[DOTOFF_4]], i64 1
+; CHECK-NEXT: [[DOTOFF_PTR_8:%.*]] = add i32 [[SRC_OFF]], 8
+; CHECK-NEXT: [[DOTOFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_8]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_2:%.*]] = insertelement <4 x i32> [[DOTSLICE_1]], i32 [[DOTOFF_8]], i64 2
+; CHECK-NEXT: [[DOTOFF_PTR_12:%.*]] = add i32 [[SRC_OFF]], 12
+; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[DST_OFF]], 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[DST_OFF]], 8
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[DST_OFF]], 12
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[SRC_OFF]], 16
-; CHECK-NEXT: [[TMP3:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_01:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_02:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_01]], i64 0
+; CHECK-NEXT: [[DOTOFF_PTR_43:%.*]] = add i32 [[TMP2]], 4
+; CHECK-NEXT: [[DOTOFF_44:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_43]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_15:%.*]] = insertelement <4 x i32> [[DOTSLICE_02]], i32 [[DOTOFF_44]], i64 1
+; CHECK-NEXT: [[DOTOFF_PTR_86:%.*]] = add i32 [[TMP2]], 8
+; CHECK-NEXT: [[DOTOFF_87:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_86]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_28:%.*]] = insertelement <4 x i32> [[DOTSLICE_15]], i32 [[DOTOFF_87]], i64 2
+; CHECK-NEXT: [[DOTOFF_PTR_129:%.*]] = add i32 [[TMP2]], 12
+; CHECK-NEXT: [[DOTOFF_1210:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_129]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[DOTSLICE_28]], i32 [[DOTOFF_1210]], i64 3
; CHECK-NEXT: [[TMP4:%.*]] = add nuw i32 [[DST_OFF]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[TMP3]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_01]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_111:%.*]] = add i32 [[TMP4]], 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_111]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_212:%.*]] = add i32 [[TMP4]], 8
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_87]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_212]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_313:%.*]] = add i32 [[TMP4]], 12
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_1210]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_313]], i32 0, i32 0)
; CHECK-NEXT: ret void
;
call void @llvm.memcpy.inline.p7.p7.i32(ptr addrspace(7) %dst, ptr addrspace(7) %src, i32 32, i1 false)
@@ -873,117 +1194,326 @@ define void @memcpy.inline_known_i64(ptr addrspace(7) inreg %src, ptr addrspace(
; CHECK-NEXT: [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], %[[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[LOOP_INDEX_C:%.*]] = trunc i64 [[LOOP_INDEX]] to i32
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[SRC_OFF]], [[LOOP_INDEX_C]]
-; CHECK-NEXT: [[DOTOFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP1]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_0:%.*]] = shufflevector <4 x i32> [[DOTOFF_0]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_0:%.*]] = shufflevector <64 x i32> poison, <64 x i32> [[DOTEXT_0]], <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_16:%.*]] = add nuw i32 [[TMP1]], 16
-; CHECK-NEXT: [[DOTOFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_16]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_4:%.*]] = shufflevector <4 x i32> [[DOTOFF_16]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_4:%.*]] = shufflevector <64 x i32> [[DOTPARTS_0]], <64 x i32> [[DOTEXT_4]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 64, i32 65, i32 66, i32 67, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_32:%.*]] = add nuw i32 [[TMP1]], 32
-; CHECK-NEXT: [[DOTOFF_32:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_32]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_8:%.*]] = shufflevector <4 x i32> [[DOTOFF_32]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_8:%.*]] = shufflevector <64 x i32> [[DOTPARTS_4]], <64 x i32> [[DOTEXT_8]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 64, i32 65, i32 66, i32 67, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_48:%.*]] = add nuw i32 [[TMP1]], 48
-; CHECK-NEXT: [[DOTOFF_48:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_48]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_12:%.*]] = shufflevector <4 x i32> [[DOTOFF_48]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_12:%.*]] = shufflevector <64 x i32> [[DOTPARTS_8]], <64 x i32> [[DOTEXT_12]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 64, i32 65, i32 66, i32 67, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_64:%.*]] = add nuw i32 [[TMP1]], 64
-; CHECK-NEXT: [[DOTOFF_64:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_64]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_16:%.*]] = shufflevector <4 x i32> [[DOTOFF_64]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_16:%.*]] = shufflevector <64 x i32> [[DOTPARTS_12]], <64 x i32> [[DOTEXT_16]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_80:%.*]] = add nuw i32 [[TMP1]], 80
-; CHECK-NEXT: [[DOTOFF_80:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_80]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_20:%.*]] = shufflevector <4 x i32> [[DOTOFF_80]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_20:%.*]] = shufflevector <64 x i32> [[DOTPARTS_16]], <64 x i32> [[DOTEXT_20]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 64, i32 65, i32 66, i32 67, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_96:%.*]] = add nuw i32 [[TMP1]], 96
-; CHECK-NEXT: [[DOTOFF_96:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_96]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_24:%.*]] = shufflevector <4 x i32> [[DOTOFF_96]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_24:%.*]] = shufflevector <64 x i32> [[DOTPARTS_20]], <64 x i32> [[DOTEXT_24]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 64, i32 65, i32 66, i32 67, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_112:%.*]] = add nuw i32 [[TMP1]], 112
-; CHECK-NEXT: [[DOTOFF_112:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_112]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_28:%.*]] = shufflevector <4 x i32> [[DOTOFF_112]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_28:%.*]] = shufflevector <64 x i32> [[DOTPARTS_24]], <64 x i32> [[DOTEXT_28]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 64, i32 65, i32 66, i32 67, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_128:%.*]] = add nuw i32 [[TMP1]], 128
-; CHECK-NEXT: [[DOTOFF_128:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_128]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_32:%.*]] = shufflevector <4 x i32> [[DOTOFF_128]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_32:%.*]] = shufflevector <64 x i32> [[DOTPARTS_28]], <64 x i32> [[DOTEXT_32]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 64, i32 65, i32 66, i32 67, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_144:%.*]] = add nuw i32 [[TMP1]], 144
-; CHECK-NEXT: [[DOTOFF_144:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_144]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_36:%.*]] = shufflevector <4 x i32> [[DOTOFF_144]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_36:%.*]] = shufflevector <64 x i32> [[DOTPARTS_32]], <64 x i32> [[DOTEXT_36]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 64, i32 65, i32 66, i32 67, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_160:%.*]] = add nuw i32 [[TMP1]], 160
-; CHECK-NEXT: [[DOTOFF_160:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_160]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_40:%.*]] = shufflevector <4 x i32> [[DOTOFF_160]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_40:%.*]] = shufflevector <64 x i32> [[DOTPARTS_36]], <64 x i32> [[DOTEXT_40]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 64, i32 65, i32 66, i32 67, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_176:%.*]] = add nuw i32 [[TMP1]], 176
-; CHECK-NEXT: [[DOTOFF_176:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_176]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_44:%.*]] = shufflevector <4 x i32> [[DOTOFF_176]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_44:%.*]] = shufflevector <64 x i32> [[DOTPARTS_40]], <64 x i32> [[DOTEXT_44]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 64, i32 65, i32 66, i32 67, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_192:%.*]] = add nuw i32 [[TMP1]], 192
-; CHECK-NEXT: [[DOTOFF_192:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_192]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_48:%.*]] = shufflevector <4 x i32> [[DOTOFF_192]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_48:%.*]] = shufflevector <64 x i32> [[DOTPARTS_44]], <64 x i32> [[DOTEXT_48]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 64, i32 65, i32 66, i32 67, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_208:%.*]] = add nuw i32 [[TMP1]], 208
-; CHECK-NEXT: [[DOTOFF_208:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_208]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_52:%.*]] = shufflevector <4 x i32> [[DOTOFF_208]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_52:%.*]] = shufflevector <64 x i32> [[DOTPARTS_48]], <64 x i32> [[DOTEXT_52]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 64, i32 65, i32 66, i32 67, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_224:%.*]] = add nuw i32 [[TMP1]], 224
-; CHECK-NEXT: [[DOTOFF_224:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_224]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_56:%.*]] = shufflevector <4 x i32> [[DOTOFF_224]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[DOTPARTS_56:%.*]] = shufflevector <64 x i32> [[DOTPARTS_52]], <64 x i32> [[DOTEXT_56]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 64, i32 65, i32 66, i32 67, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_240:%.*]] = add nuw i32 [[TMP1]], 240
-; CHECK-NEXT: [[DOTOFF_240:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_240]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTEXT_60:%.*]] = shufflevector <4 x i32> [[DOTOFF_240]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <64 x i32> [[DOTPARTS_56]], <64 x i32> [[DOTEXT_60]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 64, i32 65, i32 66, i32 67>
+; CHECK-NEXT: [[DOTOFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP1]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_0:%.*]] = insertelement <64 x i32> poison, i32 [[DOTOFF_0]], i64 0
+; CHECK-NEXT: [[DOTOFF_PTR_4:%.*]] = add i32 [[TMP1]], 4
+; CHECK-NEXT: [[DOTOFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_4]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_1:%.*]] = insertelement <64 x i32> [[DOTSLICE_0]], i32 [[DOTOFF_4]], i64 1
+; CHECK-NEXT: [[DOTOFF_PTR_8:%.*]] = add i32 [[TMP1]], 8
+; CHECK-NEXT: [[DOTOFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_8]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_2:%.*]] = insertelement <64 x i32> [[DOTSLICE_1]], i32 [[DOTOFF_8]], i64 2
+; CHECK-NEXT: [[DOTOFF_PTR_12:%.*]] = add i32 [[TMP1]], 12
+; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_3:%.*]] = insertelement <64 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
+; CHECK-NEXT: [[DOTOFF_PTR_16:%.*]] = add i32 [[TMP1]], 16
+; CHECK-NEXT: [[DOTOFF_16:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_16]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_4:%.*]] = insertelement <64 x i32> [[DOTSLICE_3]], i32 [[DOTOFF_16]], i64 4
+; CHECK-NEXT: [[DOTOFF_PTR_20:%.*]] = add i32 [[TMP1]], 20
+; CHECK-NEXT: [[DOTOFF_20:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_20]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_5:%.*]] = insertelement <64 x i32> [[DOTSLICE_4]], i32 [[DOTOFF_20]], i64 5
+; CHECK-NEXT: [[DOTOFF_PTR_24:%.*]] = add i32 [[TMP1]], 24
+; CHECK-NEXT: [[DOTOFF_24:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_24]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_6:%.*]] = insertelement <64 x i32> [[DOTSLICE_5]], i32 [[DOTOFF_24]], i64 6
+; CHECK-NEXT: [[DOTOFF_PTR_28:%.*]] = add i32 [[TMP1]], 28
+; CHECK-NEXT: [[DOTOFF_28:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_28]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_7:%.*]] = insertelement <64 x i32> [[DOTSLICE_6]], i32 [[DOTOFF_28]], i64 7
+; CHECK-NEXT: [[DOTOFF_PTR_32:%.*]] = add i32 [[TMP1]], 32
+; CHECK-NEXT: [[DOTOFF_32:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_32]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_8:%.*]] = insertelement <64 x i32> [[DOTSLICE_7]], i32 [[DOTOFF_32]], i64 8
+; CHECK-NEXT: [[DOTOFF_PTR_36:%.*]] = add i32 [[TMP1]], 36
+; CHECK-NEXT: [[DOTOFF_36:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_36]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_9:%.*]] = insertelement <64 x i32> [[DOTSLICE_8]], i32 [[DOTOFF_36]], i64 9
+; CHECK-NEXT: [[DOTOFF_PTR_40:%.*]] = add i32 [[TMP1]], 40
+; CHECK-NEXT: [[DOTOFF_40:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_40]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_10:%.*]] = insertelement <64 x i32> [[DOTSLICE_9]], i32 [[DOTOFF_40]], i64 10
+; CHECK-NEXT: [[DOTOFF_PTR_44:%.*]] = add i32 [[TMP1]], 44
+; CHECK-NEXT: [[DOTOFF_44:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_44]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_11:%.*]] = insertelement <64 x i32> [[DOTSLICE_10]], i32 [[DOTOFF_44]], i64 11
+; CHECK-NEXT: [[DOTOFF_PTR_48:%.*]] = add i32 [[TMP1]], 48
+; CHECK-NEXT: [[DOTOFF_48:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_48]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_12:%.*]] = insertelement <64 x i32> [[DOTSLICE_11]], i32 [[DOTOFF_48]], i64 12
+; CHECK-NEXT: [[DOTOFF_PTR_52:%.*]] = add i32 [[TMP1]], 52
+; CHECK-NEXT: [[DOTOFF_52:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_52]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_13:%.*]] = insertelement <64 x i32> [[DOTSLICE_12]], i32 [[DOTOFF_52]], i64 13
+; CHECK-NEXT: [[DOTOFF_PTR_56:%.*]] = add i32 [[TMP1]], 56
+; CHECK-NEXT: [[DOTOFF_56:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_56]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_14:%.*]] = insertelement <64 x i32> [[DOTSLICE_13]], i32 [[DOTOFF_56]], i64 14
+; CHECK-NEXT: [[DOTOFF_PTR_60:%.*]] = add i32 [[TMP1]], 60
+; CHECK-NEXT: [[DOTOFF_60:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_60]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_15:%.*]] = insertelement <64 x i32> [[DOTSLICE_14]], i32 [[DOTOFF_60]], i64 15
+; CHECK-NEXT: [[DOTOFF_PTR_64:%.*]] = add i32 [[TMP1]], 64
+; CHECK-NEXT: [[DOTOFF_64:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_64]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_16:%.*]] = insertelement <64 x i32> [[DOTSLICE_15]], i32 [[DOTOFF_64]], i64 16
+; CHECK-NEXT: [[DOTOFF_PTR_68:%.*]] = add i32 [[TMP1]], 68
+; CHECK-NEXT: [[DOTOFF_68:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_68]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_17:%.*]] = insertelement <64 x i32> [[DOTSLICE_16]], i32 [[DOTOFF_68]], i64 17
+; CHECK-NEXT: [[DOTOFF_PTR_72:%.*]] = add i32 [[TMP1]], 72
+; CHECK-NEXT: [[DOTOFF_72:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_72]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_18:%.*]] = insertelement <64 x i32> [[DOTSLICE_17]], i32 [[DOTOFF_72]], i64 18
+; CHECK-NEXT: [[DOTOFF_PTR_76:%.*]] = add i32 [[TMP1]], 76
+; CHECK-NEXT: [[DOTOFF_76:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_76]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_19:%.*]] = insertelement <64 x i32> [[DOTSLICE_18]], i32 [[DOTOFF_76]], i64 19
+; CHECK-NEXT: [[DOTOFF_PTR_80:%.*]] = add i32 [[TMP1]], 80
+; CHECK-NEXT: [[DOTOFF_80:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_80]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_20:%.*]] = insertelement <64 x i32> [[DOTSLICE_19]], i32 [[DOTOFF_80]], i64 20
+; CHECK-NEXT: [[DOTOFF_PTR_84:%.*]] = add i32 [[TMP1]], 84
+; CHECK-NEXT: [[DOTOFF_84:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_84]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_21:%.*]] = insertelement <64 x i32> [[DOTSLICE_20]], i32 [[DOTOFF_84]], i64 21
+; CHECK-NEXT: [[DOTOFF_PTR_88:%.*]] = add i32 [[TMP1]], 88
+; CHECK-NEXT: [[DOTOFF_88:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_88]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_22:%.*]] = insertelement <64 x i32> [[DOTSLICE_21]], i32 [[DOTOFF_88]], i64 22
+; CHECK-NEXT: [[DOTOFF_PTR_92:%.*]] = add i32 [[TMP1]], 92
+; CHECK-NEXT: [[DOTOFF_92:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_92]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_23:%.*]] = insertelement <64 x i32> [[DOTSLICE_22]], i32 [[DOTOFF_92]], i64 23
+; CHECK-NEXT: [[DOTOFF_PTR_96:%.*]] = add i32 [[TMP1]], 96
+; CHECK-NEXT: [[DOTOFF_96:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_96]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_24:%.*]] = insertelement <64 x i32> [[DOTSLICE_23]], i32 [[DOTOFF_96]], i64 24
+; CHECK-NEXT: [[DOTOFF_PTR_100:%.*]] = add i32 [[TMP1]], 100
+; CHECK-NEXT: [[DOTOFF_100:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_100]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_25:%.*]] = insertelement <64 x i32> [[DOTSLICE_24]], i32 [[DOTOFF_100]], i64 25
+; CHECK-NEXT: [[DOTOFF_PTR_104:%.*]] = add i32 [[TMP1]], 104
+; CHECK-NEXT: [[DOTOFF_104:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_104]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_26:%.*]] = insertelement <64 x i32> [[DOTSLICE_25]], i32 [[DOTOFF_104]], i64 26
+; CHECK-NEXT: [[DOTOFF_PTR_108:%.*]] = add i32 [[TMP1]], 108
+; CHECK-NEXT: [[DOTOFF_108:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_108]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_27:%.*]] = insertelement <64 x i32> [[DOTSLICE_26]], i32 [[DOTOFF_108]], i64 27
+; CHECK-NEXT: [[DOTOFF_PTR_112:%.*]] = add i32 [[TMP1]], 112
+; CHECK-NEXT: [[DOTOFF_112:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_112]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_28:%.*]] = insertelement <64 x i32> [[DOTSLICE_27]], i32 [[DOTOFF_112]], i64 28
+; CHECK-NEXT: [[DOTOFF_PTR_116:%.*]] = add i32 [[TMP1]], 116
+; CHECK-NEXT: [[DOTOFF_116:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_116]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_29:%.*]] = insertelement <64 x i32> [[DOTSLICE_28]], i32 [[DOTOFF_116]], i64 29
+; CHECK-NEXT: [[DOTOFF_PTR_120:%.*]] = add i32 [[TMP1]], 120
+; CHECK-NEXT: [[DOTOFF_120:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_120]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_30:%.*]] = insertelement <64 x i32> [[DOTSLICE_29]], i32 [[DOTOFF_120]], i64 30
+; CHECK-NEXT: [[DOTOFF_PTR_124:%.*]] = add i32 [[TMP1]], 124
+; CHECK-NEXT: [[DOTOFF_124:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_124]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_31:%.*]] = insertelement <64 x i32> [[DOTSLICE_30]], i32 [[DOTOFF_124]], i64 31
+; CHECK-NEXT: [[DOTOFF_PTR_128:%.*]] = add i32 [[TMP1]], 128
+; CHECK-NEXT: [[DOTOFF_128:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_128]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_32:%.*]] = insertelement <64 x i32> [[DOTSLICE_31]], i32 [[DOTOFF_128]], i64 32
+; CHECK-NEXT: [[DOTOFF_PTR_132:%.*]] = add i32 [[TMP1]], 132
+; CHECK-NEXT: [[DOTOFF_132:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_132]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_33:%.*]] = insertelement <64 x i32> [[DOTSLICE_32]], i32 [[DOTOFF_132]], i64 33
+; CHECK-NEXT: [[DOTOFF_PTR_136:%.*]] = add i32 [[TMP1]], 136
+; CHECK-NEXT: [[DOTOFF_136:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_136]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_34:%.*]] = insertelement <64 x i32> [[DOTSLICE_33]], i32 [[DOTOFF_136]], i64 34
+; CHECK-NEXT: [[DOTOFF_PTR_140:%.*]] = add i32 [[TMP1]], 140
+; CHECK-NEXT: [[DOTOFF_140:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_140]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_35:%.*]] = insertelement <64 x i32> [[DOTSLICE_34]], i32 [[DOTOFF_140]], i64 35
+; CHECK-NEXT: [[DOTOFF_PTR_144:%.*]] = add i32 [[TMP1]], 144
+; CHECK-NEXT: [[DOTOFF_144:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_144]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_36:%.*]] = insertelement <64 x i32> [[DOTSLICE_35]], i32 [[DOTOFF_144]], i64 36
+; CHECK-NEXT: [[DOTOFF_PTR_148:%.*]] = add i32 [[TMP1]], 148
+; CHECK-NEXT: [[DOTOFF_148:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_148]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_37:%.*]] = insertelement <64 x i32> [[DOTSLICE_36]], i32 [[DOTOFF_148]], i64 37
+; CHECK-NEXT: [[DOTOFF_PTR_152:%.*]] = add i32 [[TMP1]], 152
+; CHECK-NEXT: [[DOTOFF_152:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_152]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_38:%.*]] = insertelement <64 x i32> [[DOTSLICE_37]], i32 [[DOTOFF_152]], i64 38
+; CHECK-NEXT: [[DOTOFF_PTR_156:%.*]] = add i32 [[TMP1]], 156
+; CHECK-NEXT: [[DOTOFF_156:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_156]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_39:%.*]] = insertelement <64 x i32> [[DOTSLICE_38]], i32 [[DOTOFF_156]], i64 39
+; CHECK-NEXT: [[DOTOFF_PTR_160:%.*]] = add i32 [[TMP1]], 160
+; CHECK-NEXT: [[DOTOFF_160:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_160]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_40:%.*]] = insertelement <64 x i32> [[DOTSLICE_39]], i32 [[DOTOFF_160]], i64 40
+; CHECK-NEXT: [[DOTOFF_PTR_164:%.*]] = add i32 [[TMP1]], 164
+; CHECK-NEXT: [[DOTOFF_164:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_164]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_41:%.*]] = insertelement <64 x i32> [[DOTSLICE_40]], i32 [[DOTOFF_164]], i64 41
+; CHECK-NEXT: [[DOTOFF_PTR_168:%.*]] = add i32 [[TMP1]], 168
+; CHECK-NEXT: [[DOTOFF_168:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_168]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_42:%.*]] = insertelement <64 x i32> [[DOTSLICE_41]], i32 [[DOTOFF_168]], i64 42
+; CHECK-NEXT: [[DOTOFF_PTR_172:%.*]] = add i32 [[TMP1]], 172
+; CHECK-NEXT: [[DOTOFF_172:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_172]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_43:%.*]] = insertelement <64 x i32> [[DOTSLICE_42]], i32 [[DOTOFF_172]], i64 43
+; CHECK-NEXT: [[DOTOFF_PTR_176:%.*]] = add i32 [[TMP1]], 176
+; CHECK-NEXT: [[DOTOFF_176:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_176]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_44:%.*]] = insertelement <64 x i32> [[DOTSLICE_43]], i32 [[DOTOFF_176]], i64 44
+; CHECK-NEXT: [[DOTOFF_PTR_180:%.*]] = add i32 [[TMP1]], 180
+; CHECK-NEXT: [[DOTOFF_180:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_180]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_45:%.*]] = insertelement <64 x i32> [[DOTSLICE_44]], i32 [[DOTOFF_180]], i64 45
+; CHECK-NEXT: [[DOTOFF_PTR_184:%.*]] = add i32 [[TMP1]], 184
+; CHECK-NEXT: [[DOTOFF_184:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_184]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_46:%.*]] = insertelement <64 x i32> [[DOTSLICE_45]], i32 [[DOTOFF_184]], i64 46
+; CHECK-NEXT: [[DOTOFF_PTR_188:%.*]] = add i32 [[TMP1]], 188
+; CHECK-NEXT: [[DOTOFF_188:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_188]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_47:%.*]] = insertelement <64 x i32> [[DOTSLICE_46]], i32 [[DOTOFF_188]], i64 47
+; CHECK-NEXT: [[DOTOFF_PTR_192:%.*]] = add i32 [[TMP1]], 192
+; CHECK-NEXT: [[DOTOFF_192:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_192]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_48:%.*]] = insertelement <64 x i32> [[DOTSLICE_47]], i32 [[DOTOFF_192]], i64 48
+; CHECK-NEXT: [[DOTOFF_PTR_196:%.*]] = add i32 [[TMP1]], 196
+; CHECK-NEXT: [[DOTOFF_196:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_196]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_49:%.*]] = insertelement <64 x i32> [[DOTSLICE_48]], i32 [[DOTOFF_196]], i64 49
+; CHECK-NEXT: [[DOTOFF_PTR_200:%.*]] = add i32 [[TMP1]], 200
+; CHECK-NEXT: [[DOTOFF_200:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_200]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_50:%.*]] = insertelement <64 x i32> [[DOTSLICE_49]], i32 [[DOTOFF_200]], i64 50
+; CHECK-NEXT: [[DOTOFF_PTR_204:%.*]] = add i32 [[TMP1]], 204
+; CHECK-NEXT: [[DOTOFF_204:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_204]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_51:%.*]] = insertelement <64 x i32> [[DOTSLICE_50]], i32 [[DOTOFF_204]], i64 51
+; CHECK-NEXT: [[DOTOFF_PTR_208:%.*]] = add i32 [[TMP1]], 208
+; CHECK-NEXT: [[DOTOFF_208:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_208]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_52:%.*]] = insertelement <64 x i32> [[DOTSLICE_51]], i32 [[DOTOFF_208]], i64 52
+; CHECK-NEXT: [[DOTOFF_PTR_212:%.*]] = add i32 [[TMP1]], 212
+; CHECK-NEXT: [[DOTOFF_212:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_212]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_53:%.*]] = insertelement <64 x i32> [[DOTSLICE_52]], i32 [[DOTOFF_212]], i64 53
+; CHECK-NEXT: [[DOTOFF_PTR_216:%.*]] = add i32 [[TMP1]], 216
+; CHECK-NEXT: [[DOTOFF_216:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_216]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_54:%.*]] = insertelement <64 x i32> [[DOTSLICE_53]], i32 [[DOTOFF_216]], i64 54
+; CHECK-NEXT: [[DOTOFF_PTR_220:%.*]] = add i32 [[TMP1]], 220
+; CHECK-NEXT: [[DOTOFF_220:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_220]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_55:%.*]] = insertelement <64 x i32> [[DOTSLICE_54]], i32 [[DOTOFF_220]], i64 55
+; CHECK-NEXT: [[DOTOFF_PTR_224:%.*]] = add i32 [[TMP1]], 224
+; CHECK-NEXT: [[DOTOFF_224:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_224]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_56:%.*]] = insertelement <64 x i32> [[DOTSLICE_55]], i32 [[DOTOFF_224]], i64 56
+; CHECK-NEXT: [[DOTOFF_PTR_228:%.*]] = add i32 [[TMP1]], 228
+; CHECK-NEXT: [[DOTOFF_228:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_228]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_57:%.*]] = insertelement <64 x i32> [[DOTSLICE_56]], i32 [[DOTOFF_228]], i64 57
+; CHECK-NEXT: [[DOTOFF_PTR_232:%.*]] = add i32 [[TMP1]], 232
+; CHECK-NEXT: [[DOTOFF_232:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_232]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_58:%.*]] = insertelement <64 x i32> [[DOTSLICE_57]], i32 [[DOTOFF_232]], i64 58
+; CHECK-NEXT: [[DOTOFF_PTR_236:%.*]] = add i32 [[TMP1]], 236
+; CHECK-NEXT: [[DOTOFF_236:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_236]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_59:%.*]] = insertelement <64 x i32> [[DOTSLICE_58]], i32 [[DOTOFF_236]], i64 59
+; CHECK-NEXT: [[DOTOFF_PTR_240:%.*]] = add i32 [[TMP1]], 240
+; CHECK-NEXT: [[DOTOFF_240:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_240]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_60:%.*]] = insertelement <64 x i32> [[DOTSLICE_59]], i32 [[DOTOFF_240]], i64 60
+; CHECK-NEXT: [[DOTOFF_PTR_244:%.*]] = add i32 [[TMP1]], 244
+; CHECK-NEXT: [[DOTOFF_244:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_244]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_61:%.*]] = insertelement <64 x i32> [[DOTSLICE_60]], i32 [[DOTOFF_244]], i64 61
+; CHECK-NEXT: [[DOTOFF_PTR_248:%.*]] = add i32 [[TMP1]], 248
+; CHECK-NEXT: [[DOTOFF_248:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_248]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_62:%.*]] = insertelement <64 x i32> [[DOTSLICE_61]], i32 [[DOTOFF_248]], i64 62
+; CHECK-NEXT: [[DOTOFF_PTR_252:%.*]] = add i32 [[TMP1]], 252
+; CHECK-NEXT: [[DOTOFF_252:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_252]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <64 x i32> [[DOTSLICE_62]], i32 [[DOTOFF_252]], i64 63
; CHECK-NEXT: [[LOOP_INDEX_C1:%.*]] = trunc i64 [[LOOP_INDEX]] to i32
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX_C1]]
-; CHECK-NEXT: [[DOTSLICE_0:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_4:%.*]] = add nuw i32 [[TMP3]], 16
-; CHECK-NEXT: [[DOTSLICE_4:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_8:%.*]] = add nuw i32 [[TMP3]], 32
-; CHECK-NEXT: [[DOTSLICE_8:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_12:%.*]] = add nuw i32 [[TMP3]], 48
-; CHECK-NEXT: [[DOTSLICE_12:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_16:%.*]] = add nuw i32 [[TMP3]], 64
-; CHECK-NEXT: [[DOTSLICE_16:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_16]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_20:%.*]] = add nuw i32 [[TMP3]], 80
-; CHECK-NEXT: [[DOTSLICE_20:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_20]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_24:%.*]] = add nuw i32 [[TMP3]], 96
-; CHECK-NEXT: [[DOTSLICE_24:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_24]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_28:%.*]] = add nuw i32 [[TMP3]], 112
-; CHECK-NEXT: [[DOTSLICE_28:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_28]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_32:%.*]] = add nuw i32 [[TMP3]], 128
-; CHECK-NEXT: [[DOTSLICE_32:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_32]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_36:%.*]] = add nuw i32 [[TMP3]], 144
-; CHECK-NEXT: [[DOTSLICE_36:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 36, i32 37, i32 38, i32 39>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_36]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_40:%.*]] = add nuw i32 [[TMP3]], 160
-; CHECK-NEXT: [[DOTSLICE_40:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 40, i32 41, i32 42, i32 43>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_40]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_44:%.*]] = add nuw i32 [[TMP3]], 176
-; CHECK-NEXT: [[DOTSLICE_44:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 44, i32 45, i32 46, i32 47>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_48:%.*]] = add nuw i32 [[TMP3]], 192
-; CHECK-NEXT: [[DOTSLICE_48:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 48, i32 49, i32 50, i32 51>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_48]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_52:%.*]] = add nuw i32 [[TMP3]], 208
-; CHECK-NEXT: [[DOTSLICE_52:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 52, i32 53, i32 54, i32 55>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_52]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_56:%.*]] = add nuw i32 [[TMP3]], 224
-; CHECK-NEXT: [[DOTSLICE_56:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 56, i32 57, i32 58, i32 59>
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_56]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_60:%.*]] = add nuw i32 [[TMP3]], 240
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTOFF_240]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP3]], 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP3]], 8
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP3]], 12
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP3]], 16
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_16]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_5:%.*]] = add i32 [[TMP3]], 20
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_20]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_5]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_6:%.*]] = add i32 [[TMP3]], 24
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_24]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_6]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_7:%.*]] = add i32 [[TMP3]], 28
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_28]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_7]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP3]], 32
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_32]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_9:%.*]] = add i32 [[TMP3]], 36
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_36]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_9]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_10:%.*]] = add i32 [[TMP3]], 40
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_40]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_10]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP3]], 44
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP3]], 48
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_48]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_13:%.*]] = add i32 [[TMP3]], 52
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_52]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_13]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_14:%.*]] = add i32 [[TMP3]], 56
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_56]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_14]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_15:%.*]] = add i32 [[TMP3]], 60
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_60]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_15]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP3]], 64
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_64]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_17:%.*]] = add i32 [[TMP3]], 68
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_68]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_17]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_18:%.*]] = add i32 [[TMP3]], 72
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_72]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_18]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_19:%.*]] = add i32 [[TMP3]], 76
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_76]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_19]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP3]], 80
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_80]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_21:%.*]] = add i32 [[TMP3]], 84
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_84]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_21]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP3]], 88
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_88]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_23:%.*]] = add i32 [[TMP3]], 92
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_92]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_23]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP3]], 96
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_96]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_25:%.*]] = add i32 [[TMP3]], 100
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_100]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_25]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_26:%.*]] = add i32 [[TMP3]], 104
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_104]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_26]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_27:%.*]] = add i32 [[TMP3]], 108
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_108]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_27]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP3]], 112
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_112]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_29:%.*]] = add i32 [[TMP3]], 116
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_116]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_29]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_30:%.*]] = add i32 [[TMP3]], 120
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_120]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_30]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_31:%.*]] = add i32 [[TMP3]], 124
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_124]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_31]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP3]], 128
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_128]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP3]], 132
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_132]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_34:%.*]] = add i32 [[TMP3]], 136
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_136]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_34]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_35:%.*]] = add i32 [[TMP3]], 140
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_140]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_35]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP3]], 144
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_144]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_37:%.*]] = add i32 [[TMP3]], 148
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_148]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_37]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_38:%.*]] = add i32 [[TMP3]], 152
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_152]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_38]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_39:%.*]] = add i32 [[TMP3]], 156
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_156]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_39]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP3]], 160
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_160]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_41:%.*]] = add i32 [[TMP3]], 164
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_164]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_41]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_42:%.*]] = add i32 [[TMP3]], 168
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_168]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_42]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_43:%.*]] = add i32 [[TMP3]], 172
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_172]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_43]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP3]], 176
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_176]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_45:%.*]] = add i32 [[TMP3]], 180
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_180]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_45]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_46:%.*]] = add i32 [[TMP3]], 184
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_184]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_46]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_47:%.*]] = add i32 [[TMP3]], 188
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_188]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_47]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP3]], 192
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_192]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_49:%.*]] = add i32 [[TMP3]], 196
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_196]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_49]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_50:%.*]] = add i32 [[TMP3]], 200
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_200]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_50]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_51:%.*]] = add i32 [[TMP3]], 204
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_204]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_51]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP3]], 208
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_208]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_53:%.*]] = add i32 [[TMP3]], 212
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_212]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_53]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_54:%.*]] = add i32 [[TMP3]], 216
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_216]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_54]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_55:%.*]] = add i32 [[TMP3]], 220
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_220]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_55]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP3]], 224
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_224]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_57:%.*]] = add i32 [[TMP3]], 228
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_228]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_57]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_58:%.*]] = add i32 [[TMP3]], 232
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_232]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_58]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_59:%.*]] = add i32 [[TMP3]], 236
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_236]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_59]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP3]], 240
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_240]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_61:%.*]] = add i32 [[TMP3]], 244
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_244]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_61]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_62:%.*]] = add i32 [[TMP3]], 248
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_248]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_62]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_63:%.*]] = add i32 [[TMP3]], 252
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_252]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_63]], i32 0, i32 0)
; CHECK-NEXT: [[TMP4]] = add i64 [[LOOP_INDEX]], 256
; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 8192
; CHECK-NEXT: br i1 [[TMP5]], label %[[STATIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]], label %[[STATIC_MEMCPY_POST_LOOP_EXPANSION:.*]]
@@ -1001,12 +1531,44 @@ define void @memcpy.inline_known_i32_volatile(ptr addrspace(7) inreg %src, ptr a
; CHECK-NEXT: [[DST_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[DST]], 1
; CHECK-NEXT: [[SRC_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 0
; CHECK-NEXT: [[SRC_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[SRC]], 1
-; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 -2147483648)
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[TMP1]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTOFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[SRC_OFF]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTSLICE_0:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_0]], i64 0
+; CHECK-NEXT: [[DOTOFF_PTR_4:%.*]] = add i32 [[SRC_OFF]], 4
+; CHECK-NEXT: [[DOTOFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_4]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTSLICE_1:%.*]] = insertelement <4 x i32> [[DOTSLICE_0]], i32 [[DOTOFF_4]], i64 1
+; CHECK-NEXT: [[DOTOFF_PTR_8:%.*]] = add i32 [[SRC_OFF]], 8
+; CHECK-NEXT: [[DOTOFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_8]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTSLICE_2:%.*]] = insertelement <4 x i32> [[DOTSLICE_1]], i32 [[DOTOFF_8]], i64 2
+; CHECK-NEXT: [[DOTOFF_PTR_12:%.*]] = add i32 [[SRC_OFF]], 12
+; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DST_OFF]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[DST_OFF]], 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[DST_OFF]], 8
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[DST_OFF]], 12
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 -2147483648)
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[SRC_OFF]], 16
-; CHECK-NEXT: [[TMP3:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTOFF_01:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP2]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTSLICE_02:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_01]], i64 0
+; CHECK-NEXT: [[DOTOFF_PTR_43:%.*]] = add i32 [[TMP2]], 4
+; CHECK-NEXT: [[DOTOFF_44:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_43]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTSLICE_15:%.*]] = insertelement <4 x i32> [[DOTSLICE_02]], i32 [[DOTOFF_44]], i64 1
+; CHECK-NEXT: [[DOTOFF_PTR_86:%.*]] = add i32 [[TMP2]], 8
+; CHECK-NEXT: [[DOTOFF_87:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_86]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTSLICE_28:%.*]] = insertelement <4 x i32> [[DOTSLICE_15]], i32 [[DOTOFF_87]], i64 2
+; CHECK-NEXT: [[DOTOFF_PTR_129:%.*]] = add i32 [[TMP2]], 12
+; CHECK-NEXT: [[DOTOFF_1210:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_129]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[DOTSLICE_28]], i32 [[DOTOFF_1210]], i64 3
; CHECK-NEXT: [[TMP4:%.*]] = add nuw i32 [[DST_OFF]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[TMP3]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_01]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP4]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTPART_111:%.*]] = add i32 [[TMP4]], 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_44]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_111]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTPART_212:%.*]] = add i32 [[TMP4]], 8
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_87]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_212]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTPART_313:%.*]] = add i32 [[TMP4]], 12
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_1210]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_313]], i32 0, i32 -2147483648)
; CHECK-NEXT: ret void
;
call void @llvm.memcpy.inline.p7.p7.i32(ptr addrspace(7) %dst, ptr addrspace(7) %src, i32 32, i1 true)
@@ -1027,9 +1589,25 @@ define void @memcpy.inline_unknown(ptr addrspace(7) inreg %src, ptr addrspace(7)
; CHECK: [[DYNAMIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]]:
; CHECK-NEXT: [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP7:%.*]], %[[DYNAMIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[SRC_OFF]], [[LOOP_INDEX]]
-; CHECK-NEXT: [[TMP5:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP4]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTOFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[TMP4]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_0:%.*]] = insertelement <4 x i32> poison, i32 [[DOTOFF_0]], i64 0
+; CHECK-NEXT: [[DOTOFF_PTR_4:%.*]] = add i32 [[TMP4]], 4
+; CHECK-NEXT: [[DOTOFF_4:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_4]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_1:%.*]] = insertelement <4 x i32> [[DOTSLICE_0]], i32 [[DOTOFF_4]], i64 1
+; CHECK-NEXT: [[DOTOFF_PTR_8:%.*]] = add i32 [[TMP4]], 8
+; CHECK-NEXT: [[DOTOFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_8]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTSLICE_2:%.*]] = insertelement <4 x i32> [[DOTSLICE_1]], i32 [[DOTOFF_8]], i64 2
+; CHECK-NEXT: [[DOTOFF_PTR_12:%.*]] = add i32 [[TMP4]], 12
+; CHECK-NEXT: [[DOTOFF_12:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 1 [[SRC_RSRC]], i32 [[DOTOFF_PTR_12]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[DOTSLICE_2]], i32 [[DOTOFF_12]], i64 3
; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[TMP5]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP6]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_0]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[TMP6]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP6]], 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_4]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP6]], 8
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_8]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP6]], 12
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DOTOFF_12]], ptr addrspace(8) align 1 [[DST_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
; CHECK-NEXT: [[TMP7]] = add i32 [[LOOP_INDEX]], 16
; CHECK-NEXT: [[TMP8:%.*]] = icmp ult i32 [[TMP7]], [[TMP2]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[DYNAMIC_MEMCPY_LOOP_EXPANSION_MAIN_BODY]], label %[[DYNAMIC_MEMCPY_LOOP_EXPANSION_RESIDUAL_COND]]
@@ -1066,49 +1644,49 @@ define void @memcpy.inline_known_p1_to_p7(ptr addrspace(1) inreg %src, ptr addrs
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[DST_OFF]], [[LOOP_INDEX]]
; CHECK-NEXT: [[DOTSLICE_0:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_0]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[TMP3]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_4:%.*]] = add nuw i32 [[TMP3]], 16
+; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP3]], 16
; CHECK-NEXT: [[DOTSLICE_4:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_4]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_8:%.*]] = add nuw i32 [[TMP3]], 32
+; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP3]], 32
; CHECK-NEXT: [[DOTSLICE_8:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_8]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_12:%.*]] = add nuw i32 [[TMP3]], 48
+; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP3]], 48
; CHECK-NEXT: [[DOTSLICE_12:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_12]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_16:%.*]] = add nuw i32 [[TMP3]], 64
+; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP3]], 64
; CHECK-NEXT: [[DOTSLICE_16:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_16]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_20:%.*]] = add nuw i32 [[TMP3]], 80
+; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP3]], 80
; CHECK-NEXT: [[DOTSLICE_20:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_20]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_24:%.*]] = add nuw i32 [[TMP3]], 96
+; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP3]], 96
; CHECK-NEXT: [[DOTSLICE_24:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_24]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_28:%.*]] = add nuw i32 [[TMP3]], 112
+; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP3]], 112
; CHECK-NEXT: [[DOTSLICE_28:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_28]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_32:%.*]] = add nuw i32 [[TMP3]], 128
+; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP3]], 128
; CHECK-NEXT: [[DOTSLICE_32:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_32]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_36:%.*]] = add nuw i32 [[TMP3]], 144
+; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP3]], 144
; CHECK-NEXT: [[DOTSLICE_36:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 36, i32 37, i32 38, i32 39>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_36]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_40:%.*]] = add nuw i32 [[TMP3]], 160
+; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP3]], 160
; CHECK-NEXT: [[DOTSLICE_40:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 40, i32 41, i32 42, i32 43>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_40]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_44:%.*]] = add nuw i32 [[TMP3]], 176
+; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP3]], 176
; CHECK-NEXT: [[DOTSLICE_44:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 44, i32 45, i32 46, i32 47>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_44]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_48:%.*]] = add nuw i32 [[TMP3]], 192
+; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP3]], 192
; CHECK-NEXT: [[DOTSLICE_48:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 48, i32 49, i32 50, i32 51>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_48]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_52:%.*]] = add nuw i32 [[TMP3]], 208
+; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP3]], 208
; CHECK-NEXT: [[DOTSLICE_52:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 52, i32 53, i32 54, i32 55>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_52]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_56:%.*]] = add nuw i32 [[TMP3]], 224
+; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP3]], 224
; CHECK-NEXT: [[DOTSLICE_56:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 56, i32 57, i32 58, i32 59>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_56]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_60:%.*]] = add nuw i32 [[TMP3]], 240
+; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP3]], 240
; CHECK-NEXT: [[DOTSLICE_60:%.*]] = shufflevector <64 x i32> [[TMP2]], <64 x i32> poison, <4 x i32> <i32 60, i32 61, i32 62, i32 63>
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DOTSLICE_60]], ptr addrspace(8) align 16 [[DST_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
; CHECK-NEXT: [[TMP4]] = add i32 [[LOOP_INDEX]], 256
@@ -1133,63 +1711,63 @@ define void @memcpy.inline_known_p7_to_p1(ptr addrspace(7) inreg %src, ptr addrs
; CHECK-NEXT: [[DOTOFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[TMP1]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_0:%.*]] = shufflevector <4 x i32> [[DOTOFF_0]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_0:%.*]] = shufflevector <64 x i32> poison, <64 x i32> [[DOTEXT_0]], <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_16:%.*]] = add nuw i32 [[TMP1]], 16
+; CHECK-NEXT: [[DOTOFF_PTR_16:%.*]] = add i32 [[TMP1]], 16
; CHECK-NEXT: [[DOTOFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_16]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_4:%.*]] = shufflevector <4 x i32> [[DOTOFF_16]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_4:%.*]] = shufflevector <64 x i32> [[DOTPARTS_0]], <64 x i32> [[DOTEXT_4]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 64, i32 65, i32 66, i32 67, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_32:%.*]] = add nuw i32 [[TMP1]], 32
+; CHECK-NEXT: [[DOTOFF_PTR_32:%.*]] = add i32 [[TMP1]], 32
; CHECK-NEXT: [[DOTOFF_32:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_32]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_8:%.*]] = shufflevector <4 x i32> [[DOTOFF_32]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_8:%.*]] = shufflevector <64 x i32> [[DOTPARTS_4]], <64 x i32> [[DOTEXT_8]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 64, i32 65, i32 66, i32 67, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_48:%.*]] = add nuw i32 [[TMP1]], 48
+; CHECK-NEXT: [[DOTOFF_PTR_48:%.*]] = add i32 [[TMP1]], 48
; CHECK-NEXT: [[DOTOFF_48:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_48]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_12:%.*]] = shufflevector <4 x i32> [[DOTOFF_48]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_12:%.*]] = shufflevector <64 x i32> [[DOTPARTS_8]], <64 x i32> [[DOTEXT_12]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 64, i32 65, i32 66, i32 67, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_64:%.*]] = add nuw i32 [[TMP1]], 64
+; CHECK-NEXT: [[DOTOFF_PTR_64:%.*]] = add i32 [[TMP1]], 64
; CHECK-NEXT: [[DOTOFF_64:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_64]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_16:%.*]] = shufflevector <4 x i32> [[DOTOFF_64]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_16:%.*]] = shufflevector <64 x i32> [[DOTPARTS_12]], <64 x i32> [[DOTEXT_16]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_80:%.*]] = add nuw i32 [[TMP1]], 80
+; CHECK-NEXT: [[DOTOFF_PTR_80:%.*]] = add i32 [[TMP1]], 80
; CHECK-NEXT: [[DOTOFF_80:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_80]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_20:%.*]] = shufflevector <4 x i32> [[DOTOFF_80]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_20:%.*]] = shufflevector <64 x i32> [[DOTPARTS_16]], <64 x i32> [[DOTEXT_20]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 64, i32 65, i32 66, i32 67, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_96:%.*]] = add nuw i32 [[TMP1]], 96
+; CHECK-NEXT: [[DOTOFF_PTR_96:%.*]] = add i32 [[TMP1]], 96
; CHECK-NEXT: [[DOTOFF_96:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_96]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_24:%.*]] = shufflevector <4 x i32> [[DOTOFF_96]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_24:%.*]] = shufflevector <64 x i32> [[DOTPARTS_20]], <64 x i32> [[DOTEXT_24]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 64, i32 65, i32 66, i32 67, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_112:%.*]] = add nuw i32 [[TMP1]], 112
+; CHECK-NEXT: [[DOTOFF_PTR_112:%.*]] = add i32 [[TMP1]], 112
; CHECK-NEXT: [[DOTOFF_112:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_112]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_28:%.*]] = shufflevector <4 x i32> [[DOTOFF_112]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_28:%.*]] = shufflevector <64 x i32> [[DOTPARTS_24]], <64 x i32> [[DOTEXT_28]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 64, i32 65, i32 66, i32 67, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_128:%.*]] = add nuw i32 [[TMP1]], 128
+; CHECK-NEXT: [[DOTOFF_PTR_128:%.*]] = add i32 [[TMP1]], 128
; CHECK-NEXT: [[DOTOFF_128:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_128]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_32:%.*]] = shufflevector <4 x i32> [[DOTOFF_128]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_32:%.*]] = shufflevector <64 x i32> [[DOTPARTS_28]], <64 x i32> [[DOTEXT_32]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 64, i32 65, i32 66, i32 67, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_144:%.*]] = add nuw i32 [[TMP1]], 144
+; CHECK-NEXT: [[DOTOFF_PTR_144:%.*]] = add i32 [[TMP1]], 144
; CHECK-NEXT: [[DOTOFF_144:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_144]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_36:%.*]] = shufflevector <4 x i32> [[DOTOFF_144]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_36:%.*]] = shufflevector <64 x i32> [[DOTPARTS_32]], <64 x i32> [[DOTEXT_36]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 64, i32 65, i32 66, i32 67, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_160:%.*]] = add nuw i32 [[TMP1]], 160
+; CHECK-NEXT: [[DOTOFF_PTR_160:%.*]] = add i32 [[TMP1]], 160
; CHECK-NEXT: [[DOTOFF_160:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_160]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_40:%.*]] = shufflevector <4 x i32> [[DOTOFF_160]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_40:%.*]] = shufflevector <64 x i32> [[DOTPARTS_36]], <64 x i32> [[DOTEXT_40]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 64, i32 65, i32 66, i32 67, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_176:%.*]] = add nuw i32 [[TMP1]], 176
+; CHECK-NEXT: [[DOTOFF_PTR_176:%.*]] = add i32 [[TMP1]], 176
; CHECK-NEXT: [[DOTOFF_176:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_176]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_44:%.*]] = shufflevector <4 x i32> [[DOTOFF_176]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_44:%.*]] = shufflevector <64 x i32> [[DOTPARTS_40]], <64 x i32> [[DOTEXT_44]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 64, i32 65, i32 66, i32 67, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_192:%.*]] = add nuw i32 [[TMP1]], 192
+; CHECK-NEXT: [[DOTOFF_PTR_192:%.*]] = add i32 [[TMP1]], 192
; CHECK-NEXT: [[DOTOFF_192:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_192]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_48:%.*]] = shufflevector <4 x i32> [[DOTOFF_192]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_48:%.*]] = shufflevector <64 x i32> [[DOTPARTS_44]], <64 x i32> [[DOTEXT_48]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 64, i32 65, i32 66, i32 67, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_208:%.*]] = add nuw i32 [[TMP1]], 208
+; CHECK-NEXT: [[DOTOFF_PTR_208:%.*]] = add i32 [[TMP1]], 208
; CHECK-NEXT: [[DOTOFF_208:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_208]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_52:%.*]] = shufflevector <4 x i32> [[DOTOFF_208]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_52:%.*]] = shufflevector <64 x i32> [[DOTPARTS_48]], <64 x i32> [[DOTEXT_52]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 64, i32 65, i32 66, i32 67, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_224:%.*]] = add nuw i32 [[TMP1]], 224
+; CHECK-NEXT: [[DOTOFF_PTR_224:%.*]] = add i32 [[TMP1]], 224
; CHECK-NEXT: [[DOTOFF_224:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_224]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_56:%.*]] = shufflevector <4 x i32> [[DOTOFF_224]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_56:%.*]] = shufflevector <64 x i32> [[DOTPARTS_52]], <64 x i32> [[DOTEXT_56]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 64, i32 65, i32 66, i32 67, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_240:%.*]] = add nuw i32 [[TMP1]], 240
+; CHECK-NEXT: [[DOTOFF_PTR_240:%.*]] = add i32 [[TMP1]], 240
; CHECK-NEXT: [[DOTOFF_240:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_240]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_60:%.*]] = shufflevector <4 x i32> [[DOTOFF_240]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <64 x i32> [[DOTPARTS_56]], <64 x i32> [[DOTEXT_60]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 64, i32 65, i32 66, i32 67>
@@ -1246,63 +1824,63 @@ define void @memcpy.inline_known_p7_to_p3_long(ptr addrspace(7) inreg %src, ptr
; CHECK-NEXT: [[DOTOFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[TMP1]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_0:%.*]] = shufflevector <4 x i32> [[DOTOFF_0]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_0:%.*]] = shufflevector <64 x i32> poison, <64 x i32> [[DOTEXT_0]], <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_16:%.*]] = add nuw i32 [[TMP1]], 16
+; CHECK-NEXT: [[DOTOFF_PTR_16:%.*]] = add i32 [[TMP1]], 16
; CHECK-NEXT: [[DOTOFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_16]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_4:%.*]] = shufflevector <4 x i32> [[DOTOFF_16]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_4:%.*]] = shufflevector <64 x i32> [[DOTPARTS_0]], <64 x i32> [[DOTEXT_4]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 64, i32 65, i32 66, i32 67, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_32:%.*]] = add nuw i32 [[TMP1]], 32
+; CHECK-NEXT: [[DOTOFF_PTR_32:%.*]] = add i32 [[TMP1]], 32
; CHECK-NEXT: [[DOTOFF_32:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_32]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_8:%.*]] = shufflevector <4 x i32> [[DOTOFF_32]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_8:%.*]] = shufflevector <64 x i32> [[DOTPARTS_4]], <64 x i32> [[DOTEXT_8]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 64, i32 65, i32 66, i32 67, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_48:%.*]] = add nuw i32 [[TMP1]], 48
+; CHECK-NEXT: [[DOTOFF_PTR_48:%.*]] = add i32 [[TMP1]], 48
; CHECK-NEXT: [[DOTOFF_48:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_48]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_12:%.*]] = shufflevector <4 x i32> [[DOTOFF_48]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_12:%.*]] = shufflevector <64 x i32> [[DOTPARTS_8]], <64 x i32> [[DOTEXT_12]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 64, i32 65, i32 66, i32 67, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_64:%.*]] = add nuw i32 [[TMP1]], 64
+; CHECK-NEXT: [[DOTOFF_PTR_64:%.*]] = add i32 [[TMP1]], 64
; CHECK-NEXT: [[DOTOFF_64:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_64]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_16:%.*]] = shufflevector <4 x i32> [[DOTOFF_64]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_16:%.*]] = shufflevector <64 x i32> [[DOTPARTS_12]], <64 x i32> [[DOTEXT_16]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_80:%.*]] = add nuw i32 [[TMP1]], 80
+; CHECK-NEXT: [[DOTOFF_PTR_80:%.*]] = add i32 [[TMP1]], 80
; CHECK-NEXT: [[DOTOFF_80:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_80]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_20:%.*]] = shufflevector <4 x i32> [[DOTOFF_80]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_20:%.*]] = shufflevector <64 x i32> [[DOTPARTS_16]], <64 x i32> [[DOTEXT_20]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 64, i32 65, i32 66, i32 67, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_96:%.*]] = add nuw i32 [[TMP1]], 96
+; CHECK-NEXT: [[DOTOFF_PTR_96:%.*]] = add i32 [[TMP1]], 96
; CHECK-NEXT: [[DOTOFF_96:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_96]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_24:%.*]] = shufflevector <4 x i32> [[DOTOFF_96]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_24:%.*]] = shufflevector <64 x i32> [[DOTPARTS_20]], <64 x i32> [[DOTEXT_24]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 64, i32 65, i32 66, i32 67, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_112:%.*]] = add nuw i32 [[TMP1]], 112
+; CHECK-NEXT: [[DOTOFF_PTR_112:%.*]] = add i32 [[TMP1]], 112
; CHECK-NEXT: [[DOTOFF_112:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_112]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_28:%.*]] = shufflevector <4 x i32> [[DOTOFF_112]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_28:%.*]] = shufflevector <64 x i32> [[DOTPARTS_24]], <64 x i32> [[DOTEXT_28]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 64, i32 65, i32 66, i32 67, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_128:%.*]] = add nuw i32 [[TMP1]], 128
+; CHECK-NEXT: [[DOTOFF_PTR_128:%.*]] = add i32 [[TMP1]], 128
; CHECK-NEXT: [[DOTOFF_128:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_128]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_32:%.*]] = shufflevector <4 x i32> [[DOTOFF_128]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_32:%.*]] = shufflevector <64 x i32> [[DOTPARTS_28]], <64 x i32> [[DOTEXT_32]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 64, i32 65, i32 66, i32 67, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_144:%.*]] = add nuw i32 [[TMP1]], 144
+; CHECK-NEXT: [[DOTOFF_PTR_144:%.*]] = add i32 [[TMP1]], 144
; CHECK-NEXT: [[DOTOFF_144:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_144]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_36:%.*]] = shufflevector <4 x i32> [[DOTOFF_144]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_36:%.*]] = shufflevector <64 x i32> [[DOTPARTS_32]], <64 x i32> [[DOTEXT_36]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 64, i32 65, i32 66, i32 67, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_160:%.*]] = add nuw i32 [[TMP1]], 160
+; CHECK-NEXT: [[DOTOFF_PTR_160:%.*]] = add i32 [[TMP1]], 160
; CHECK-NEXT: [[DOTOFF_160:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_160]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_40:%.*]] = shufflevector <4 x i32> [[DOTOFF_160]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_40:%.*]] = shufflevector <64 x i32> [[DOTPARTS_36]], <64 x i32> [[DOTEXT_40]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 64, i32 65, i32 66, i32 67, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_176:%.*]] = add nuw i32 [[TMP1]], 176
+; CHECK-NEXT: [[DOTOFF_PTR_176:%.*]] = add i32 [[TMP1]], 176
; CHECK-NEXT: [[DOTOFF_176:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_176]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_44:%.*]] = shufflevector <4 x i32> [[DOTOFF_176]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_44:%.*]] = shufflevector <64 x i32> [[DOTPARTS_40]], <64 x i32> [[DOTEXT_44]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 64, i32 65, i32 66, i32 67, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_192:%.*]] = add nuw i32 [[TMP1]], 192
+; CHECK-NEXT: [[DOTOFF_PTR_192:%.*]] = add i32 [[TMP1]], 192
; CHECK-NEXT: [[DOTOFF_192:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_192]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_48:%.*]] = shufflevector <4 x i32> [[DOTOFF_192]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_48:%.*]] = shufflevector <64 x i32> [[DOTPARTS_44]], <64 x i32> [[DOTEXT_48]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 64, i32 65, i32 66, i32 67, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_208:%.*]] = add nuw i32 [[TMP1]], 208
+; CHECK-NEXT: [[DOTOFF_PTR_208:%.*]] = add i32 [[TMP1]], 208
; CHECK-NEXT: [[DOTOFF_208:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_208]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_52:%.*]] = shufflevector <4 x i32> [[DOTOFF_208]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_52:%.*]] = shufflevector <64 x i32> [[DOTPARTS_48]], <64 x i32> [[DOTEXT_52]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 64, i32 65, i32 66, i32 67, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_224:%.*]] = add nuw i32 [[TMP1]], 224
+; CHECK-NEXT: [[DOTOFF_PTR_224:%.*]] = add i32 [[TMP1]], 224
; CHECK-NEXT: [[DOTOFF_224:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_224]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_56:%.*]] = shufflevector <4 x i32> [[DOTOFF_224]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[DOTPARTS_56:%.*]] = shufflevector <64 x i32> [[DOTPARTS_52]], <64 x i32> [[DOTEXT_56]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 64, i32 65, i32 66, i32 67, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[DOTOFF_PTR_240:%.*]] = add nuw i32 [[TMP1]], 240
+; CHECK-NEXT: [[DOTOFF_PTR_240:%.*]] = add i32 [[TMP1]], 240
; CHECK-NEXT: [[DOTOFF_240:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[SRC_RSRC]], i32 [[DOTOFF_PTR_240]], i32 0, i32 0)
; CHECK-NEXT: [[DOTEXT_60:%.*]] = shufflevector <4 x i32> [[DOTOFF_240]], <4 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <64 x i32> [[DOTPARTS_56]], <64 x i32> [[DOTEXT_60]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 64, i32 65, i32 66, i32 67>
@@ -1333,35 +1911,35 @@ define void @memset_known(ptr addrspace(7) inreg %ptr) {
; CHECK-NEXT: [[TMP1:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP3:%.*]], %[[STATIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[PTR_OFF]], [[TMP1]]
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 0), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 1), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 2), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 3)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_4:%.*]] = add nuw i32 [[TMP2]], 16
+; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP2]], 16
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 4), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 5), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 6), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 7)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_8:%.*]] = add nuw i32 [[TMP2]], 32
+; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP2]], 32
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 8), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 9), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 10), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 11)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_12:%.*]] = add nuw i32 [[TMP2]], 48
+; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP2]], 48
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 12), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 13), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 14), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 15)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_16:%.*]] = add nuw i32 [[TMP2]], 64
+; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP2]], 64
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 16), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 17), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 18), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 19)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_20:%.*]] = add nuw i32 [[TMP2]], 80
+; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP2]], 80
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 20), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 21), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 22), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 23)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_24:%.*]] = add nuw i32 [[TMP2]], 96
+; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP2]], 96
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 24), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 25), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 26), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 27)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_28:%.*]] = add nuw i32 [[TMP2]], 112
+; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP2]], 112
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 28), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 29), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 30), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 31)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_32:%.*]] = add nuw i32 [[TMP2]], 128
+; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP2]], 128
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 32), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 33), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 34), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 35)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_36:%.*]] = add nuw i32 [[TMP2]], 144
+; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP2]], 144
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 36), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 37), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 38), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 39)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_40:%.*]] = add nuw i32 [[TMP2]], 160
+; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP2]], 160
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 40), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 41), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 42), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 43)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_44:%.*]] = add nuw i32 [[TMP2]], 176
+; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP2]], 176
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 44), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 45), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 46), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 47)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_48:%.*]] = add nuw i32 [[TMP2]], 192
+; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP2]], 192
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 48), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 49), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 50), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 51)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_52:%.*]] = add nuw i32 [[TMP2]], 208
+; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP2]], 208
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 52), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 53), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 54), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 55)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_56:%.*]] = add nuw i32 [[TMP2]], 224
+; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP2]], 224
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 56), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 57), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 58), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 59)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_60:%.*]] = add nuw i32 [[TMP2]], 240
+; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP2]], 240
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 60), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 61), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 62), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 63)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
; CHECK-NEXT: [[TMP3]] = add i32 [[TMP1]], 256
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i32 [[TMP3]], 8192
@@ -1378,9 +1956,21 @@ define void @memset_known_small(ptr addrspace(7) inreg %ptr) {
; CHECK-SAME: { ptr addrspace(8), i32 } inreg [[PTR:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0
; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[PTR_OFF]], 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[PTR_OFF]], 8
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[PTR_OFF]], 12
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
; CHECK-NEXT: [[TMP1:%.*]] = add nuw i32 [[PTR_OFF]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP1]], 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP1]], 8
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP1]], 12
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0)
; CHECK-NEXT: ret void
;
call void @llvm.memset.p7.i32(ptr addrspace(7) %ptr, i8 1, i32 32, i1 false)
@@ -1427,37 +2017,133 @@ define void @memset_known_i64(ptr addrspace(7) inreg %ptr) {
; CHECK-NEXT: [[TMP1:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP3:%.*]], %[[STATIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[DOTC:%.*]] = trunc i64 [[TMP1]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[PTR_OFF]], [[DOTC]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 0), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 1), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 2), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 3)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_4:%.*]] = add nuw i32 [[TMP2]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 4), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 5), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 6), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 7)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_8:%.*]] = add nuw i32 [[TMP2]], 32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 8), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 9), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 10), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 11)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_12:%.*]] = add nuw i32 [[TMP2]], 48
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 12), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 13), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 14), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 15)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_16:%.*]] = add nuw i32 [[TMP2]], 64
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 16), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 17), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 18), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 19)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_20:%.*]] = add nuw i32 [[TMP2]], 80
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 20), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 21), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 22), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 23)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_24:%.*]] = add nuw i32 [[TMP2]], 96
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 24), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 25), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 26), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 27)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_28:%.*]] = add nuw i32 [[TMP2]], 112
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 28), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 29), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 30), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 31)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_32:%.*]] = add nuw i32 [[TMP2]], 128
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 32), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 33), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 34), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 35)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_36:%.*]] = add nuw i32 [[TMP2]], 144
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 36), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 37), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 38), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 39)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_40:%.*]] = add nuw i32 [[TMP2]], 160
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 40), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 41), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 42), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 43)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_44:%.*]] = add nuw i32 [[TMP2]], 176
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 44), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 45), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 46), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 47)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_48:%.*]] = add nuw i32 [[TMP2]], 192
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 48), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 49), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 50), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 51)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_52:%.*]] = add nuw i32 [[TMP2]], 208
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 52), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 53), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 54), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 55)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_56:%.*]] = add nuw i32 [[TMP2]], 224
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 56), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 57), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 58), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 59)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_60:%.*]] = add nuw i32 [[TMP2]], 240
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 60), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 61), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 62), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 63)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP2]], 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP2]], 8
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP2]], 12
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP2]], 16
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 4), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_5:%.*]] = add i32 [[TMP2]], 20
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 5), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_5]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_6:%.*]] = add i32 [[TMP2]], 24
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 6), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_6]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_7:%.*]] = add i32 [[TMP2]], 28
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 7), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_7]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP2]], 32
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 8), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_9:%.*]] = add i32 [[TMP2]], 36
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 9), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_9]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_10:%.*]] = add i32 [[TMP2]], 40
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 10), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_10]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP2]], 44
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 11), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP2]], 48
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 12), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_13:%.*]] = add i32 [[TMP2]], 52
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 13), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_13]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_14:%.*]] = add i32 [[TMP2]], 56
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 14), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_14]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_15:%.*]] = add i32 [[TMP2]], 60
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 15), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_15]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP2]], 64
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 16), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_17:%.*]] = add i32 [[TMP2]], 68
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 17), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_17]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_18:%.*]] = add i32 [[TMP2]], 72
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 18), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_18]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_19:%.*]] = add i32 [[TMP2]], 76
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 19), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_19]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP2]], 80
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 20), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_21:%.*]] = add i32 [[TMP2]], 84
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 21), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_21]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP2]], 88
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 22), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_23:%.*]] = add i32 [[TMP2]], 92
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 23), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_23]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP2]], 96
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 24), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_25:%.*]] = add i32 [[TMP2]], 100
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 25), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_25]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_26:%.*]] = add i32 [[TMP2]], 104
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 26), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_26]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_27:%.*]] = add i32 [[TMP2]], 108
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 27), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_27]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP2]], 112
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 28), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_29:%.*]] = add i32 [[TMP2]], 116
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 29), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_29]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_30:%.*]] = add i32 [[TMP2]], 120
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 30), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_30]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_31:%.*]] = add i32 [[TMP2]], 124
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 31), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_31]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP2]], 128
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 32), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP2]], 132
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 33), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_34:%.*]] = add i32 [[TMP2]], 136
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 34), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_34]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_35:%.*]] = add i32 [[TMP2]], 140
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 35), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_35]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP2]], 144
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 36), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_37:%.*]] = add i32 [[TMP2]], 148
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 37), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_37]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_38:%.*]] = add i32 [[TMP2]], 152
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 38), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_38]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_39:%.*]] = add i32 [[TMP2]], 156
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 39), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_39]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP2]], 160
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 40), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_41:%.*]] = add i32 [[TMP2]], 164
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 41), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_41]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_42:%.*]] = add i32 [[TMP2]], 168
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 42), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_42]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_43:%.*]] = add i32 [[TMP2]], 172
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 43), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_43]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP2]], 176
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 44), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_45:%.*]] = add i32 [[TMP2]], 180
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 45), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_45]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_46:%.*]] = add i32 [[TMP2]], 184
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 46), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_46]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_47:%.*]] = add i32 [[TMP2]], 188
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 47), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_47]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP2]], 192
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 48), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_49:%.*]] = add i32 [[TMP2]], 196
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 49), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_49]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_50:%.*]] = add i32 [[TMP2]], 200
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 50), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_50]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_51:%.*]] = add i32 [[TMP2]], 204
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 51), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_51]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP2]], 208
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 52), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_53:%.*]] = add i32 [[TMP2]], 212
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 53), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_53]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_54:%.*]] = add i32 [[TMP2]], 216
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 54), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_54]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_55:%.*]] = add i32 [[TMP2]], 220
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 55), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_55]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP2]], 224
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 56), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_57:%.*]] = add i32 [[TMP2]], 228
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 57), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_57]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_58:%.*]] = add i32 [[TMP2]], 232
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 58), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_58]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_59:%.*]] = add i32 [[TMP2]], 236
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 59), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_59]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP2]], 240
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 60), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_61:%.*]] = add i32 [[TMP2]], 244
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 61), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_61]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_62:%.*]] = add i32 [[TMP2]], 248
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 62), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_62]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_63:%.*]] = add i32 [[TMP2]], 252
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 63), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_63]], i32 0, i32 0)
; CHECK-NEXT: [[TMP3]] = add i64 [[TMP1]], 256
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i64 [[TMP3]], 8192
; CHECK-NEXT: br i1 [[TMP4]], label %[[STATIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]], label %[[STATIC_MEMSET_POST_LOOP_EXPANSION:.*]]
@@ -1473,9 +2159,21 @@ define void @memset_known_i32_volatile(ptr addrspace(7) inreg %ptr) {
; CHECK-SAME: { ptr addrspace(8), i32 } inreg [[PTR:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0
; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[PTR_OFF]], 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[PTR_OFF]], 8
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[PTR_OFF]], 12
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 -2147483648)
; CHECK-NEXT: [[TMP1:%.*]] = add nuw i32 [[PTR_OFF]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP1]], 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP1]], 8
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP1]], 12
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 -2147483648)
; CHECK-NEXT: ret void
;
call void @llvm.memset.p7.i32(ptr addrspace(7) %ptr, i8 1, i32 32, i1 true)
@@ -1494,7 +2192,13 @@ define void @memset_unknown(ptr addrspace(7) inreg %ptr, i32 inreg %length) {
; CHECK: [[DYNAMIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]]:
; CHECK-NEXT: [[TMP2:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[DYNAMIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[PTR_OFF]], [[TMP2]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP3]], 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP3]], 8
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP3]], 12
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
; CHECK-NEXT: [[TMP5]] = add i32 [[TMP2]], 16
; CHECK-NEXT: [[TMP6:%.*]] = icmp ult i32 [[TMP5]], [[TMP4]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[DYNAMIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]], label %[[DYNAMIC_MEMSET_LOOP_EXPANSION_RESIDUAL_COND]]
@@ -1531,35 +2235,35 @@ define void @memset.inline_known(ptr addrspace(7) inreg %ptr) {
; CHECK-NEXT: [[TMP1:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP3:%.*]], %[[STATIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[PTR_OFF]], [[TMP1]]
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 0), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 1), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 2), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 3)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_4:%.*]] = add nuw i32 [[TMP2]], 16
+; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP2]], 16
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 4), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 5), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 6), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 7)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_8:%.*]] = add nuw i32 [[TMP2]], 32
+; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP2]], 32
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 8), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 9), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 10), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 11)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_12:%.*]] = add nuw i32 [[TMP2]], 48
+; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP2]], 48
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 12), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 13), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 14), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 15)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_16:%.*]] = add nuw i32 [[TMP2]], 64
+; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP2]], 64
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 16), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 17), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 18), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 19)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_20:%.*]] = add nuw i32 [[TMP2]], 80
+; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP2]], 80
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 20), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 21), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 22), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 23)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_24:%.*]] = add nuw i32 [[TMP2]], 96
+; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP2]], 96
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 24), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 25), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 26), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 27)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_28:%.*]] = add nuw i32 [[TMP2]], 112
+; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP2]], 112
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 28), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 29), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 30), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 31)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_32:%.*]] = add nuw i32 [[TMP2]], 128
+; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP2]], 128
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 32), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 33), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 34), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 35)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_36:%.*]] = add nuw i32 [[TMP2]], 144
+; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP2]], 144
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 36), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 37), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 38), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 39)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_40:%.*]] = add nuw i32 [[TMP2]], 160
+; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP2]], 160
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 40), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 41), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 42), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 43)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_44:%.*]] = add nuw i32 [[TMP2]], 176
+; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP2]], 176
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 44), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 45), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 46), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 47)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_48:%.*]] = add nuw i32 [[TMP2]], 192
+; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP2]], 192
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 48), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 49), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 50), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 51)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_52:%.*]] = add nuw i32 [[TMP2]], 208
+; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP2]], 208
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 52), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 53), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 54), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 55)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_56:%.*]] = add nuw i32 [[TMP2]], 224
+; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP2]], 224
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 56), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 57), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 58), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 59)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_60:%.*]] = add nuw i32 [[TMP2]], 240
+; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP2]], 240
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 60), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 61), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 62), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 63)>, ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
; CHECK-NEXT: [[TMP3]] = add i32 [[TMP1]], 256
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i32 [[TMP3]], 8192
@@ -1576,9 +2280,21 @@ define void @memset.inline_known_small(ptr addrspace(7) inreg %ptr) {
; CHECK-SAME: { ptr addrspace(8), i32 } inreg [[PTR:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0
; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[PTR_OFF]], 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[PTR_OFF]], 8
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[PTR_OFF]], 12
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
; CHECK-NEXT: [[TMP1:%.*]] = add nuw i32 [[PTR_OFF]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP1]], 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP1]], 8
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP1]], 12
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0)
; CHECK-NEXT: ret void
;
call void @llvm.memset.inline.p7.i32(ptr addrspace(7) %ptr, i8 1, i32 32, i1 false)
@@ -1625,37 +2341,133 @@ define void @memset.inline_known_i64(ptr addrspace(7) inreg %ptr) {
; CHECK-NEXT: [[TMP1:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP3:%.*]], %[[STATIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[DOTC:%.*]] = trunc i64 [[TMP1]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[PTR_OFF]], [[DOTC]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 0), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 1), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 2), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 3)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_4:%.*]] = add nuw i32 [[TMP2]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 4), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 5), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 6), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 7)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_8:%.*]] = add nuw i32 [[TMP2]], 32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 8), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 9), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 10), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 11)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_12:%.*]] = add nuw i32 [[TMP2]], 48
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 12), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 13), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 14), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 15)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_16:%.*]] = add nuw i32 [[TMP2]], 64
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 16), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 17), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 18), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 19)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_20:%.*]] = add nuw i32 [[TMP2]], 80
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 20), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 21), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 22), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 23)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_24:%.*]] = add nuw i32 [[TMP2]], 96
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 24), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 25), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 26), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 27)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_28:%.*]] = add nuw i32 [[TMP2]], 112
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 28), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 29), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 30), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 31)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_32:%.*]] = add nuw i32 [[TMP2]], 128
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 32), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 33), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 34), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 35)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_36:%.*]] = add nuw i32 [[TMP2]], 144
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 36), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 37), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 38), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 39)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_40:%.*]] = add nuw i32 [[TMP2]], 160
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 40), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 41), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 42), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 43)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_44:%.*]] = add nuw i32 [[TMP2]], 176
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 44), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 45), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 46), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 47)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_48:%.*]] = add nuw i32 [[TMP2]], 192
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 48), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 49), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 50), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 51)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_52:%.*]] = add nuw i32 [[TMP2]], 208
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 52), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 53), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 54), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 55)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_56:%.*]] = add nuw i32 [[TMP2]], 224
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 56), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 57), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 58), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 59)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_60:%.*]] = add nuw i32 [[TMP2]], 240
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> <i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 60), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 61), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 62), i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i32 63)>, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP2]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP2]], 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP2]], 8
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP2]], 12
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP2]], 16
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 4), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_5:%.*]] = add i32 [[TMP2]], 20
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 5), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_5]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_6:%.*]] = add i32 [[TMP2]], 24
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 6), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_6]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_7:%.*]] = add i32 [[TMP2]], 28
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 7), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_7]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP2]], 32
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 8), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_9:%.*]] = add i32 [[TMP2]], 36
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 9), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_9]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_10:%.*]] = add i32 [[TMP2]], 40
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 10), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_10]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP2]], 44
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 11), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP2]], 48
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 12), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_13:%.*]] = add i32 [[TMP2]], 52
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 13), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_13]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_14:%.*]] = add i32 [[TMP2]], 56
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 14), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_14]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_15:%.*]] = add i32 [[TMP2]], 60
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 15), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_15]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP2]], 64
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 16), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_17:%.*]] = add i32 [[TMP2]], 68
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 17), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_17]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_18:%.*]] = add i32 [[TMP2]], 72
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 18), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_18]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_19:%.*]] = add i32 [[TMP2]], 76
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 19), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_19]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP2]], 80
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 20), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_21:%.*]] = add i32 [[TMP2]], 84
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 21), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_21]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP2]], 88
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 22), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_23:%.*]] = add i32 [[TMP2]], 92
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 23), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_23]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP2]], 96
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 24), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_25:%.*]] = add i32 [[TMP2]], 100
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 25), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_25]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_26:%.*]] = add i32 [[TMP2]], 104
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 26), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_26]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_27:%.*]] = add i32 [[TMP2]], 108
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 27), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_27]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP2]], 112
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 28), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_29:%.*]] = add i32 [[TMP2]], 116
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 29), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_29]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_30:%.*]] = add i32 [[TMP2]], 120
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 30), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_30]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_31:%.*]] = add i32 [[TMP2]], 124
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 31), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_31]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP2]], 128
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 32), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP2]], 132
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 33), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_34:%.*]] = add i32 [[TMP2]], 136
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 34), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_34]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_35:%.*]] = add i32 [[TMP2]], 140
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 35), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_35]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP2]], 144
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 36), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_37:%.*]] = add i32 [[TMP2]], 148
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 37), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_37]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_38:%.*]] = add i32 [[TMP2]], 152
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 38), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_38]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_39:%.*]] = add i32 [[TMP2]], 156
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 39), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_39]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP2]], 160
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 40), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_41:%.*]] = add i32 [[TMP2]], 164
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 41), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_41]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_42:%.*]] = add i32 [[TMP2]], 168
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 42), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_42]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_43:%.*]] = add i32 [[TMP2]], 172
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 43), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_43]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP2]], 176
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 44), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_45:%.*]] = add i32 [[TMP2]], 180
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 45), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_45]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_46:%.*]] = add i32 [[TMP2]], 184
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 46), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_46]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_47:%.*]] = add i32 [[TMP2]], 188
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 47), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_47]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP2]], 192
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 48), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_49:%.*]] = add i32 [[TMP2]], 196
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 49), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_49]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_50:%.*]] = add i32 [[TMP2]], 200
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 50), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_50]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_51:%.*]] = add i32 [[TMP2]], 204
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 51), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_51]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP2]], 208
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 52), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_53:%.*]] = add i32 [[TMP2]], 212
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 53), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_53]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_54:%.*]] = add i32 [[TMP2]], 216
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 54), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_54]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_55:%.*]] = add i32 [[TMP2]], 220
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 55), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_55]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP2]], 224
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 56), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_57:%.*]] = add i32 [[TMP2]], 228
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 57), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_57]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_58:%.*]] = add i32 [[TMP2]], 232
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 58), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_58]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_59:%.*]] = add i32 [[TMP2]], 236
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 59), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_59]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP2]], 240
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 60), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_61:%.*]] = add i32 [[TMP2]], 244
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 61), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_61]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_62:%.*]] = add i32 [[TMP2]], 248
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 62), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_62]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_63:%.*]] = add i32 [[TMP2]], 252
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<64 x i32> bitcast (<256 x i8> splat (i8 1) to <64 x i32>), i64 63), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_63]], i32 0, i32 0)
; CHECK-NEXT: [[TMP3]] = add i64 [[TMP1]], 256
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i64 [[TMP3]], 8192
; CHECK-NEXT: br i1 [[TMP4]], label %[[STATIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]], label %[[STATIC_MEMSET_POST_LOOP_EXPANSION:.*]]
@@ -1671,9 +2483,21 @@ define void @memset.inline_known_i32_volatile(ptr addrspace(7) inreg %ptr) {
; CHECK-SAME: { ptr addrspace(8), i32 } inreg [[PTR:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0
; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[PTR_OFF]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[PTR_OFF]], 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[PTR_OFF]], 8
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[PTR_OFF]], 12
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 -2147483648)
; CHECK-NEXT: [[TMP1:%.*]] = add nuw i32 [[PTR_OFF]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 -2147483648)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP1]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP1]], 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP1]], 8
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 -2147483648)
+; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP1]], 12
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 -2147483648)
; CHECK-NEXT: ret void
;
call void @llvm.memset.inline.p7.i32(ptr addrspace(7) %ptr, i8 1, i32 32, i1 true)
@@ -1692,7 +2516,13 @@ define void @memset.inline_unknown(ptr addrspace(7) inreg %ptr, i32 inreg %lengt
; CHECK: [[DYNAMIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]]:
; CHECK-NEXT: [[TMP2:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[DYNAMIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]] ]
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[PTR_OFF]], [[TMP2]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 0), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP3]], 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP3]], 8
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 2), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP3]], 12
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 extractelement (<4 x i32> bitcast (<16 x i8> splat (i8 1) to <4 x i32>), i64 3), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
; CHECK-NEXT: [[TMP5]] = add i32 [[TMP2]], 16
; CHECK-NEXT: [[TMP6:%.*]] = icmp ult i32 [[TMP5]], [[TMP4]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[DYNAMIC_MEMSET_LOOP_EXPANSION_MAIN_BODY]], label %[[DYNAMIC_MEMSET_LOOP_EXPANSION_RESIDUAL_COND]]
@@ -1730,35 +2560,35 @@ define void @memset_pattern_known(ptr addrspace(7) inreg %ptr) {
; CHECK-NEXT: [[DOTIDX1:%.*]] = mul nsw i32 [[LOOP_INDEX]], 256
; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[PTR_OFF]], [[DOTIDX1]]
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[TMP5]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_4:%.*]] = add nuw i32 [[TMP5]], 16
+; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP5]], 16
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_8:%.*]] = add nuw i32 [[TMP5]], 32
+; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP5]], 32
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_12:%.*]] = add nuw i32 [[TMP5]], 48
+; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP5]], 48
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_16:%.*]] = add nuw i32 [[TMP5]], 64
+; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP5]], 64
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_20:%.*]] = add nuw i32 [[TMP5]], 80
+; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP5]], 80
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_24:%.*]] = add nuw i32 [[TMP5]], 96
+; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP5]], 96
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_28:%.*]] = add nuw i32 [[TMP5]], 112
+; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP5]], 112
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_32:%.*]] = add nuw i32 [[TMP5]], 128
+; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP5]], 128
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_36:%.*]] = add nuw i32 [[TMP5]], 144
+; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP5]], 144
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_40:%.*]] = add nuw i32 [[TMP5]], 160
+; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP5]], 160
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_44:%.*]] = add nuw i32 [[TMP5]], 176
+; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP5]], 176
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_48:%.*]] = add nuw i32 [[TMP5]], 192
+; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP5]], 192
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_52:%.*]] = add nuw i32 [[TMP5]], 208
+; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP5]], 208
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_56:%.*]] = add nuw i32 [[TMP5]], 224
+; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP5]], 224
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_60:%.*]] = add nuw i32 [[TMP5]], 240
+; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP5]], 240
; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 16 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
; CHECK-NEXT: [[TMP4]] = add i32 [[LOOP_INDEX]], 64
; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i32 [[TMP4]], 8192
@@ -1802,37 +2632,133 @@ define void @memset_pattern_known_i64(ptr addrspace(7) inreg %ptr) {
; CHECK-NEXT: [[DOTC:%.*]] = trunc i64 [[TMP1]] to i32
; CHECK-NEXT: [[DOTIDX1:%.*]] = mul nsw i32 [[DOTC]], 256
; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[PTR_OFF]], [[DOTIDX1]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP8]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_4:%.*]] = add nuw i32 [[TMP8]], 16
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_8:%.*]] = add nuw i32 [[TMP8]], 32
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_12:%.*]] = add nuw i32 [[TMP8]], 48
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_16:%.*]] = add nuw i32 [[TMP8]], 64
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_20:%.*]] = add nuw i32 [[TMP8]], 80
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_24:%.*]] = add nuw i32 [[TMP8]], 96
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_28:%.*]] = add nuw i32 [[TMP8]], 112
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_32:%.*]] = add nuw i32 [[TMP8]], 128
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_36:%.*]] = add nuw i32 [[TMP8]], 144
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_40:%.*]] = add nuw i32 [[TMP8]], 160
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_44:%.*]] = add nuw i32 [[TMP8]], 176
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_48:%.*]] = add nuw i32 [[TMP8]], 192
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_52:%.*]] = add nuw i32 [[TMP8]], 208
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_56:%.*]] = add nuw i32 [[TMP8]], 224
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
-; CHECK-NEXT: [[DOTPART_60:%.*]] = add nuw i32 [[TMP8]], 240
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP8]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP8]], 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP8]], 8
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP8]], 12
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_4:%.*]] = add i32 [[TMP8]], 16
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_4]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_5:%.*]] = add i32 [[TMP8]], 20
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_5]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_6:%.*]] = add i32 [[TMP8]], 24
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_6]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_7:%.*]] = add i32 [[TMP8]], 28
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_7]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_8:%.*]] = add i32 [[TMP8]], 32
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_8]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_9:%.*]] = add i32 [[TMP8]], 36
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_9]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_10:%.*]] = add i32 [[TMP8]], 40
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_10]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_11:%.*]] = add i32 [[TMP8]], 44
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_11]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_12:%.*]] = add i32 [[TMP8]], 48
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_12]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_13:%.*]] = add i32 [[TMP8]], 52
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_13]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_14:%.*]] = add i32 [[TMP8]], 56
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_14]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_15:%.*]] = add i32 [[TMP8]], 60
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_15]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_16:%.*]] = add i32 [[TMP8]], 64
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_16]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_17:%.*]] = add i32 [[TMP8]], 68
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_17]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_18:%.*]] = add i32 [[TMP8]], 72
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_18]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_19:%.*]] = add i32 [[TMP8]], 76
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_19]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_20:%.*]] = add i32 [[TMP8]], 80
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_20]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_21:%.*]] = add i32 [[TMP8]], 84
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_21]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_22:%.*]] = add i32 [[TMP8]], 88
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_22]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_23:%.*]] = add i32 [[TMP8]], 92
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_23]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_24:%.*]] = add i32 [[TMP8]], 96
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_24]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_25:%.*]] = add i32 [[TMP8]], 100
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_25]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_26:%.*]] = add i32 [[TMP8]], 104
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_26]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_27:%.*]] = add i32 [[TMP8]], 108
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_27]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_28:%.*]] = add i32 [[TMP8]], 112
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_28]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_29:%.*]] = add i32 [[TMP8]], 116
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_29]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_30:%.*]] = add i32 [[TMP8]], 120
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_30]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_31:%.*]] = add i32 [[TMP8]], 124
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_31]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_32:%.*]] = add i32 [[TMP8]], 128
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_32]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_33:%.*]] = add i32 [[TMP8]], 132
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_33]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_34:%.*]] = add i32 [[TMP8]], 136
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_34]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_35:%.*]] = add i32 [[TMP8]], 140
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_35]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_36:%.*]] = add i32 [[TMP8]], 144
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_36]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_37:%.*]] = add i32 [[TMP8]], 148
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_37]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_38:%.*]] = add i32 [[TMP8]], 152
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_38]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_39:%.*]] = add i32 [[TMP8]], 156
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_39]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_40:%.*]] = add i32 [[TMP8]], 160
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_40]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_41:%.*]] = add i32 [[TMP8]], 164
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_41]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_42:%.*]] = add i32 [[TMP8]], 168
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_42]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_43:%.*]] = add i32 [[TMP8]], 172
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_43]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_44:%.*]] = add i32 [[TMP8]], 176
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_44]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_45:%.*]] = add i32 [[TMP8]], 180
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_45]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_46:%.*]] = add i32 [[TMP8]], 184
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_46]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_47:%.*]] = add i32 [[TMP8]], 188
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_47]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_48:%.*]] = add i32 [[TMP8]], 192
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_48]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_49:%.*]] = add i32 [[TMP8]], 196
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_49]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_50:%.*]] = add i32 [[TMP8]], 200
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_50]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_51:%.*]] = add i32 [[TMP8]], 204
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_51]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_52:%.*]] = add i32 [[TMP8]], 208
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_52]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_53:%.*]] = add i32 [[TMP8]], 212
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_53]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_54:%.*]] = add i32 [[TMP8]], 216
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_54]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_55:%.*]] = add i32 [[TMP8]], 220
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_55]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_56:%.*]] = add i32 [[TMP8]], 224
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_56]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_57:%.*]] = add i32 [[TMP8]], 228
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_57]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_58:%.*]] = add i32 [[TMP8]], 232
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_58]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_59:%.*]] = add i32 [[TMP8]], 236
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_59]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_60:%.*]] = add i32 [[TMP8]], 240
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_60]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_61:%.*]] = add i32 [[TMP8]], 244
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_61]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_62:%.*]] = add i32 [[TMP8]], 248
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_62]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_63:%.*]] = add i32 [[TMP8]], 252
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_63]], i32 0, i32 0)
; CHECK-NEXT: [[TMP5]] = add i64 [[TMP1]], 64
; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP5]], 8192
; CHECK-NEXT: br i1 [[TMP3]], label %[[LOADSTORELOOP]], label %[[SPLIT:.*]]
@@ -1877,7 +2803,13 @@ define void @memset_pattern_unknown(ptr addrspace(7) inreg %ptr, i32 inreg %leng
; CHECK-NEXT: [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[LOADSTORELOOP]] ]
; CHECK-NEXT: [[DOTIDX1:%.*]] = mul nsw i32 [[LOOP_INDEX]], 16
; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[PTR_OFF]], [[DOTIDX1]]
-; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> splat (i32 1), ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP4]], i32 0, i32 0)
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[TMP4]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_1:%.*]] = add i32 [[TMP4]], 4
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_1]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_2:%.*]] = add i32 [[TMP4]], 8
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_2]], i32 0, i32 0)
+; CHECK-NEXT: [[DOTPART_3:%.*]] = add i32 [[TMP4]], 12
+; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 1, ptr addrspace(8) align 1 [[PTR_RSRC]], i32 [[DOTPART_3]], i32 0, i32 0)
; CHECK-NEXT: [[TMP5]] = add i32 [[LOOP_INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp ult i32 [[TMP5]], [[TMP8]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[LOADSTORELOOP]], label %[[MEMSET_PATTERN_EXPANSION_RESIDUAL_COND]]
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll
index 42a4829c18784..1ab2a145f6b73 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
-; RUN: opt -S -mcpu=gfx900 -amdgpu-lower-buffer-fat-pointers -check-debugify < %s | FileCheck %s
-; RUN: opt -S -mcpu=gfx900 -passes=amdgpu-lower-buffer-fat-pointers,check-debugify < %s | FileCheck %s
+; RUN: opt -S -mcpu=gfx900 -mattr=+unaligned-access-mode -amdgpu-lower-buffer-fat-pointers -check-debugify < %s | FileCheck %s
+; RUN: opt -S -mcpu=gfx900 -mattr=+unaligned-access-mode -passes=amdgpu-lower-buffer-fat-pointers,check-debugify < %s | FileCheck %s
target triple = "amdgcn--"
@@ -95,7 +95,7 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo
!llvm.dbg.cu = !{!0}
!llvm.debugify = !{!2, !3}
-!llvm.module.flags = !{!4}
+!llvm.module.flags = !{!4, !41}
!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
!1 = !DIFile(filename: "<stdin>", directory: "/")
@@ -132,3 +132,5 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo
!32 = !DILocation(line: 13, column: 1, scope: !5)
!33 = !DILocation(line: 14, column: 1, scope: !5)
!40 = distinct !DIAssignID()
+
+!41 = !{i32 7, !"amdgpu.buffer.oob.mode", i32 1}
diff --git a/llvm/test/CodeGen/AMDGPU/memset-pattern.ll b/llvm/test/CodeGen/AMDGPU/memset-pattern.ll
index d388ae2020da8..f2e7192df56ea 100644
--- a/llvm/test/CodeGen/AMDGPU/memset-pattern.ll
+++ b/llvm/test/CodeGen/AMDGPU/memset-pattern.ll
@@ -856,7 +856,7 @@ define void @memset_pattern_i64_as7_dynlen(ptr addrspace(7) inreg align 16 %a, i
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
%p = getelementptr inbounds i8, ptr addrspace(7) %a, i32 %offset
- call void @llvm.experimental.memset.pattern(ptr addrspace(7) %p, i64 u0xaaaabbbbccccdddd, i64 %len, i1 false)
+ call void @llvm.experimental.memset.pattern(ptr addrspace(7) align(16) %p, i64 u0xaaaabbbbccccdddd, i64 %len, i1 false)
ret void
}
@@ -972,7 +972,7 @@ define void @memset_pattern_i64_as7_dynlen_dynval(ptr addrspace(7) inreg align 1
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
%p = getelementptr inbounds i8, ptr addrspace(7) %a, i32 %offset
- call void @llvm.experimental.memset.pattern(ptr addrspace(7) %p, i64 %val, i64 %len, i1 false)
+ call void @llvm.experimental.memset.pattern(ptr addrspace(7) align(16) %p, i64 %val, i64 %len, i1 false)
ret void
}
@@ -1084,6 +1084,113 @@ define void @memset_pattern_i64_as7_len33_dynval(ptr addrspace(7) inreg align 16
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
%p = getelementptr inbounds i8, ptr addrspace(7) %a, i32 %offset
- call void @llvm.experimental.memset.pattern(ptr addrspace(7) %p, i64 %val, i64 33, i1 false)
+ call void @llvm.experimental.memset.pattern(ptr addrspace(7) align(16) %p, i64 %val, i64 33, i1 false)
+ ret void
+}
+
+define void @memset_pattern_i64_as7_dynlen_unaligned_uniform_len(ptr addrspace(7) inreg align 4 %a, i32 %offset, i64 inreg %len) {
+; GFX942-SDAG-LABEL: memset_pattern_i64_as7_dynlen_unaligned_uniform_len:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_mov_b32 s9, s18
+; GFX942-SDAG-NEXT: s_and_b32 s4, s17, 1
+; GFX942-SDAG-NEXT: s_and_b32 s8, s17, -2
+; GFX942-SDAG-NEXT: s_mov_b32 s5, 0
+; GFX942-SDAG-NEXT: s_cmp_eq_u64 s[8:9], 0
+; GFX942-SDAG-NEXT: s_mov_b64 s[6:7], 0
+; GFX942-SDAG-NEXT: s_cbranch_scc1 .LBB15_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 0xccccdddd
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, 0xaaaabbbb
+; GFX942-SDAG-NEXT: v_add_u32_e32 v1, s16, v0
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[8:9]
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-SDAG-NEXT: .LBB15_2: ; %memset.pattern-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s6, s6, 2
+; GFX942-SDAG-NEXT: s_addc_u32 s7, s7, 0
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[6:7]
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen
+; GFX942-SDAG-NEXT: v_add_u32_e32 v1, 32, v1
+; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB15_2
+; GFX942-SDAG-NEXT: .LBB15_3: ; %memset.pattern-expansion-residual-cond
+; GFX942-SDAG-NEXT: s_cmp_eq_u64 s[4:5], 0
+; GFX942-SDAG-NEXT: s_cbranch_scc1 .LBB15_6
+; GFX942-SDAG-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
+; GFX942-SDAG-NEXT: s_lshl_b32 s6, s17, 3
+; GFX942-SDAG-NEXT: s_and_b32 s6, s6, -16
+; GFX942-SDAG-NEXT: s_add_i32 s16, s16, s6
+; GFX942-SDAG-NEXT: v_add_u32_e32 v4, s16, v0
+; GFX942-SDAG-NEXT: s_mov_b64 s[6:7], 0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0xccccdddd
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, 0xaaaabbbb
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[4:5]
+; GFX942-SDAG-NEXT: .LBB15_5: ; %memset.pattern-expansion-residual-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s6, s6, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s7, s7, 0
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
+; GFX942-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v4, s[0:3], 0 offen
+; GFX942-SDAG-NEXT: v_add_u32_e32 v4, 8, v4
+; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB15_5
+; GFX942-SDAG-NEXT: .LBB15_6: ; %memset.pattern-post-expansion
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_pattern_i64_as7_dynlen_unaligned_uniform_len:
+; GFX942-GISEL: ; %bb.0:
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_mov_b32 s6, s17
+; GFX942-GISEL-NEXT: s_mov_b32 s7, s18
+; GFX942-GISEL-NEXT: s_and_b64 s[4:5], s[6:7], 1
+; GFX942-GISEL-NEXT: s_sub_u32 s8, s17, s4
+; GFX942-GISEL-NEXT: s_subb_u32 s9, s18, s5
+; GFX942-GISEL-NEXT: s_cmp_eq_u64 s[8:9], 0
+; GFX942-GISEL-NEXT: s_mov_b64 s[10:11], 0
+; GFX942-GISEL-NEXT: s_cbranch_scc1 .LBB15_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 0xccccdddd
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 0xaaaabbbb
+; GFX942-GISEL-NEXT: v_add_u32_e32 v1, s16, v0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-GISEL-NEXT: .LBB15_2: ; %memset.pattern-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: s_add_u32 s10, s10, 2
+; GFX942-GISEL-NEXT: s_addc_u32 s11, s11, 0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[10:11]
+; GFX942-GISEL-NEXT: v_cmp_gt_u64_e32 vcc, s[8:9], v[6:7]
+; GFX942-GISEL-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen
+; GFX942-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX942-GISEL-NEXT: v_add_u32_e32 v1, 32, v1
+; GFX942-GISEL-NEXT: s_cbranch_scc1 .LBB15_2
+; GFX942-GISEL-NEXT: .LBB15_3: ; %memset.pattern-expansion-residual-cond
+; GFX942-GISEL-NEXT: s_xor_b32 s8, s4, 1
+; GFX942-GISEL-NEXT: s_cmp_lg_u32 s8, 0
+; GFX942-GISEL-NEXT: s_cbranch_scc1 .LBB15_6
+; GFX942-GISEL-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
+; GFX942-GISEL-NEXT: s_lshr_b64 s[6:7], s[6:7], 1
+; GFX942-GISEL-NEXT: s_lshl_b32 s6, s6, 4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-GISEL-NEXT: v_add3_u32 v2, v0, s6, v1
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], 0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0xccccdddd
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 0xaaaabbbb
+; GFX942-GISEL-NEXT: .LBB15_5: ; %memset.pattern-expansion-residual-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: s_add_u32 s6, s6, 1
+; GFX942-GISEL-NEXT: s_addc_u32 s7, s7, 0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[6:7]
+; GFX942-GISEL-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]
+; GFX942-GISEL-NEXT: buffer_store_dwordx2 v[0:1], v2, s[0:3], 0 offen
+; GFX942-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX942-GISEL-NEXT: v_add_u32_e32 v2, 8, v2
+; GFX942-GISEL-NEXT: s_cbranch_scc1 .LBB15_5
+; GFX942-GISEL-NEXT: .LBB15_6: ; %memset.pattern-post-expansion
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %p = getelementptr i8, ptr addrspace(7) %a, i32 %offset
+ call void @llvm.experimental.memset.pattern(ptr addrspace(7) align(4) %p, i64 u0xaaaabbbbccccdddd, i64 %len, i1 false)
ret void
}
>From 000f6bf9cb7ec4223ef1de829158d09d4bfa2946 Mon Sep 17 00:00:00 2001
From: Zhen Wang <zhenw at nvidia.com>
Date: Mon, 6 Jul 2026 11:38:59 -0700
Subject: [PATCH 41/46] [flang][cuda] Recognize on_device() by its Fortran leaf
name in CUFFunctionRewrite (#207298)
CUFFunctionRewrite folds on_device() to a compile-time constant: true
inside a gpu.module (device), false on the host. Match on the callee's
Fortran leaf name, recovered independently of the external name-mangling
convention and of where the pass runs in the pipeline.
---
.../Transforms/CUDA/CUFFunctionRewrite.cpp | 55 +++++---
.../test/Fir/CUDA/cuda-function-rewrite.mlir | 124 ++++++++++++++++++
2 files changed, 163 insertions(+), 16 deletions(-)
diff --git a/flang/lib/Optimizer/Transforms/CUDA/CUFFunctionRewrite.cpp b/flang/lib/Optimizer/Transforms/CUDA/CUFFunctionRewrite.cpp
index bcbfb5294c56f..0c02aabfcc935 100644
--- a/flang/lib/Optimizer/Transforms/CUDA/CUFFunctionRewrite.cpp
+++ b/flang/lib/Optimizer/Transforms/CUDA/CUFFunctionRewrite.cpp
@@ -11,21 +11,23 @@
#include "flang/Optimizer/Dialect/FIROps.h"
#include "flang/Optimizer/Dialect/FIRType.h"
#include "flang/Optimizer/Support/DataLayout.h"
+#include "flang/Optimizer/Support/Utils.h"
#include "flang/Optimizer/Transforms/Passes.h"
#include "mlir/Dialect/GPU/IR/GPUDialect.h"
#include "mlir/IR/BuiltinTypes.h"
#include "mlir/IR/MLIRContext.h"
#include "mlir/IR/PatternMatch.h"
+#include "mlir/IR/SymbolTable.h"
#include "mlir/IR/ValueRange.h"
+#include "mlir/Interfaces/FunctionInterfaces.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Support/LogicalResult.h"
#include "mlir/Transforms/DialectConversion.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/StringSet.h"
#include "llvm/Support/Debug.h"
-#include <string_view>
#define DEBUG_TYPE "flang-cuf-function-rewrite"
@@ -52,32 +54,53 @@ class CallConversion : public OpRewritePattern<fir::CallOp> {
auto callee = op.getCallee();
if (!callee)
return failure();
- auto name = callee->getRootReference().getValue();
-
- if (genMappings_.contains(name)) {
- auto fct = genMappings_.find(name);
- mlir::Value result = fct->second(rewriter, op);
- if (result)
- rewriter.replaceOp(op, result);
- else
- rewriter.eraseOp(op);
- return success();
- }
- return failure();
+
+ // Match on the callee's Fortran leaf name rather than on its symbol name so
+ // this does not depend on the target's name-mangling convention or on where
+ // in the pipeline the pass runs. getPresentableFunctionName restores the
+ // original name saved by external-name conversion and returns the
+ // deconstructed leaf name.
+ auto func = mlir::dyn_cast_or_null<mlir::FunctionOpInterface>(
+ mlir::SymbolTable::lookupNearestSymbolFrom(op, *callee));
+ if (!func)
+ return failure();
+
+ auto fct = genMappings_.find(fir::getPresentableFunctionName(func));
+ if (fct == genMappings_.end())
+ return failure();
+
+ // Only rewrite a compiler-provided declaration, never a user-defined
+ // procedure that happens to share the name.
+ if (!func.isExternal())
+ return failure();
+
+ mlir::Value result = fct->second(rewriter, op);
+ if (!result)
+ return failure();
+ rewriter.replaceOp(op, result);
+ return success();
}
private:
static mlir::Value genOnDevice(mlir::PatternRewriter &rewriter,
fir::CallOp op) {
- assert(op.getArgs().size() == 0 && "expect 0 arguments");
+ // Only fold calls that match the intrinsic's shape: no arguments and a
+ // single logical result.
+ if (!op.getArgs().empty() || op.getNumResults() != 1)
+ return {};
+ mlir::Type resTy = op.getResult(0).getType();
+ if (!mlir::isa<fir::LogicalType>(resTy))
+ return {};
mlir::Location loc = op.getLoc();
unsigned inGPUMod = op->getParentOfType<gpu::GPUModuleOp>() ? 1 : 0;
mlir::Type i1Ty = rewriter.getIntegerType(1);
mlir::Value t = mlir::arith::ConstantOp::create(
rewriter, loc, i1Ty, rewriter.getIntegerAttr(i1Ty, inGPUMod));
- return fir::ConvertOp::create(rewriter, loc, op.getResult(0).getType(), t);
+ return fir::ConvertOp::create(rewriter, loc, resTy, t);
}
+ // Recognized by Fortran leaf name; see matchAndRewrite for how the leaf name
+ // is recovered independently of external name mangling.
const llvm::StringMap<genFunctionType> genMappings_ = {
{"on_device", &genOnDevice}};
};
diff --git a/flang/test/Fir/CUDA/cuda-function-rewrite.mlir b/flang/test/Fir/CUDA/cuda-function-rewrite.mlir
index da1d601a2eb8b..649064a46f54a 100644
--- a/flang/test/Fir/CUDA/cuda-function-rewrite.mlir
+++ b/flang/test/Fir/CUDA/cuda-function-rewrite.mlir
@@ -1,6 +1,8 @@
// RUN: fir-opt --split-input-file --cuf-function-rewrite %s | FileCheck %s
+// Test the bind(c) name "on_device" in device context.
gpu.module @cuda_device_mod {
+ func.func private @on_device() -> !fir.logical<4>
func.func @_QMmtestsPdo2(%arg0: !fir.ref<i32> {cuf.data_attr = #cuf.cuda<device>, fir.bindc_name = "c"}, %arg1: !fir.ref<i32> {cuf.data_attr = #cuf.cuda<device>, fir.bindc_name = "i"}) attributes {cuf.proc_attr = #cuf.cuda_proc<host_device>} {
%c2_i32 = arith.constant 2 : i32
%c1_i32 = arith.constant 1 : i32
@@ -24,6 +26,8 @@ gpu.module @cuda_device_mod {
// -----
+// Test the bind(c) name "on_device" on host side.
+func.func private @on_device() -> !fir.logical<4>
func.func @_QMmtestsPdo3(%arg0: !fir.ref<i32> {cuf.data_attr = #cuf.cuda<device>, fir.bindc_name = "c"}, %arg1: !fir.ref<i32> {cuf.data_attr = #cuf.cuda<device>, fir.bindc_name = "i"}) attributes {cuf.proc_attr = #cuf.cuda_proc<host_device>} {
%c2_i32 = arith.constant 2 : i32
%c1_i32 = arith.constant 1 : i32
@@ -42,3 +46,123 @@ func.func @_QMmtestsPdo3(%arg0: !fir.ref<i32> {cuf.data_attr = #cuf.cuda<device>
// CHECK-LABEL: func.func @_QMmtestsPdo3
// CHECK: fir.if %false
+
+// -----
+
+// Test on_device() with Fortran name mangling (_QPon_device) in device context.
+gpu.module @acc_device_mod {
+ func.func private @_QPon_device() -> !fir.logical<4>
+ func.func @_QMmtestPsub_device() {
+ %c2_i32 = arith.constant 2 : i32
+ %c1_i32 = arith.constant 1 : i32
+ %0 = fir.alloca i32
+ %13 = fir.call @_QPon_device() fastmath<contract> : () -> !fir.logical<4>
+ %14 = fir.convert %13 : (!fir.logical<4>) -> i1
+ fir.if %14 {
+ fir.store %c1_i32 to %0 : !fir.ref<i32>
+ } else {
+ fir.store %c2_i32 to %0 : !fir.ref<i32>
+ }
+ return
+ }
+}
+
+// CHECK-LABEL: gpu.module @acc_device_mod
+// CHECK: func.func @_QMmtestPsub_device
+// CHECK: fir.if %true
+
+// -----
+
+// Test _QPon_device on host side.
+func.func private @_QPon_device() -> !fir.logical<4>
+func.func @_QMmtestPsub_host() {
+ %c2_i32 = arith.constant 2 : i32
+ %c1_i32 = arith.constant 1 : i32
+ %0 = fir.alloca i32
+ %13 = fir.call @_QPon_device() fastmath<contract> : () -> !fir.logical<4>
+ %14 = fir.convert %13 : (!fir.logical<4>) -> i1
+ fir.if %14 {
+ fir.store %c1_i32 to %0 : !fir.ref<i32>
+ } else {
+ fir.store %c2_i32 to %0 : !fir.ref<i32>
+ }
+ return
+}
+
+// CHECK-LABEL: func.func @_QMmtestPsub_host
+// CHECK: fir.if %false
+
+// -----
+
+// Test externally-mangled on_device_ (after ExternalNameConversion) in device
+// context. The original name is recovered from the fir.internal_name attribute.
+gpu.module @acc_extname_device_mod {
+ func.func private @on_device_() -> !fir.logical<4> attributes {fir.internal_name = "_QPon_device"}
+ func.func @_QMmtestPsub_extname_device() {
+ %c2_i32 = arith.constant 2 : i32
+ %c1_i32 = arith.constant 1 : i32
+ %0 = fir.alloca i32
+ %13 = fir.call @on_device_() fastmath<contract> : () -> !fir.logical<4>
+ %14 = fir.convert %13 : (!fir.logical<4>) -> i1
+ fir.if %14 {
+ fir.store %c1_i32 to %0 : !fir.ref<i32>
+ } else {
+ fir.store %c2_i32 to %0 : !fir.ref<i32>
+ }
+ return
+ }
+}
+
+// CHECK-LABEL: gpu.module @acc_extname_device_mod
+// CHECK: func.func @_QMmtestPsub_extname_device
+// CHECK: fir.if %true
+
+// -----
+
+// Test on_device_ on host side (original name recovered from fir.internal_name).
+func.func private @on_device_() -> !fir.logical<4> attributes {fir.internal_name = "_QPon_device"}
+func.func @_QMmtestPsub_extname_host() {
+ %c2_i32 = arith.constant 2 : i32
+ %c1_i32 = arith.constant 1 : i32
+ %0 = fir.alloca i32
+ %13 = fir.call @on_device_() fastmath<contract> : () -> !fir.logical<4>
+ %14 = fir.convert %13 : (!fir.logical<4>) -> i1
+ fir.if %14 {
+ fir.store %c1_i32 to %0 : !fir.ref<i32>
+ } else {
+ fir.store %c2_i32 to %0 : !fir.ref<i32>
+ }
+ return
+}
+
+// CHECK-LABEL: func.func @_QMmtestPsub_extname_host
+// CHECK: fir.if %false
+
+// -----
+
+// A user-defined procedure named on_device (with a body) must not be folded.
+func.func @_QPon_device() -> !fir.logical<4> {
+ %true = arith.constant true
+ %0 = fir.convert %true : (i1) -> !fir.logical<4>
+ return %0 : !fir.logical<4>
+}
+func.func @_QMmtestPsub_userdef() -> !fir.logical<4> {
+ %13 = fir.call @_QPon_device() fastmath<contract> : () -> !fir.logical<4>
+ return %13 : !fir.logical<4>
+}
+
+// CHECK-LABEL: func.func @_QMmtestPsub_userdef
+// CHECK: fir.call @_QPon_device()
+
+// -----
+
+// A call whose signature does not match the intrinsic (extra argument) must not
+// be folded.
+func.func private @_QPon_device(i32) -> !fir.logical<4>
+func.func @_QMmtestPsub_badsig(%arg0: i32) -> !fir.logical<4> {
+ %13 = fir.call @_QPon_device(%arg0) fastmath<contract> : (i32) -> !fir.logical<4>
+ return %13 : !fir.logical<4>
+}
+
+// CHECK-LABEL: func.func @_QMmtestPsub_badsig
+// CHECK: fir.call @_QPon_device(%arg0)
>From 3ae61eb220dc8f1fa0dd551a9575237b84dae61d Mon Sep 17 00:00:00 2001
From: Ken Matsui <26405363+ken-matsui at users.noreply.github.com>
Date: Mon, 6 Jul 2026 14:39:14 -0400
Subject: [PATCH 42/46] [InstCombine] Turn Add into Or even when undef
(#171556)
Since 03d4a9d94da30590ebfc444cf13a8763f47b7bb9, Add is turned into
DisjointOr, and since 5c3496ff33ce8e4cc6f8c18edd7ae5fc65d23fdf, Add
isn't turned into DisjointOr when undef. However, Add can still be
turned into Or even when undef.
This patch restores the transformation behavior we had in LLVM 17, while
continuing to generate DisjointOr when possible.
Alive2: https://alive2.llvm.org/ce/z/ZT0ZFj
---
llvm/include/llvm/Analysis/ValueTracking.h | 16 +
llvm/lib/Analysis/ValueTracking.cpp | 83 ++--
.../InstCombine/InstCombineAddSub.cpp | 9 +-
llvm/test/Transforms/InstCombine/add.ll | 387 ++++++++++++++++--
.../InstCombine/masked-merge-add.ll | 184 ++++++++-
llvm/test/Transforms/InstCombine/pr53357.ll | 113 ++++-
llvm/unittests/Analysis/ValueTrackingTest.cpp | 116 ++++++
7 files changed, 840 insertions(+), 68 deletions(-)
diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index 684c22fdab2e4..c87e39bca8215 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -123,6 +123,22 @@ LLVM_ABI void adjustKnownFPClassForSelectArm(KnownFPClass &Known, Value *Cond,
const SimplifyQuery &Q,
unsigned Depth = 0);
+enum class NoCommonBitsSetResult {
+ /// Not known to have no common set bits.
+ Unknown,
+
+ /// Known to have no common set bits only if undef values are ignored.
+ OnlyIfUndefIgnored,
+
+ /// Known to have no common set bits.
+ Known,
+};
+
+/// Return how strongly LHS and RHS are known to have no common set bits.
+LLVM_ABI NoCommonBitsSetResult getNoCommonBitsSetResult(
+ const WithCache<const Value *> &LHSCache,
+ const WithCache<const Value *> &RHSCache, const SimplifyQuery &SQ);
+
/// Return true if LHS and RHS have no common bits set.
LLVM_ABI bool haveNoCommonBitsSet(const WithCache<const Value *> &LHSCache,
const WithCache<const Value *> &RHSCache,
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 5869ab28c6226..0271a59fc4450 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -178,46 +178,54 @@ KnownBits llvm::computeKnownBits(const Value *V, const APInt &DemandedElts,
SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo), Depth);
}
-static bool haveNoCommonBitsSetSpecialCases(const Value *LHS, const Value *RHS,
- const SimplifyQuery &SQ) {
+static NoCommonBitsSetResult
+haveNoCommonBitsSetSpecialCases(const Value *LHS, const Value *RHS,
+ const SimplifyQuery &SQ) {
// Look for an inverted mask: (X & ~M) op (Y & M).
{
Value *M;
if (match(LHS, m_c_And(m_Not(m_Value(M)), m_Value())) &&
- match(RHS, m_c_And(m_Specific(M), m_Value())) &&
- isGuaranteedNotToBeUndef(M, SQ.AC, SQ.CxtI, SQ.DT))
- return true;
+ match(RHS, m_c_And(m_Specific(M), m_Value())))
+ return isGuaranteedNotToBeUndef(M, SQ.AC, SQ.CxtI, SQ.DT)
+ ? NoCommonBitsSetResult::Known
+ : NoCommonBitsSetResult::OnlyIfUndefIgnored;
}
// X op (Y & ~X)
- if (match(RHS, m_c_And(m_Not(m_Specific(LHS)), m_Value())) &&
- isGuaranteedNotToBeUndef(LHS, SQ.AC, SQ.CxtI, SQ.DT))
- return true;
+ if (match(RHS, m_c_And(m_Not(m_Specific(LHS)), m_Value())))
+ return isGuaranteedNotToBeUndef(LHS, SQ.AC, SQ.CxtI, SQ.DT)
+ ? NoCommonBitsSetResult::Known
+ : NoCommonBitsSetResult::OnlyIfUndefIgnored;
// X op ((X & Y) ^ Y) -- this is the canonical form of the previous pattern
// for constant Y.
Value *Y;
if (match(RHS,
- m_c_Xor(m_c_And(m_Specific(LHS), m_Value(Y)), m_Deferred(Y))) &&
- isGuaranteedNotToBeUndef(LHS, SQ.AC, SQ.CxtI, SQ.DT) &&
- isGuaranteedNotToBeUndef(Y, SQ.AC, SQ.CxtI, SQ.DT))
- return true;
+ m_c_Xor(m_c_And(m_Specific(LHS), m_Value(Y)), m_Deferred(Y)))) {
+ bool IsNoUndef = isGuaranteedNotToBeUndef(LHS, SQ.AC, SQ.CxtI, SQ.DT) &&
+ isGuaranteedNotToBeUndef(Y, SQ.AC, SQ.CxtI, SQ.DT);
+ return IsNoUndef ? NoCommonBitsSetResult::Known
+ : NoCommonBitsSetResult::OnlyIfUndefIgnored;
+ }
// Peek through extends to find a 'not' of the other side:
// (ext Y) op ext(~Y)
if (match(LHS, m_ZExtOrSExt(m_Value(Y))) &&
- match(RHS, m_ZExtOrSExt(m_Not(m_Specific(Y)))) &&
- isGuaranteedNotToBeUndef(Y, SQ.AC, SQ.CxtI, SQ.DT))
- return true;
+ match(RHS, m_ZExtOrSExt(m_Not(m_Specific(Y)))))
+ return isGuaranteedNotToBeUndef(Y, SQ.AC, SQ.CxtI, SQ.DT)
+ ? NoCommonBitsSetResult::Known
+ : NoCommonBitsSetResult::OnlyIfUndefIgnored;
// Look for: (A & B) op ~(A | B)
{
Value *A, *B;
if (match(LHS, m_And(m_Value(A), m_Value(B))) &&
- match(RHS, m_Not(m_c_Or(m_Specific(A), m_Specific(B)))) &&
- isGuaranteedNotToBeUndef(A, SQ.AC, SQ.CxtI, SQ.DT) &&
- isGuaranteedNotToBeUndef(B, SQ.AC, SQ.CxtI, SQ.DT))
- return true;
+ match(RHS, m_Not(m_c_Or(m_Specific(A), m_Specific(B))))) {
+ bool IsNoUndef = isGuaranteedNotToBeUndef(A, SQ.AC, SQ.CxtI, SQ.DT) &&
+ isGuaranteedNotToBeUndef(B, SQ.AC, SQ.CxtI, SQ.DT);
+ return IsNoUndef ? NoCommonBitsSetResult::Known
+ : NoCommonBitsSetResult::OnlyIfUndefIgnored;
+ }
}
// Look for: (X << V) op (Y >> (BitWidth - V))
@@ -230,13 +238,14 @@ static bool haveNoCommonBitsSetSpecialCases(const Value *LHS, const Value *RHS,
(match(RHS, m_LShr(m_Value(), m_Sub(m_APInt(R), m_Value(V)))) &&
match(LHS, m_Shl(m_Value(), m_Specific(V))))) &&
R->uge(LHS->getType()->getScalarSizeInBits()))
- return true;
+ return NoCommonBitsSetResult::Known;
}
- return false;
+ return NoCommonBitsSetResult::Unknown;
}
-bool llvm::haveNoCommonBitsSet(const WithCache<const Value *> &LHSCache,
+NoCommonBitsSetResult
+llvm::getNoCommonBitsSetResult(const WithCache<const Value *> &LHSCache,
const WithCache<const Value *> &RHSCache,
const SimplifyQuery &SQ) {
const Value *LHS = LHSCache.getValue();
@@ -247,12 +256,32 @@ bool llvm::haveNoCommonBitsSet(const WithCache<const Value *> &LHSCache,
assert(LHS->getType()->isIntOrIntVectorTy() &&
"LHS and RHS should be integers");
- if (haveNoCommonBitsSetSpecialCases(LHS, RHS, SQ) ||
- haveNoCommonBitsSetSpecialCases(RHS, LHS, SQ))
- return true;
+ NoCommonBitsSetResult Result = haveNoCommonBitsSetSpecialCases(LHS, RHS, SQ);
+ if (Result == NoCommonBitsSetResult::Known)
+ return NoCommonBitsSetResult::Known;
+
+ NoCommonBitsSetResult CommuteResult =
+ haveNoCommonBitsSetSpecialCases(RHS, LHS, SQ);
+ if (CommuteResult == NoCommonBitsSetResult::Known)
+ return NoCommonBitsSetResult::Known;
+
+ if (KnownBits::haveNoCommonBitsSet(LHSCache.getKnownBits(SQ),
+ RHSCache.getKnownBits(SQ)))
+ return NoCommonBitsSetResult::Known;
- return KnownBits::haveNoCommonBitsSet(LHSCache.getKnownBits(SQ),
- RHSCache.getKnownBits(SQ));
+ if (Result == NoCommonBitsSetResult::OnlyIfUndefIgnored ||
+ CommuteResult == NoCommonBitsSetResult::OnlyIfUndefIgnored)
+ return NoCommonBitsSetResult::OnlyIfUndefIgnored;
+
+ return NoCommonBitsSetResult::Unknown;
+}
+
+bool llvm::haveNoCommonBitsSet(const WithCache<const Value *> &LHSCache,
+ const WithCache<const Value *> &RHSCache,
+ const SimplifyQuery &SQ) {
+ NoCommonBitsSetResult Result =
+ getNoCommonBitsSetResult(LHSCache, RHSCache, SQ);
+ return Result == NoCommonBitsSetResult::Known;
}
bool llvm::isOnlyUsedInZeroComparison(const Instruction *I) {
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index fc81e0070ffb6..6770b4c0e2da6 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -1770,8 +1770,15 @@ Instruction *InstCombinerImpl::visitAdd(BinaryOperator &I) {
// A+B --> A|B iff A and B have no bits set in common.
WithCache<const Value *> LHSCache(LHS), RHSCache(RHS);
- if (haveNoCommonBitsSet(LHSCache, RHSCache, SQ.getWithInstruction(&I)))
+ switch (
+ getNoCommonBitsSetResult(LHSCache, RHSCache, SQ.getWithInstruction(&I))) {
+ case NoCommonBitsSetResult::Known:
return BinaryOperator::CreateDisjointOr(LHS, RHS);
+ case NoCommonBitsSetResult::OnlyIfUndefIgnored:
+ return BinaryOperator::CreateOr(LHS, RHS);
+ case NoCommonBitsSetResult::Unknown:
+ break;
+ }
if (Instruction *Ext = narrowMathIfNoOverflow(I))
return Ext;
diff --git a/llvm/test/Transforms/InstCombine/add.ll b/llvm/test/Transforms/InstCombine/add.ll
index c8f774a3f72ec..1b84ecaf5eeeb 100644
--- a/llvm/test/Transforms/InstCombine/add.ll
+++ b/llvm/test/Transforms/InstCombine/add.ll
@@ -1996,7 +1996,7 @@ define i8 @add_like_or_disjoint(i8 %x) {
ret i8 %r
}
-define i8 @add_and_xor(i8 noundef %x, i8 %y) {
+define i8 @add_and_xor(i8 %x, i8 %y) {
; CHECK-LABEL: @add_and_xor(
; CHECK-NEXT: [[ADD:%.*]] = or i8 [[Y:%.*]], [[X:%.*]]
; CHECK-NEXT: ret i8 [[ADD]]
@@ -2007,6 +2007,17 @@ define i8 @add_and_xor(i8 noundef %x, i8 %y) {
ret i8 %add
}
+define i8 @add_and_xor_noundef(i8 noundef %x, i8 %y) {
+; CHECK-LABEL: @add_and_xor_noundef(
+; CHECK-NEXT: [[ADD:%.*]] = or i8 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT: ret i8 [[ADD]]
+;
+ %xor = xor i8 %x, -1
+ %and = and i8 %xor, %y
+ %add = add i8 %and, %x
+ ret i8 %add
+}
+
define i8 @add_and_xor_wrong_const(i8 %x, i8 %y) {
; CHECK-LABEL: @add_and_xor_wrong_const(
; CHECK-NEXT: [[XOR:%.*]] = xor i8 [[X:%.*]], -2
@@ -2033,7 +2044,7 @@ define i8 @add_and_xor_wrong_op(i8 %x, i8 %y, i8 %z) {
ret i8 %add
}
-define i8 @add_and_xor_commuted1(i8 noundef %x, i8 %_y) {
+define i8 @add_and_xor_commuted1(i8 %x, i8 %_y) {
; CHECK-LABEL: @add_and_xor_commuted1(
; CHECK-NEXT: [[Y:%.*]] = udiv i8 42, [[_Y:%.*]]
; CHECK-NEXT: [[ADD:%.*]] = or i8 [[Y]], [[X:%.*]]
@@ -2046,7 +2057,20 @@ define i8 @add_and_xor_commuted1(i8 noundef %x, i8 %_y) {
ret i8 %add
}
-define i8 @add_and_xor_commuted2(i8 noundef %_x, i8 %y) {
+define i8 @add_and_xor_commuted1_noundef(i8 noundef %x, i8 %_y) {
+; CHECK-LABEL: @add_and_xor_commuted1_noundef(
+; CHECK-NEXT: [[Y:%.*]] = udiv i8 42, [[_Y:%.*]]
+; CHECK-NEXT: [[ADD:%.*]] = or i8 [[Y]], [[X:%.*]]
+; CHECK-NEXT: ret i8 [[ADD]]
+;
+ %y = udiv i8 42, %_y ; thwart complexity-based canonicalization
+ %xor = xor i8 %x, -1
+ %and = and i8 %y, %xor
+ %add = add i8 %and, %x
+ ret i8 %add
+}
+
+define i8 @add_and_xor_commuted2(i8 %_x, i8 %y) {
; CHECK-LABEL: @add_and_xor_commuted2(
; CHECK-NEXT: [[X:%.*]] = udiv i8 42, [[_X:%.*]]
; CHECK-NEXT: [[ADD:%.*]] = or i8 [[X]], [[Y:%.*]]
@@ -2059,7 +2083,20 @@ define i8 @add_and_xor_commuted2(i8 noundef %_x, i8 %y) {
ret i8 %add
}
-define i8 @add_and_xor_commuted3(i8 noundef %_x, i8 %_y) {
+define i8 @add_and_xor_commuted2_noundef(i8 noundef %_x, i8 %y) {
+; CHECK-LABEL: @add_and_xor_commuted2_noundef(
+; CHECK-NEXT: [[X:%.*]] = udiv i8 42, [[_X:%.*]]
+; CHECK-NEXT: [[ADD:%.*]] = or i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT: ret i8 [[ADD]]
+;
+ %x = udiv i8 42, %_x ; thwart complexity-based canonicalization
+ %xor = xor i8 %x, -1
+ %and = and i8 %xor, %y
+ %add = add i8 %x, %and
+ ret i8 %add
+}
+
+define i8 @add_and_xor_commuted3(i8 %_x, i8 %_y) {
; CHECK-LABEL: @add_and_xor_commuted3(
; CHECK-NEXT: [[X:%.*]] = udiv i8 42, [[_X:%.*]]
; CHECK-NEXT: [[Y:%.*]] = udiv i8 42, [[_Y:%.*]]
@@ -2074,7 +2111,22 @@ define i8 @add_and_xor_commuted3(i8 noundef %_x, i8 %_y) {
ret i8 %add
}
-define i8 @add_and_xor_extra_use(i8 noundef %x, i8 %y) {
+define i8 @add_and_xor_commuted3_noundef(i8 noundef %_x, i8 %_y) {
+; CHECK-LABEL: @add_and_xor_commuted3_noundef(
+; CHECK-NEXT: [[X:%.*]] = udiv i8 42, [[_X:%.*]]
+; CHECK-NEXT: [[Y:%.*]] = udiv i8 42, [[_Y:%.*]]
+; CHECK-NEXT: [[ADD:%.*]] = or i8 [[X]], [[Y]]
+; CHECK-NEXT: ret i8 [[ADD]]
+;
+ %x = udiv i8 42, %_x ; thwart complexity-based canonicalization
+ %y = udiv i8 42, %_y ; thwart complexity-based canonicalization
+ %xor = xor i8 %x, -1
+ %and = and i8 %y, %xor
+ %add = add i8 %x, %and
+ ret i8 %add
+}
+
+define i8 @add_and_xor_extra_use(i8 %x, i8 %y) {
; CHECK-LABEL: @add_and_xor_extra_use(
; CHECK-NEXT: [[XOR:%.*]] = xor i8 [[X:%.*]], -1
; CHECK-NEXT: call void @use(i8 [[XOR]])
@@ -2091,7 +2143,24 @@ define i8 @add_and_xor_extra_use(i8 noundef %x, i8 %y) {
ret i8 %add
}
-define i8 @add_xor_and_const(i8 noundef %x) {
+define i8 @add_and_xor_extra_use_noundef(i8 noundef %x, i8 %y) {
+; CHECK-LABEL: @add_and_xor_extra_use_noundef(
+; CHECK-NEXT: [[XOR:%.*]] = xor i8 [[X:%.*]], -1
+; CHECK-NEXT: call void @use(i8 [[XOR]])
+; CHECK-NEXT: [[AND:%.*]] = and i8 [[Y:%.*]], [[XOR]]
+; CHECK-NEXT: call void @use(i8 [[AND]])
+; CHECK-NEXT: [[ADD:%.*]] = or i8 [[Y]], [[X]]
+; CHECK-NEXT: ret i8 [[ADD]]
+;
+ %xor = xor i8 %x, -1
+ call void @use(i8 %xor)
+ %and = and i8 %xor, %y
+ call void @use(i8 %and)
+ %add = add i8 %and, %x
+ ret i8 %add
+}
+
+define i8 @add_xor_and_const(i8 %x) {
; CHECK-LABEL: @add_xor_and_const(
; CHECK-NEXT: [[ADD:%.*]] = or i8 [[X:%.*]], 42
; CHECK-NEXT: ret i8 [[ADD]]
@@ -2102,6 +2171,17 @@ define i8 @add_xor_and_const(i8 noundef %x) {
ret i8 %add
}
+define i8 @add_xor_and_const_noundef(i8 noundef %x) {
+; CHECK-LABEL: @add_xor_and_const_noundef(
+; CHECK-NEXT: [[ADD:%.*]] = or i8 [[X:%.*]], 42
+; CHECK-NEXT: ret i8 [[ADD]]
+;
+ %and = and i8 %x, 42
+ %xor = xor i8 %and, 42
+ %add = add i8 %xor, %x
+ ret i8 %add
+}
+
define i8 @add_xor_and_const_wrong_const(i8 %x) {
; CHECK-LABEL: @add_xor_and_const_wrong_const(
; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], 42
@@ -2115,7 +2195,7 @@ define i8 @add_xor_and_const_wrong_const(i8 %x) {
ret i8 %add
}
-define i8 @add_xor_and_var(i8 noundef %x, i8 noundef %y) {
+define i8 @add_xor_and_var(i8 %x, i8 %y) {
; CHECK-LABEL: @add_xor_and_var(
; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], [[Y:%.*]]
; CHECK-NEXT: call void @use(i8 [[AND]])
@@ -2129,6 +2209,20 @@ define i8 @add_xor_and_var(i8 noundef %x, i8 noundef %y) {
ret i8 %add
}
+define i8 @add_xor_and_var_noundef(i8 noundef %x, i8 noundef %y) {
+; CHECK-LABEL: @add_xor_and_var_noundef(
+; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: call void @use(i8 [[AND]])
+; CHECK-NEXT: [[ADD:%.*]] = or i8 [[Y]], [[X]]
+; CHECK-NEXT: ret i8 [[ADD]]
+;
+ %and = and i8 %x, %y
+ call void @use(i8 %and)
+ %xor = xor i8 %and, %y
+ %add = add i8 %xor, %x
+ ret i8 %add
+}
+
define i8 @add_xor_and_var_wrong_op1(i8 %x, i8 %y, i8 %z) {
; CHECK-LABEL: @add_xor_and_var_wrong_op1(
; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], [[Y:%.*]]
@@ -2159,7 +2253,7 @@ define i8 @add_xor_and_var_wrong_op2(i8 %x, i8 %y, i8 %z) {
ret i8 %add
}
-define i8 @add_xor_and_var_commuted1(i8 noundef %x, i8 noundef %y) {
+define i8 @add_xor_and_var_commuted1(i8 %x, i8 %y) {
; CHECK-LABEL: @add_xor_and_var_commuted1(
; CHECK-NEXT: [[AND:%.*]] = and i8 [[Y:%.*]], [[X:%.*]]
; CHECK-NEXT: call void @use(i8 [[AND]])
@@ -2173,7 +2267,21 @@ define i8 @add_xor_and_var_commuted1(i8 noundef %x, i8 noundef %y) {
ret i8 %add
}
-define i8 @add_xor_and_var_commuted2(i8 noundef %_x, i8 noundef %_y) {
+define i8 @add_xor_and_var_commuted1_noundef(i8 noundef %x, i8 noundef %y) {
+; CHECK-LABEL: @add_xor_and_var_commuted1_noundef(
+; CHECK-NEXT: [[AND:%.*]] = and i8 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT: call void @use(i8 [[AND]])
+; CHECK-NEXT: [[ADD:%.*]] = or i8 [[Y]], [[X]]
+; CHECK-NEXT: ret i8 [[ADD]]
+;
+ %and = and i8 %y, %x
+ call void @use(i8 %and)
+ %xor = xor i8 %and, %y
+ %add = add i8 %xor, %x
+ ret i8 %add
+}
+
+define i8 @add_xor_and_var_commuted2(i8 %_x, i8 %_y) {
; CHECK-LABEL: @add_xor_and_var_commuted2(
; CHECK-NEXT: [[X:%.*]] = udiv i8 42, [[_X:%.*]]
; CHECK-NEXT: [[Y:%.*]] = udiv i8 42, [[_Y:%.*]]
@@ -2191,7 +2299,25 @@ define i8 @add_xor_and_var_commuted2(i8 noundef %_x, i8 noundef %_y) {
ret i8 %add
}
-define i8 @add_xor_and_var_commuted3(i8 noundef %x, i8 noundef %_y) {
+define i8 @add_xor_and_var_commuted2_noundef(i8 noundef %_x, i8 noundef %_y) {
+; CHECK-LABEL: @add_xor_and_var_commuted2_noundef(
+; CHECK-NEXT: [[X:%.*]] = udiv i8 42, [[_X:%.*]]
+; CHECK-NEXT: [[Y:%.*]] = udiv i8 42, [[_Y:%.*]]
+; CHECK-NEXT: [[AND:%.*]] = and i8 [[X]], [[Y]]
+; CHECK-NEXT: call void @use(i8 [[AND]])
+; CHECK-NEXT: [[ADD:%.*]] = or i8 [[Y]], [[X]]
+; CHECK-NEXT: ret i8 [[ADD]]
+;
+ %x = udiv i8 42, %_x ; thwart complexity-based canonicalization
+ %y = udiv i8 42, %_y ; thwart complexity-based canonicalization
+ %and = and i8 %x, %y
+ call void @use(i8 %and)
+ %xor = xor i8 %y, %and
+ %add = add i8 %xor, %x
+ ret i8 %add
+}
+
+define i8 @add_xor_and_var_commuted3(i8 %x, i8 %_y) {
; CHECK-LABEL: @add_xor_and_var_commuted3(
; CHECK-NEXT: [[Y:%.*]] = udiv i8 42, [[_Y:%.*]]
; CHECK-NEXT: [[AND:%.*]] = and i8 [[Y]], [[X:%.*]]
@@ -2207,7 +2333,23 @@ define i8 @add_xor_and_var_commuted3(i8 noundef %x, i8 noundef %_y) {
ret i8 %add
}
-define i8 @add_xor_and_var_commuted4(i8 noundef %_x, i8 noundef %y) {
+define i8 @add_xor_and_var_commuted3_noundef(i8 noundef %x, i8 noundef %_y) {
+; CHECK-LABEL: @add_xor_and_var_commuted3_noundef(
+; CHECK-NEXT: [[Y:%.*]] = udiv i8 42, [[_Y:%.*]]
+; CHECK-NEXT: [[AND:%.*]] = and i8 [[Y]], [[X:%.*]]
+; CHECK-NEXT: call void @use(i8 [[AND]])
+; CHECK-NEXT: [[ADD:%.*]] = or i8 [[Y]], [[X]]
+; CHECK-NEXT: ret i8 [[ADD]]
+;
+ %y = udiv i8 42, %_y ; thwart complexity-based canonicalization
+ %and = and i8 %y, %x
+ call void @use(i8 %and)
+ %xor = xor i8 %y, %and
+ %add = add i8 %xor, %x
+ ret i8 %add
+}
+
+define i8 @add_xor_and_var_commuted4(i8 %_x, i8 %y) {
; CHECK-LABEL: @add_xor_and_var_commuted4(
; CHECK-NEXT: [[X:%.*]] = udiv i8 42, [[_X:%.*]]
; CHECK-NEXT: [[AND:%.*]] = and i8 [[X]], [[Y:%.*]]
@@ -2223,7 +2365,23 @@ define i8 @add_xor_and_var_commuted4(i8 noundef %_x, i8 noundef %y) {
ret i8 %add
}
-define i8 @add_xor_and_var_commuted5(i8 noundef %_x, i8 noundef %_y) {
+define i8 @add_xor_and_var_commuted4_noundef(i8 noundef %_x, i8 noundef %y) {
+; CHECK-LABEL: @add_xor_and_var_commuted4_noundef(
+; CHECK-NEXT: [[X:%.*]] = udiv i8 42, [[_X:%.*]]
+; CHECK-NEXT: [[AND:%.*]] = and i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT: call void @use(i8 [[AND]])
+; CHECK-NEXT: [[ADD:%.*]] = or i8 [[X]], [[Y]]
+; CHECK-NEXT: ret i8 [[ADD]]
+;
+ %x = udiv i8 42, %_x ; thwart complexity-based canonicalization
+ %and = and i8 %x, %y
+ call void @use(i8 %and)
+ %xor = xor i8 %and, %y
+ %add = add i8 %x, %xor
+ ret i8 %add
+}
+
+define i8 @add_xor_and_var_commuted5(i8 %_x, i8 %_y) {
; CHECK-LABEL: @add_xor_and_var_commuted5(
; CHECK-NEXT: [[X:%.*]] = udiv i8 42, [[_X:%.*]]
; CHECK-NEXT: [[Y:%.*]] = udiv i8 42, [[_Y:%.*]]
@@ -2241,7 +2399,25 @@ define i8 @add_xor_and_var_commuted5(i8 noundef %_x, i8 noundef %_y) {
ret i8 %add
}
-define i8 @add_xor_and_var_commuted6(i8 noundef %_x, i8 noundef %_y) {
+define i8 @add_xor_and_var_commuted5_noundef(i8 noundef %_x, i8 noundef %_y) {
+; CHECK-LABEL: @add_xor_and_var_commuted5_noundef(
+; CHECK-NEXT: [[X:%.*]] = udiv i8 42, [[_X:%.*]]
+; CHECK-NEXT: [[Y:%.*]] = udiv i8 42, [[_Y:%.*]]
+; CHECK-NEXT: [[AND:%.*]] = and i8 [[Y]], [[X]]
+; CHECK-NEXT: call void @use(i8 [[AND]])
+; CHECK-NEXT: [[ADD:%.*]] = or i8 [[X]], [[Y]]
+; CHECK-NEXT: ret i8 [[ADD]]
+;
+ %x = udiv i8 42, %_x ; thwart complexity-based canonicalization
+ %y = udiv i8 42, %_y ; thwart complexity-based canonicalization
+ %and = and i8 %y, %x
+ call void @use(i8 %and)
+ %xor = xor i8 %and, %y
+ %add = add i8 %x, %xor
+ ret i8 %add
+}
+
+define i8 @add_xor_and_var_commuted6(i8 %_x, i8 %_y) {
; CHECK-LABEL: @add_xor_and_var_commuted6(
; CHECK-NEXT: [[X:%.*]] = udiv i8 42, [[_X:%.*]]
; CHECK-NEXT: [[Y:%.*]] = udiv i8 42, [[_Y:%.*]]
@@ -2259,7 +2435,25 @@ define i8 @add_xor_and_var_commuted6(i8 noundef %_x, i8 noundef %_y) {
ret i8 %add
}
-define i8 @add_xor_and_var_commuted7(i8 noundef %_x, i8 noundef %_y) {
+define i8 @add_xor_and_var_commuted6_noundef(i8 noundef %_x, i8 noundef %_y) {
+; CHECK-LABEL: @add_xor_and_var_commuted6_noundef(
+; CHECK-NEXT: [[X:%.*]] = udiv i8 42, [[_X:%.*]]
+; CHECK-NEXT: [[Y:%.*]] = udiv i8 42, [[_Y:%.*]]
+; CHECK-NEXT: [[AND:%.*]] = and i8 [[X]], [[Y]]
+; CHECK-NEXT: call void @use(i8 [[AND]])
+; CHECK-NEXT: [[ADD:%.*]] = or i8 [[X]], [[Y]]
+; CHECK-NEXT: ret i8 [[ADD]]
+;
+ %x = udiv i8 42, %_x ; thwart complexity-based canonicalization
+ %y = udiv i8 42, %_y ; thwart complexity-based canonicalization
+ %and = and i8 %x, %y
+ call void @use(i8 %and)
+ %xor = xor i8 %y, %and
+ %add = add i8 %x, %xor
+ ret i8 %add
+}
+
+define i8 @add_xor_and_var_commuted7(i8 %_x, i8 %_y) {
; CHECK-LABEL: @add_xor_and_var_commuted7(
; CHECK-NEXT: [[X:%.*]] = udiv i8 42, [[_X:%.*]]
; CHECK-NEXT: [[Y:%.*]] = udiv i8 42, [[_Y:%.*]]
@@ -2277,7 +2471,25 @@ define i8 @add_xor_and_var_commuted7(i8 noundef %_x, i8 noundef %_y) {
ret i8 %add
}
-define i8 @add_xor_and_var_extra_use(i8 noundef %x, i8 noundef %y) {
+define i8 @add_xor_and_var_commuted7_noundef(i8 noundef %_x, i8 noundef %_y) {
+; CHECK-LABEL: @add_xor_and_var_commuted7_noundef(
+; CHECK-NEXT: [[X:%.*]] = udiv i8 42, [[_X:%.*]]
+; CHECK-NEXT: [[Y:%.*]] = udiv i8 42, [[_Y:%.*]]
+; CHECK-NEXT: [[AND:%.*]] = and i8 [[Y]], [[X]]
+; CHECK-NEXT: call void @use(i8 [[AND]])
+; CHECK-NEXT: [[ADD:%.*]] = or i8 [[X]], [[Y]]
+; CHECK-NEXT: ret i8 [[ADD]]
+;
+ %x = udiv i8 42, %_x ; thwart complexity-based canonicalization
+ %y = udiv i8 42, %_y ; thwart complexity-based canonicalization
+ %and = and i8 %y, %x
+ call void @use(i8 %and)
+ %xor = xor i8 %y, %and
+ %add = add i8 %x, %xor
+ ret i8 %add
+}
+
+define i8 @add_xor_and_var_extra_use(i8 %x, i8 %y) {
; CHECK-LABEL: @add_xor_and_var_extra_use(
; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], [[Y:%.*]]
; CHECK-NEXT: call void @use(i8 [[AND]])
@@ -2294,6 +2506,23 @@ define i8 @add_xor_and_var_extra_use(i8 noundef %x, i8 noundef %y) {
ret i8 %add
}
+define i8 @add_xor_and_var_extra_use_noundef(i8 noundef %x, i8 noundef %y) {
+; CHECK-LABEL: @add_xor_and_var_extra_use_noundef(
+; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: call void @use(i8 [[AND]])
+; CHECK-NEXT: [[XOR:%.*]] = xor i8 [[AND]], [[Y]]
+; CHECK-NEXT: call void @use(i8 [[XOR]])
+; CHECK-NEXT: [[ADD:%.*]] = or i8 [[Y]], [[X]]
+; CHECK-NEXT: ret i8 [[ADD]]
+;
+ %and = and i8 %x, %y
+ call void @use(i8 %and)
+ %xor = xor i8 %and, %y
+ call void @use(i8 %xor)
+ %add = add i8 %xor, %x
+ ret i8 %add
+}
+
define i32 @add_add_add(i32 %A, i32 %B, i32 %C, i32 %D) {
; CHECK-LABEL: @add_add_add(
; CHECK-NEXT: [[E:%.*]] = add i32 [[A:%.*]], [[B:%.*]]
@@ -3058,7 +3287,7 @@ define <vscale x 1 x i32> @add_to_or_scalable(<vscale x 1 x i32> %in) {
ret <vscale x 1 x i32> %add
}
-define i5 @zext_zext_not(i3 noundef %x) {
+define i5 @zext_zext_not(i3 %x) {
; CHECK-LABEL: @zext_zext_not(
; CHECK-NEXT: ret i5 7
;
@@ -3069,7 +3298,18 @@ define i5 @zext_zext_not(i3 noundef %x) {
ret i5 %r
}
-define <2 x i5> @zext_zext_not_commute(<2 x i3> noundef %x) {
+define i5 @zext_zext_not_noundef(i3 noundef %x) {
+; CHECK-LABEL: @zext_zext_not_noundef(
+; CHECK-NEXT: ret i5 7
+;
+ %zx = zext i3 %x to i5
+ %notx = xor i3 %x, -1
+ %znotx = zext i3 %notx to i5
+ %r = add i5 %zx, %znotx
+ ret i5 %r
+}
+
+define <2 x i5> @zext_zext_not_commute(<2 x i3> %x) {
; CHECK-LABEL: @zext_zext_not_commute(
; CHECK-NEXT: ret <2 x i5> splat (i5 7)
;
@@ -3080,7 +3320,18 @@ define <2 x i5> @zext_zext_not_commute(<2 x i3> noundef %x) {
ret <2 x i5> %r
}
-define i9 @sext_sext_not(i3 noundef %x) {
+define <2 x i5> @zext_zext_not_commute_noundef(<2 x i3> noundef %x) {
+; CHECK-LABEL: @zext_zext_not_commute_noundef(
+; CHECK-NEXT: ret <2 x i5> splat (i5 7)
+;
+ %zx = zext <2 x i3> %x to <2 x i5>
+ %notx = xor <2 x i3> %x, <i3 -1, i3 poison>
+ %znotx = zext <2 x i3> %notx to <2 x i5>
+ %r = add <2 x i5> %znotx, %zx
+ ret <2 x i5> %r
+}
+
+define i9 @sext_sext_not(i3 %x) {
; CHECK-LABEL: @sext_sext_not(
; CHECK-NEXT: ret i9 -1
;
@@ -3091,7 +3342,18 @@ define i9 @sext_sext_not(i3 noundef %x) {
ret i9 %r
}
-define i8 @sext_sext_not_commute(i3 noundef %x) {
+define i9 @sext_sext_not_noundef(i3 noundef %x) {
+; CHECK-LABEL: @sext_sext_not_noundef(
+; CHECK-NEXT: ret i9 -1
+;
+ %sx = sext i3 %x to i9
+ %notx = xor i3 %x, -1
+ %snotx = sext i3 %notx to i9
+ %r = add i9 %sx, %snotx
+ ret i9 %r
+}
+
+define i8 @sext_sext_not_commute(i3 %x) {
; CHECK-LABEL: @sext_sext_not_commute(
; CHECK-NEXT: [[SX:%.*]] = sext i3 [[X:%.*]] to i8
; CHECK-NEXT: call void @use(i8 [[SX]])
@@ -3106,11 +3368,41 @@ define i8 @sext_sext_not_commute(i3 noundef %x) {
ret i8 %r
}
-define i5 @zext_sext_not(i4 noundef %x) {
+define i8 @sext_sext_not_commute_noundef(i3 noundef %x) {
+; CHECK-LABEL: @sext_sext_not_commute_noundef(
+; CHECK-NEXT: [[SX:%.*]] = sext i3 [[X:%.*]] to i8
+; CHECK-NEXT: call void @use(i8 [[SX]])
+; CHECK-NEXT: ret i8 -1
+;
+
+ %sx = sext i3 %x to i8
+ call void @use(i8 %sx)
+ %notx = xor i3 %x, -1
+ %snotx = sext i3 %notx to i8
+ %r = add i8 %snotx, %sx
+ ret i8 %r
+}
+
+define i5 @zext_sext_not(i4 %x) {
; CHECK-LABEL: @zext_sext_not(
; CHECK-NEXT: [[ZX:%.*]] = zext i4 [[X:%.*]] to i5
; CHECK-NEXT: [[NOTX:%.*]] = xor i4 [[X]], -1
; CHECK-NEXT: [[SNOTX:%.*]] = sext i4 [[NOTX]] to i5
+; CHECK-NEXT: [[R:%.*]] = or i5 [[ZX]], [[SNOTX]]
+; CHECK-NEXT: ret i5 [[R]]
+;
+ %zx = zext i4 %x to i5
+ %notx = xor i4 %x, -1
+ %snotx = sext i4 %notx to i5
+ %r = add i5 %zx, %snotx
+ ret i5 %r
+}
+
+define i5 @zext_sext_not_noundef(i4 noundef %x) {
+; CHECK-LABEL: @zext_sext_not_noundef(
+; CHECK-NEXT: [[ZX:%.*]] = zext i4 [[X:%.*]] to i5
+; CHECK-NEXT: [[NOTX:%.*]] = xor i4 [[X]], -1
+; CHECK-NEXT: [[SNOTX:%.*]] = sext i4 [[NOTX]] to i5
; CHECK-NEXT: [[R:%.*]] = or disjoint i5 [[ZX]], [[SNOTX]]
; CHECK-NEXT: ret i5 [[R]]
;
@@ -3121,13 +3413,32 @@ define i5 @zext_sext_not(i4 noundef %x) {
ret i5 %r
}
-define i8 @zext_sext_not_commute(i4 noundef %x) {
+define i8 @zext_sext_not_commute(i4 %x) {
; CHECK-LABEL: @zext_sext_not_commute(
; CHECK-NEXT: [[ZX:%.*]] = zext i4 [[X:%.*]] to i8
; CHECK-NEXT: call void @use(i8 [[ZX]])
; CHECK-NEXT: [[NOTX:%.*]] = xor i4 [[X]], -1
; CHECK-NEXT: [[SNOTX:%.*]] = sext i4 [[NOTX]] to i8
; CHECK-NEXT: call void @use(i8 [[SNOTX]])
+; CHECK-NEXT: [[R:%.*]] = or i8 [[SNOTX]], [[ZX]]
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %zx = zext i4 %x to i8
+ call void @use(i8 %zx)
+ %notx = xor i4 %x, -1
+ %snotx = sext i4 %notx to i8
+ call void @use(i8 %snotx)
+ %r = add i8 %snotx, %zx
+ ret i8 %r
+}
+
+define i8 @zext_sext_not_commute_noundef(i4 noundef %x) {
+; CHECK-LABEL: @zext_sext_not_commute_noundef(
+; CHECK-NEXT: [[ZX:%.*]] = zext i4 [[X:%.*]] to i8
+; CHECK-NEXT: call void @use(i8 [[ZX]])
+; CHECK-NEXT: [[NOTX:%.*]] = xor i4 [[X]], -1
+; CHECK-NEXT: [[SNOTX:%.*]] = sext i4 [[NOTX]] to i8
+; CHECK-NEXT: call void @use(i8 [[SNOTX]])
; CHECK-NEXT: [[R:%.*]] = or disjoint i8 [[SNOTX]], [[ZX]]
; CHECK-NEXT: ret i8 [[R]]
;
@@ -3140,11 +3451,26 @@ define i8 @zext_sext_not_commute(i4 noundef %x) {
ret i8 %r
}
-define i9 @sext_zext_not(i4 noundef %x) {
+define i9 @sext_zext_not(i4 %x) {
; CHECK-LABEL: @sext_zext_not(
; CHECK-NEXT: [[SX:%.*]] = sext i4 [[X:%.*]] to i9
; CHECK-NEXT: [[NOTX:%.*]] = xor i4 [[X]], -1
; CHECK-NEXT: [[ZNOTX:%.*]] = zext i4 [[NOTX]] to i9
+; CHECK-NEXT: [[R:%.*]] = or i9 [[SX]], [[ZNOTX]]
+; CHECK-NEXT: ret i9 [[R]]
+;
+ %sx = sext i4 %x to i9
+ %notx = xor i4 %x, -1
+ %znotx = zext i4 %notx to i9
+ %r = add i9 %sx, %znotx
+ ret i9 %r
+}
+
+define i9 @sext_zext_not_noundef(i4 noundef %x) {
+; CHECK-LABEL: @sext_zext_not_noundef(
+; CHECK-NEXT: [[SX:%.*]] = sext i4 [[X:%.*]] to i9
+; CHECK-NEXT: [[NOTX:%.*]] = xor i4 [[X]], -1
+; CHECK-NEXT: [[ZNOTX:%.*]] = zext i4 [[NOTX]] to i9
; CHECK-NEXT: [[R:%.*]] = or disjoint i9 [[SX]], [[ZNOTX]]
; CHECK-NEXT: ret i9 [[R]]
;
@@ -3155,11 +3481,26 @@ define i9 @sext_zext_not(i4 noundef %x) {
ret i9 %r
}
-define i9 @sext_zext_not_commute(i4 noundef %x) {
+define i9 @sext_zext_not_commute(i4 %x) {
; CHECK-LABEL: @sext_zext_not_commute(
; CHECK-NEXT: [[SX:%.*]] = sext i4 [[X:%.*]] to i9
; CHECK-NEXT: [[NOTX:%.*]] = xor i4 [[X]], -1
; CHECK-NEXT: [[ZNOTX:%.*]] = zext i4 [[NOTX]] to i9
+; CHECK-NEXT: [[R:%.*]] = or i9 [[ZNOTX]], [[SX]]
+; CHECK-NEXT: ret i9 [[R]]
+;
+ %sx = sext i4 %x to i9
+ %notx = xor i4 %x, -1
+ %znotx = zext i4 %notx to i9
+ %r = add i9 %znotx, %sx
+ ret i9 %r
+}
+
+define i9 @sext_zext_not_commute_noundef(i4 noundef %x) {
+; CHECK-LABEL: @sext_zext_not_commute_noundef(
+; CHECK-NEXT: [[SX:%.*]] = sext i4 [[X:%.*]] to i9
+; CHECK-NEXT: [[NOTX:%.*]] = xor i4 [[X]], -1
+; CHECK-NEXT: [[ZNOTX:%.*]] = zext i4 [[NOTX]] to i9
; CHECK-NEXT: [[R:%.*]] = or disjoint i9 [[ZNOTX]], [[SX]]
; CHECK-NEXT: ret i9 [[R]]
;
diff --git a/llvm/test/Transforms/InstCombine/masked-merge-add.ll b/llvm/test/Transforms/InstCombine/masked-merge-add.ll
index d78a9ee59c714..c455f119b5025 100644
--- a/llvm/test/Transforms/InstCombine/masked-merge-add.ll
+++ b/llvm/test/Transforms/InstCombine/masked-merge-add.ll
@@ -16,11 +16,26 @@
; Most basic positive tests
; ============================================================================ ;
-define i32 @p(i32 %x, i32 %y, i32 noundef %m) {
+define i32 @p(i32 %x, i32 %y, i32 %m) {
; CHECK-LABEL: @p(
; CHECK-NEXT: [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
; CHECK-NEXT: [[NEG:%.*]] = xor i32 [[M]], -1
; CHECK-NEXT: [[AND1:%.*]] = and i32 [[Y:%.*]], [[NEG]]
+; CHECK-NEXT: [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT: ret i32 [[RET]]
+;
+ %and = and i32 %x, %m
+ %neg = xor i32 %m, -1
+ %and1 = and i32 %neg, %y
+ %ret = add i32 %and, %and1
+ ret i32 %ret
+}
+
+define i32 @p_noundef(i32 %x, i32 %y, i32 noundef %m) {
+; CHECK-LABEL: @p_noundef(
+; CHECK-NEXT: [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT: [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT: [[AND1:%.*]] = and i32 [[Y:%.*]], [[NEG]]
; CHECK-NEXT: [[RET:%.*]] = or disjoint i32 [[AND]], [[AND1]]
; CHECK-NEXT: ret i32 [[RET]]
;
@@ -31,11 +46,26 @@ define i32 @p(i32 %x, i32 %y, i32 noundef %m) {
ret i32 %ret
}
-define <2 x i32> @p_splatvec(<2 x i32> %x, <2 x i32> %y, <2 x i32> noundef %m) {
+define <2 x i32> @p_splatvec(<2 x i32> %x, <2 x i32> %y, <2 x i32> %m) {
; CHECK-LABEL: @p_splatvec(
; CHECK-NEXT: [[AND:%.*]] = and <2 x i32> [[X:%.*]], [[M:%.*]]
; CHECK-NEXT: [[NEG:%.*]] = xor <2 x i32> [[M]], splat (i32 -1)
; CHECK-NEXT: [[AND1:%.*]] = and <2 x i32> [[Y:%.*]], [[NEG]]
+; CHECK-NEXT: [[RET:%.*]] = or <2 x i32> [[AND]], [[AND1]]
+; CHECK-NEXT: ret <2 x i32> [[RET]]
+;
+ %and = and <2 x i32> %x, %m
+ %neg = xor <2 x i32> %m, <i32 -1, i32 -1>
+ %and1 = and <2 x i32> %neg, %y
+ %ret = add <2 x i32> %and, %and1
+ ret <2 x i32> %ret
+}
+
+define <2 x i32> @p_splatvec_noundef(<2 x i32> %x, <2 x i32> %y, <2 x i32> noundef %m) {
+; CHECK-LABEL: @p_splatvec_noundef(
+; CHECK-NEXT: [[AND:%.*]] = and <2 x i32> [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT: [[NEG:%.*]] = xor <2 x i32> [[M]], splat (i32 -1)
+; CHECK-NEXT: [[AND1:%.*]] = and <2 x i32> [[Y:%.*]], [[NEG]]
; CHECK-NEXT: [[RET:%.*]] = or disjoint <2 x i32> [[AND]], [[AND1]]
; CHECK-NEXT: ret <2 x i32> [[RET]]
;
@@ -195,11 +225,26 @@ define <3 x i32> @p_constmask2_vec_undef(<3 x i32> %x, <3 x i32> %y) {
; Used to make sure that the IR complexity sorting does not interfere.
declare i32 @gen32()
-define i32 @p_commutative0(i32 %x, i32 %y, i32 noundef %m) {
+define i32 @p_commutative0(i32 %x, i32 %y, i32 %m) {
; CHECK-LABEL: @p_commutative0(
; CHECK-NEXT: [[AND:%.*]] = and i32 [[M:%.*]], [[X:%.*]]
; CHECK-NEXT: [[NEG:%.*]] = xor i32 [[M]], -1
; CHECK-NEXT: [[AND1:%.*]] = and i32 [[Y:%.*]], [[NEG]]
+; CHECK-NEXT: [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT: ret i32 [[RET]]
+;
+ %and = and i32 %m, %x ; swapped order
+ %neg = xor i32 %m, -1
+ %and1 = and i32 %neg, %y
+ %ret = add i32 %and, %and1
+ ret i32 %ret
+}
+
+define i32 @p_commutative0_noundef(i32 %x, i32 %y, i32 noundef %m) {
+; CHECK-LABEL: @p_commutative0_noundef(
+; CHECK-NEXT: [[AND:%.*]] = and i32 [[M:%.*]], [[X:%.*]]
+; CHECK-NEXT: [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT: [[AND1:%.*]] = and i32 [[Y:%.*]], [[NEG]]
; CHECK-NEXT: [[RET:%.*]] = or disjoint i32 [[AND]], [[AND1]]
; CHECK-NEXT: ret i32 [[RET]]
;
@@ -210,12 +255,29 @@ define i32 @p_commutative0(i32 %x, i32 %y, i32 noundef %m) {
ret i32 %ret
}
-define i32 @p_commutative1(i32 %x, i32 noundef %m) {
+define i32 @p_commutative1(i32 %x, i32 %m) {
; CHECK-LABEL: @p_commutative1(
; CHECK-NEXT: [[Y:%.*]] = call i32 @gen32()
; CHECK-NEXT: [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
; CHECK-NEXT: [[NEG:%.*]] = xor i32 [[M]], -1
; CHECK-NEXT: [[AND1:%.*]] = and i32 [[Y]], [[NEG]]
+; CHECK-NEXT: [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT: ret i32 [[RET]]
+;
+ %y = call i32 @gen32()
+ %and = and i32 %x, %m
+ %neg = xor i32 %m, -1
+ %and1 = and i32 %y, %neg; swapped order
+ %ret = add i32 %and, %and1
+ ret i32 %ret
+}
+
+define i32 @p_commutative1_noundef(i32 %x, i32 noundef %m) {
+; CHECK-LABEL: @p_commutative1_noundef(
+; CHECK-NEXT: [[Y:%.*]] = call i32 @gen32()
+; CHECK-NEXT: [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT: [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT: [[AND1:%.*]] = and i32 [[Y]], [[NEG]]
; CHECK-NEXT: [[RET:%.*]] = or disjoint i32 [[AND]], [[AND1]]
; CHECK-NEXT: ret i32 [[RET]]
;
@@ -227,11 +289,26 @@ define i32 @p_commutative1(i32 %x, i32 noundef %m) {
ret i32 %ret
}
-define i32 @p_commutative2(i32 %x, i32 %y, i32 noundef %m) {
+define i32 @p_commutative2(i32 %x, i32 %y, i32 %m) {
; CHECK-LABEL: @p_commutative2(
; CHECK-NEXT: [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
; CHECK-NEXT: [[NEG:%.*]] = xor i32 [[M]], -1
; CHECK-NEXT: [[AND1:%.*]] = and i32 [[Y:%.*]], [[NEG]]
+; CHECK-NEXT: [[RET:%.*]] = or i32 [[AND1]], [[AND]]
+; CHECK-NEXT: ret i32 [[RET]]
+;
+ %and = and i32 %x, %m
+ %neg = xor i32 %m, -1
+ %and1 = and i32 %neg, %y
+ %ret = add i32 %and1, %and ; swapped order
+ ret i32 %ret
+}
+
+define i32 @p_commutative2_noundef(i32 %x, i32 %y, i32 noundef %m) {
+; CHECK-LABEL: @p_commutative2_noundef(
+; CHECK-NEXT: [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT: [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT: [[AND1:%.*]] = and i32 [[Y:%.*]], [[NEG]]
; CHECK-NEXT: [[RET:%.*]] = or disjoint i32 [[AND1]], [[AND]]
; CHECK-NEXT: ret i32 [[RET]]
;
@@ -242,12 +319,29 @@ define i32 @p_commutative2(i32 %x, i32 %y, i32 noundef %m) {
ret i32 %ret
}
-define i32 @p_commutative3(i32 %x, i32 noundef %m) {
+define i32 @p_commutative3(i32 %x, i32 %m) {
; CHECK-LABEL: @p_commutative3(
; CHECK-NEXT: [[Y:%.*]] = call i32 @gen32()
; CHECK-NEXT: [[AND:%.*]] = and i32 [[M:%.*]], [[X:%.*]]
; CHECK-NEXT: [[NEG:%.*]] = xor i32 [[M]], -1
; CHECK-NEXT: [[AND1:%.*]] = and i32 [[Y]], [[NEG]]
+; CHECK-NEXT: [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT: ret i32 [[RET]]
+;
+ %y = call i32 @gen32()
+ %and = and i32 %m, %x ; swapped order
+ %neg = xor i32 %m, -1
+ %and1 = and i32 %y, %neg; swapped order
+ %ret = add i32 %and, %and1
+ ret i32 %ret
+}
+
+define i32 @p_commutative3_noundef(i32 %x, i32 noundef %m) {
+; CHECK-LABEL: @p_commutative3_noundef(
+; CHECK-NEXT: [[Y:%.*]] = call i32 @gen32()
+; CHECK-NEXT: [[AND:%.*]] = and i32 [[M:%.*]], [[X:%.*]]
+; CHECK-NEXT: [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT: [[AND1:%.*]] = and i32 [[Y]], [[NEG]]
; CHECK-NEXT: [[RET:%.*]] = or disjoint i32 [[AND]], [[AND1]]
; CHECK-NEXT: ret i32 [[RET]]
;
@@ -259,11 +353,26 @@ define i32 @p_commutative3(i32 %x, i32 noundef %m) {
ret i32 %ret
}
-define i32 @p_commutative4(i32 %x, i32 %y, i32 noundef %m) {
+define i32 @p_commutative4(i32 %x, i32 %y, i32 %m) {
; CHECK-LABEL: @p_commutative4(
; CHECK-NEXT: [[AND:%.*]] = and i32 [[M:%.*]], [[X:%.*]]
; CHECK-NEXT: [[NEG:%.*]] = xor i32 [[M]], -1
; CHECK-NEXT: [[AND1:%.*]] = and i32 [[Y:%.*]], [[NEG]]
+; CHECK-NEXT: [[RET:%.*]] = or i32 [[AND1]], [[AND]]
+; CHECK-NEXT: ret i32 [[RET]]
+;
+ %and = and i32 %m, %x ; swapped order
+ %neg = xor i32 %m, -1
+ %and1 = and i32 %neg, %y
+ %ret = add i32 %and1, %and ; swapped order
+ ret i32 %ret
+}
+
+define i32 @p_commutative4_noundef(i32 %x, i32 %y, i32 noundef %m) {
+; CHECK-LABEL: @p_commutative4_noundef(
+; CHECK-NEXT: [[AND:%.*]] = and i32 [[M:%.*]], [[X:%.*]]
+; CHECK-NEXT: [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT: [[AND1:%.*]] = and i32 [[Y:%.*]], [[NEG]]
; CHECK-NEXT: [[RET:%.*]] = or disjoint i32 [[AND1]], [[AND]]
; CHECK-NEXT: ret i32 [[RET]]
;
@@ -274,12 +383,29 @@ define i32 @p_commutative4(i32 %x, i32 %y, i32 noundef %m) {
ret i32 %ret
}
-define i32 @p_commutative5(i32 %x, i32 noundef %m) {
+define i32 @p_commutative5(i32 %x, i32 %m) {
; CHECK-LABEL: @p_commutative5(
; CHECK-NEXT: [[Y:%.*]] = call i32 @gen32()
; CHECK-NEXT: [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
; CHECK-NEXT: [[NEG:%.*]] = xor i32 [[M]], -1
; CHECK-NEXT: [[AND1:%.*]] = and i32 [[Y]], [[NEG]]
+; CHECK-NEXT: [[RET:%.*]] = or i32 [[AND1]], [[AND]]
+; CHECK-NEXT: ret i32 [[RET]]
+;
+ %y = call i32 @gen32()
+ %and = and i32 %x, %m
+ %neg = xor i32 %m, -1
+ %and1 = and i32 %y, %neg; swapped order
+ %ret = add i32 %and1, %and ; swapped order
+ ret i32 %ret
+}
+
+define i32 @p_commutative5_noundef(i32 %x, i32 noundef %m) {
+; CHECK-LABEL: @p_commutative5_noundef(
+; CHECK-NEXT: [[Y:%.*]] = call i32 @gen32()
+; CHECK-NEXT: [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT: [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT: [[AND1:%.*]] = and i32 [[Y]], [[NEG]]
; CHECK-NEXT: [[RET:%.*]] = or disjoint i32 [[AND1]], [[AND]]
; CHECK-NEXT: ret i32 [[RET]]
;
@@ -291,12 +417,29 @@ define i32 @p_commutative5(i32 %x, i32 noundef %m) {
ret i32 %ret
}
-define i32 @p_commutative6(i32 %x, i32 noundef %m) {
+define i32 @p_commutative6(i32 %x, i32 %m) {
; CHECK-LABEL: @p_commutative6(
; CHECK-NEXT: [[Y:%.*]] = call i32 @gen32()
; CHECK-NEXT: [[AND:%.*]] = and i32 [[M:%.*]], [[X:%.*]]
; CHECK-NEXT: [[NEG:%.*]] = xor i32 [[M]], -1
; CHECK-NEXT: [[AND1:%.*]] = and i32 [[Y]], [[NEG]]
+; CHECK-NEXT: [[RET:%.*]] = or i32 [[AND1]], [[AND]]
+; CHECK-NEXT: ret i32 [[RET]]
+;
+ %y = call i32 @gen32()
+ %and = and i32 %m, %x ; swapped order
+ %neg = xor i32 %m, -1
+ %and1 = and i32 %y, %neg; swapped order
+ %ret = add i32 %and1, %and ; swapped order
+ ret i32 %ret
+}
+
+define i32 @p_commutative6_noundef(i32 %x, i32 noundef %m) {
+; CHECK-LABEL: @p_commutative6_noundef(
+; CHECK-NEXT: [[Y:%.*]] = call i32 @gen32()
+; CHECK-NEXT: [[AND:%.*]] = and i32 [[M:%.*]], [[X:%.*]]
+; CHECK-NEXT: [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT: [[AND1:%.*]] = and i32 [[Y]], [[NEG]]
; CHECK-NEXT: [[RET:%.*]] = or disjoint i32 [[AND1]], [[AND]]
; CHECK-NEXT: ret i32 [[RET]]
;
@@ -329,11 +472,32 @@ define i32 @p_constmask_commutative(i32 %x, i32 %y) {
declare void @use32(i32)
-define i32 @n0_oneuse(i32 %x, i32 %y, i32 noundef %m) {
+define i32 @n0_oneuse(i32 %x, i32 %y, i32 %m) {
; CHECK-LABEL: @n0_oneuse(
; CHECK-NEXT: [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
; CHECK-NEXT: [[NEG:%.*]] = xor i32 [[M]], -1
; CHECK-NEXT: [[AND1:%.*]] = and i32 [[Y:%.*]], [[NEG]]
+; CHECK-NEXT: [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT: call void @use32(i32 [[AND]])
+; CHECK-NEXT: call void @use32(i32 [[NEG]])
+; CHECK-NEXT: call void @use32(i32 [[AND1]])
+; CHECK-NEXT: ret i32 [[RET]]
+;
+ %and = and i32 %x, %m
+ %neg = xor i32 %m, -1
+ %and1 = and i32 %neg, %y
+ %ret = add i32 %and, %and1
+ call void @use32(i32 %and)
+ call void @use32(i32 %neg)
+ call void @use32(i32 %and1)
+ ret i32 %ret
+}
+
+define i32 @n0_oneuse_noundef(i32 %x, i32 %y, i32 noundef %m) {
+; CHECK-LABEL: @n0_oneuse_noundef(
+; CHECK-NEXT: [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT: [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT: [[AND1:%.*]] = and i32 [[Y:%.*]], [[NEG]]
; CHECK-NEXT: [[RET:%.*]] = or disjoint i32 [[AND]], [[AND1]]
; CHECK-NEXT: call void @use32(i32 [[AND]])
; CHECK-NEXT: call void @use32(i32 [[NEG]])
diff --git a/llvm/test/Transforms/InstCombine/pr53357.ll b/llvm/test/Transforms/InstCombine/pr53357.ll
index 2292abd0c9068..d6a2d7e8adf99 100644
--- a/llvm/test/Transforms/InstCombine/pr53357.ll
+++ b/llvm/test/Transforms/InstCombine/pr53357.ll
@@ -3,7 +3,7 @@
; RUN: opt < %s -passes=instcombine -S | FileCheck %s
; (x & y) + ~(x | y)
-define i32 @src(i32 noundef %0, i32 noundef %1) {
+define i32 @src(i32 %0, i32 %1) {
; CHECK-LABEL: @src(
; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP1:%.*]], [[TMP0:%.*]]
; CHECK-NEXT: [[TMP4:%.*]] = xor i32 [[TMP3]], -1
@@ -16,8 +16,22 @@ define i32 @src(i32 noundef %0, i32 noundef %1) {
ret i32 %6
}
+; (x & y) + ~(x | y) where x and y are noundef
+define i32 @src_noundef(i32 noundef %0, i32 noundef %1) {
+; CHECK-LABEL: @src_noundef(
+; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP1:%.*]], [[TMP0:%.*]]
+; CHECK-NEXT: [[TMP4:%.*]] = xor i32 [[TMP3]], -1
+; CHECK-NEXT: ret i32 [[TMP4]]
+;
+ %3 = and i32 %1, %0
+ %4 = or i32 %1, %0
+ %5 = xor i32 %4, -1
+ %6 = add i32 %3, %5
+ ret i32 %6
+}
+
; vector version of src
-define <2 x i32> @src_vec(<2 x i32> noundef %0, <2 x i32> noundef %1) {
+define <2 x i32> @src_vec(<2 x i32> %0, <2 x i32> %1) {
; CHECK-LABEL: @src_vec(
; CHECK-NEXT: [[TMP3:%.*]] = xor <2 x i32> [[TMP1:%.*]], [[TMP0:%.*]]
; CHECK-NEXT: [[TMP4:%.*]] = xor <2 x i32> [[TMP3]], splat (i32 -1)
@@ -30,8 +44,22 @@ define <2 x i32> @src_vec(<2 x i32> noundef %0, <2 x i32> noundef %1) {
ret <2 x i32> %6
}
+; vector version of src_noundef
+define <2 x i32> @src_vec_noundef(<2 x i32> noundef %0, <2 x i32> noundef %1) {
+; CHECK-LABEL: @src_vec_noundef(
+; CHECK-NEXT: [[TMP3:%.*]] = xor <2 x i32> [[TMP1:%.*]], [[TMP0:%.*]]
+; CHECK-NEXT: [[TMP4:%.*]] = xor <2 x i32> [[TMP3]], splat (i32 -1)
+; CHECK-NEXT: ret <2 x i32> [[TMP4]]
+;
+ %3 = and <2 x i32> %1, %0
+ %4 = or <2 x i32> %1, %0
+ %5 = xor <2 x i32> %4, <i32 -1, i32 -1>
+ %6 = add <2 x i32> %3, %5
+ ret <2 x i32> %6
+}
+
; vector version of src with poison values
-define <2 x i32> @src_vec_poison(<2 x i32> noundef %0, <2 x i32> noundef %1) {
+define <2 x i32> @src_vec_poison(<2 x i32> %0, <2 x i32> %1) {
; CHECK-LABEL: @src_vec_poison(
; CHECK-NEXT: [[TMP3:%.*]] = xor <2 x i32> [[TMP1:%.*]], [[TMP0:%.*]]
; CHECK-NEXT: [[TMP4:%.*]] = xor <2 x i32> [[TMP3]], splat (i32 -1)
@@ -44,8 +72,22 @@ define <2 x i32> @src_vec_poison(<2 x i32> noundef %0, <2 x i32> noundef %1) {
ret <2 x i32> %6
}
+; vector version of src_noundef with poison values
+define <2 x i32> @src_vec_poison_noundef(<2 x i32> noundef %0, <2 x i32> noundef %1) {
+; CHECK-LABEL: @src_vec_poison_noundef(
+; CHECK-NEXT: [[TMP3:%.*]] = xor <2 x i32> [[TMP1:%.*]], [[TMP0:%.*]]
+; CHECK-NEXT: [[TMP4:%.*]] = xor <2 x i32> [[TMP3]], splat (i32 -1)
+; CHECK-NEXT: ret <2 x i32> [[TMP4]]
+;
+ %3 = and <2 x i32> %1, %0
+ %4 = or <2 x i32> %1, %0
+ %5 = xor <2 x i32> %4, <i32 -1, i32 poison>
+ %6 = add <2 x i32> %3, %5
+ ret <2 x i32> %6
+}
+
; (x & y) + ~(y | x)
-define i32 @src2(i32 noundef %0, i32 noundef %1) {
+define i32 @src2(i32 %0, i32 %1) {
; CHECK-LABEL: @src2(
; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP1:%.*]], [[TMP0:%.*]]
; CHECK-NEXT: [[TMP4:%.*]] = xor i32 [[TMP3]], -1
@@ -58,8 +100,22 @@ define i32 @src2(i32 noundef %0, i32 noundef %1) {
ret i32 %6
}
+; (x & y) + ~(y | x) where x and y are noundef
+define i32 @src2_noundef(i32 noundef %0, i32 noundef %1) {
+; CHECK-LABEL: @src2_noundef(
+; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP1:%.*]], [[TMP0:%.*]]
+; CHECK-NEXT: [[TMP4:%.*]] = xor i32 [[TMP3]], -1
+; CHECK-NEXT: ret i32 [[TMP4]]
+;
+ %3 = and i32 %1, %0
+ %4 = or i32 %0, %1
+ %5 = xor i32 %4, -1
+ %6 = add i32 %3, %5
+ ret i32 %6
+}
+
; (x & y) + (~x & ~y)
-define i32 @src3(i32 noundef %0, i32 noundef %1) {
+define i32 @src3(i32 %0, i32 %1) {
; CHECK-LABEL: @src3(
; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP1:%.*]], [[TMP0:%.*]]
; CHECK-NEXT: [[TMP4:%.*]] = xor i32 [[TMP3]], -1
@@ -73,8 +129,23 @@ define i32 @src3(i32 noundef %0, i32 noundef %1) {
ret i32 %7
}
+; (x & y) + (~x & ~y) where x and y are noundef
+define i32 @src3_noundef(i32 noundef %0, i32 noundef %1) {
+; CHECK-LABEL: @src3_noundef(
+; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP1:%.*]], [[TMP0:%.*]]
+; CHECK-NEXT: [[TMP4:%.*]] = xor i32 [[TMP3]], -1
+; CHECK-NEXT: ret i32 [[TMP4]]
+;
+ %3 = and i32 %1, %0
+ %4 = xor i32 %0, -1
+ %5 = xor i32 %1, -1
+ %6 = and i32 %4, %5
+ %7 = add i32 %3, %6
+ ret i32 %7
+}
+
; ~(x | y) + (y & x)
-define i32 @src4(i32 noundef %0, i32 noundef %1) {
+define i32 @src4(i32 %0, i32 %1) {
; CHECK-LABEL: @src4(
; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP0:%.*]], [[TMP1:%.*]]
; CHECK-NEXT: [[TMP4:%.*]] = xor i32 [[TMP3]], -1
@@ -87,8 +158,22 @@ define i32 @src4(i32 noundef %0, i32 noundef %1) {
ret i32 %6
}
+; ~(x | y) + (y & x) where x and y are noundef
+define i32 @src4_noundef(i32 noundef %0, i32 noundef %1) {
+; CHECK-LABEL: @src4_noundef(
+; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP0:%.*]], [[TMP1:%.*]]
+; CHECK-NEXT: [[TMP4:%.*]] = xor i32 [[TMP3]], -1
+; CHECK-NEXT: ret i32 [[TMP4]]
+;
+ %3 = and i32 %0, %1
+ %4 = or i32 %1, %0
+ %5 = xor i32 %4, -1
+ %6 = add i32 %3, %5
+ ret i32 %6
+}
+
; ~(x | y) + (x & y)
-define i32 @src5(i32 noundef %0, i32 noundef %1) {
+define i32 @src5(i32 %0, i32 %1) {
; CHECK-LABEL: @src5(
; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP1:%.*]], [[TMP0:%.*]]
; CHECK-NEXT: [[TMP4:%.*]] = xor i32 [[TMP3]], -1
@@ -101,6 +186,20 @@ define i32 @src5(i32 noundef %0, i32 noundef %1) {
ret i32 %6
}
+; ~(x | y) + (x & y) where x and y are noundef
+define i32 @src5_noundef(i32 noundef %0, i32 noundef %1) {
+; CHECK-LABEL: @src5_noundef(
+; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP1:%.*]], [[TMP0:%.*]]
+; CHECK-NEXT: [[TMP4:%.*]] = xor i32 [[TMP3]], -1
+; CHECK-NEXT: ret i32 [[TMP4]]
+;
+ %3 = or i32 %1, %0
+ %4 = xor i32 %3, -1
+ %5 = and i32 %1, %0
+ %6 = add i32 %4, %5
+ ret i32 %6
+}
+
; (a & b) + ~(c | d)
define i32 @src6(i32 %0, i32 %1, i32 %2, i32 %3) {
; CHECK-LABEL: @src6(
diff --git a/llvm/unittests/Analysis/ValueTrackingTest.cpp b/llvm/unittests/Analysis/ValueTrackingTest.cpp
index f9e6bce1a24f7..d40022d640290 100644
--- a/llvm/unittests/Analysis/ValueTrackingTest.cpp
+++ b/llvm/unittests/Analysis/ValueTrackingTest.cpp
@@ -3114,6 +3114,29 @@ TEST_F(ValueTrackingTest, HaveNoCommonBitsSet) {
{
// Check for an inverted mask: (X & ~M) op (Y & M).
auto M = parseModule(R"(
+ define i32 @test(i32 %X, i32 %Y, i32 %M) {
+ %1 = xor i32 %M, -1
+ %LHS = and i32 %1, %X
+ %RHS = and i32 %Y, %M
+ %Ret = add i32 %LHS, %RHS
+ ret i32 %Ret
+ })");
+
+ auto *F = M->getFunction("test");
+ auto *LHS = findInstructionByNameOrNull(F, "LHS");
+ auto *RHS = findInstructionByNameOrNull(F, "RHS");
+
+ const DataLayout &DL = M->getDataLayout();
+ EXPECT_FALSE(haveNoCommonBitsSet(LHS, RHS, DL));
+ EXPECT_EQ(NoCommonBitsSetResult::OnlyIfUndefIgnored,
+ getNoCommonBitsSetResult(LHS, RHS, DL));
+ EXPECT_FALSE(haveNoCommonBitsSet(RHS, LHS, DL));
+ EXPECT_EQ(NoCommonBitsSetResult::OnlyIfUndefIgnored,
+ getNoCommonBitsSetResult(RHS, LHS, DL));
+ }
+ {
+ // Check for an inverted mask: (X & ~M) op (Y & M) where M is noundef.
+ auto M = parseModule(R"(
define i32 @test(i32 %X, i32 %Y, i32 noundef %M) {
%1 = xor i32 %M, -1
%LHS = and i32 %1, %X
@@ -3128,11 +3151,51 @@ TEST_F(ValueTrackingTest, HaveNoCommonBitsSet) {
const DataLayout &DL = M->getDataLayout();
EXPECT_TRUE(haveNoCommonBitsSet(LHS, RHS, DL));
+ EXPECT_EQ(NoCommonBitsSetResult::Known,
+ getNoCommonBitsSetResult(LHS, RHS, DL));
EXPECT_TRUE(haveNoCommonBitsSet(RHS, LHS, DL));
+ EXPECT_EQ(NoCommonBitsSetResult::Known,
+ getNoCommonBitsSetResult(RHS, LHS, DL));
}
{
// Check for (A & B) and ~(A | B)
auto M = parseModule(R"(
+ define void @test(i32 %A, i32 %B) {
+ %LHS = and i32 %A, %B
+ %or = or i32 %A, %B
+ %RHS = xor i32 %or, -1
+
+ %LHS2 = and i32 %B, %A
+ %or2 = or i32 %A, %B
+ %RHS2 = xor i32 %or2, -1
+
+ ret void
+ })");
+
+ auto *F = M->getFunction("test");
+ const DataLayout &DL = M->getDataLayout();
+
+ auto *LHS = findInstructionByNameOrNull(F, "LHS");
+ auto *RHS = findInstructionByNameOrNull(F, "RHS");
+ EXPECT_FALSE(haveNoCommonBitsSet(LHS, RHS, DL));
+ EXPECT_EQ(NoCommonBitsSetResult::OnlyIfUndefIgnored,
+ getNoCommonBitsSetResult(LHS, RHS, DL));
+ EXPECT_FALSE(haveNoCommonBitsSet(RHS, LHS, DL));
+ EXPECT_EQ(NoCommonBitsSetResult::OnlyIfUndefIgnored,
+ getNoCommonBitsSetResult(RHS, LHS, DL));
+
+ auto *LHS2 = findInstructionByNameOrNull(F, "LHS2");
+ auto *RHS2 = findInstructionByNameOrNull(F, "RHS2");
+ EXPECT_FALSE(haveNoCommonBitsSet(LHS2, RHS2, DL));
+ EXPECT_EQ(NoCommonBitsSetResult::OnlyIfUndefIgnored,
+ getNoCommonBitsSetResult(LHS2, RHS2, DL));
+ EXPECT_FALSE(haveNoCommonBitsSet(RHS2, LHS2, DL));
+ EXPECT_EQ(NoCommonBitsSetResult::OnlyIfUndefIgnored,
+ getNoCommonBitsSetResult(RHS2, LHS2, DL));
+ }
+ {
+ // Check for (A & B) and ~(A | B) where A and B are noundef
+ auto M = parseModule(R"(
define void @test(i32 noundef %A, i32 noundef %B) {
%LHS = and i32 %A, %B
%or = or i32 %A, %B
@@ -3151,16 +3214,61 @@ TEST_F(ValueTrackingTest, HaveNoCommonBitsSet) {
auto *LHS = findInstructionByNameOrNull(F, "LHS");
auto *RHS = findInstructionByNameOrNull(F, "RHS");
EXPECT_TRUE(haveNoCommonBitsSet(LHS, RHS, DL));
+ EXPECT_EQ(NoCommonBitsSetResult::Known,
+ getNoCommonBitsSetResult(LHS, RHS, DL));
EXPECT_TRUE(haveNoCommonBitsSet(RHS, LHS, DL));
+ EXPECT_EQ(NoCommonBitsSetResult::Known,
+ getNoCommonBitsSetResult(RHS, LHS, DL));
auto *LHS2 = findInstructionByNameOrNull(F, "LHS2");
auto *RHS2 = findInstructionByNameOrNull(F, "RHS2");
EXPECT_TRUE(haveNoCommonBitsSet(LHS2, RHS2, DL));
+ EXPECT_EQ(NoCommonBitsSetResult::Known,
+ getNoCommonBitsSetResult(LHS2, RHS2, DL));
EXPECT_TRUE(haveNoCommonBitsSet(RHS2, LHS2, DL));
+ EXPECT_EQ(NoCommonBitsSetResult::Known,
+ getNoCommonBitsSetResult(RHS2, LHS2, DL));
}
{
// Check for (A & B) and ~(A | B) in vector version
auto M = parseModule(R"(
+ define void @test(<2 x i32> %A, <2 x i32> %B) {
+ %LHS = and <2 x i32> %A, %B
+ %or = or <2 x i32> %A, %B
+ %RHS = xor <2 x i32> %or, <i32 -1, i32 -1>
+
+ %LHS2 = and <2 x i32> %B, %A
+ %or2 = or <2 x i32> %A, %B
+ %RHS2 = xor <2 x i32> %or2, <i32 -1, i32 -1>
+
+ ret void
+ })");
+
+ auto *F = M->getFunction("test");
+ const DataLayout &DL = M->getDataLayout();
+
+ auto *LHS = findInstructionByNameOrNull(F, "LHS");
+ auto *RHS = findInstructionByNameOrNull(F, "RHS");
+ EXPECT_FALSE(haveNoCommonBitsSet(LHS, RHS, DL));
+ EXPECT_EQ(NoCommonBitsSetResult::OnlyIfUndefIgnored,
+ getNoCommonBitsSetResult(LHS, RHS, DL));
+ EXPECT_FALSE(haveNoCommonBitsSet(RHS, LHS, DL));
+ EXPECT_EQ(NoCommonBitsSetResult::OnlyIfUndefIgnored,
+ getNoCommonBitsSetResult(RHS, LHS, DL));
+
+ auto *LHS2 = findInstructionByNameOrNull(F, "LHS2");
+ auto *RHS2 = findInstructionByNameOrNull(F, "RHS2");
+ EXPECT_FALSE(haveNoCommonBitsSet(LHS2, RHS2, DL));
+ EXPECT_EQ(NoCommonBitsSetResult::OnlyIfUndefIgnored,
+ getNoCommonBitsSetResult(LHS2, RHS2, DL));
+ EXPECT_FALSE(haveNoCommonBitsSet(RHS2, LHS2, DL));
+ EXPECT_EQ(NoCommonBitsSetResult::OnlyIfUndefIgnored,
+ getNoCommonBitsSetResult(RHS2, LHS2, DL));
+ }
+ {
+ // Check for (A & B) and ~(A | B) in vector version where A and B are
+ // noundef
+ auto M = parseModule(R"(
define void @test(<2 x i32> noundef %A, <2 x i32> noundef %B) {
%LHS = and <2 x i32> %A, %B
%or = or <2 x i32> %A, %B
@@ -3179,12 +3287,20 @@ TEST_F(ValueTrackingTest, HaveNoCommonBitsSet) {
auto *LHS = findInstructionByNameOrNull(F, "LHS");
auto *RHS = findInstructionByNameOrNull(F, "RHS");
EXPECT_TRUE(haveNoCommonBitsSet(LHS, RHS, DL));
+ EXPECT_EQ(NoCommonBitsSetResult::Known,
+ getNoCommonBitsSetResult(LHS, RHS, DL));
EXPECT_TRUE(haveNoCommonBitsSet(RHS, LHS, DL));
+ EXPECT_EQ(NoCommonBitsSetResult::Known,
+ getNoCommonBitsSetResult(RHS, LHS, DL));
auto *LHS2 = findInstructionByNameOrNull(F, "LHS2");
auto *RHS2 = findInstructionByNameOrNull(F, "RHS2");
EXPECT_TRUE(haveNoCommonBitsSet(LHS2, RHS2, DL));
+ EXPECT_EQ(NoCommonBitsSetResult::Known,
+ getNoCommonBitsSetResult(LHS2, RHS2, DL));
EXPECT_TRUE(haveNoCommonBitsSet(RHS2, LHS2, DL));
+ EXPECT_EQ(NoCommonBitsSetResult::Known,
+ getNoCommonBitsSetResult(RHS2, LHS2, DL));
}
}
>From 5e5490f2df9c9c329689929b62d51ee0fb95fb30 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Mon, 6 Jul 2026 19:39:37 +0100
Subject: [PATCH 43/46] [X86] combineToHorizontalAddSub - use PostShuffleMask
to determine undemanded subvectors (#207789)
For cases where the middle-end has created a 256-bit HADD/SUB pattern
with unused upper 128-bit subvectors, use the PostShuffleMask undefined
ranges to remove any unused 128-bit HADD/SUB ops.
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 47 +++++++++++++++----------
llvm/test/CodeGen/X86/haddsub-undef.ll | 6 ++--
llvm/test/CodeGen/X86/phaddsub-undef.ll | 47 +++++--------------------
3 files changed, 39 insertions(+), 61 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index d3c6363930806..9209134a055c6 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -55077,12 +55077,9 @@ static bool isHorizontalBinOp(unsigned HOpcode, SDValue &LHS, SDValue &RHS,
SDValue NewLHS = A.getNode() ? A : B; // If A is 'UNDEF', use B for it.
SDValue NewRHS = B.getNode() ? B : A; // If B is 'UNDEF', use A for it.
+ // Avoid 128-bit multi lane shuffles if pre-AVX2 and FP (integer will split).
bool IsIdentityPostShuffle =
isSequentialOrUndefInRange(PostShuffleMask, 0, NumElts, 0);
- if (IsIdentityPostShuffle)
- PostShuffleMask.clear();
-
- // Avoid 128-bit multi lane shuffles if pre-AVX2 and FP (integer will split).
if (!IsIdentityPostShuffle && !Subtarget.hasAVX2() && VT.isFloatingPoint() &&
isMultiLaneShuffleMask(128, VT.getScalarSizeInBits(), PostShuffleMask))
return false;
@@ -55126,6 +55123,21 @@ static SDValue combineToHorizontalAddSub(SDNode *N, SelectionDAG &DAG,
N->user_begin()->getOperand(1).getOpcode() == HorizOpcode);
};
+ auto CanonicalizeHorizOps = [&](const SDLoc &DL, SDValue &LHS, SDValue &RHS) {
+ assert(PostShuffleMask.size() == VT.getVectorNumElements() &&
+ "Illegal shuffle mask");
+ LHS = DAG.getBitcast(VT, LHS);
+ RHS = DAG.getBitcast(VT, RHS);
+ if (VT.is256BitVector() && isUndefUpperHalf(PostShuffleMask) &&
+ !isLaneCrossingShuffleMask(128, VT.getScalarSizeInBits(),
+ PostShuffleMask)) {
+ LHS = extract128BitVector(LHS, 0, DAG, DL);
+ RHS = extract128BitVector(RHS, 0, DAG, DL);
+ PostShuffleMask.truncate(PostShuffleMask.size() / 2);
+ }
+ return LHS.getSimpleValueType();
+ };
+
switch (Opcode) {
case ISD::FADD:
case ISD::FSUB:
@@ -55136,13 +55148,12 @@ static SDValue combineToHorizontalAddSub(SDNode *N, SelectionDAG &DAG,
auto HorizOpcode = IsAdd ? X86ISD::FHADD : X86ISD::FHSUB;
if (isHorizontalBinOp(HorizOpcode, LHS, RHS, DAG, Subtarget, IsAdd,
PostShuffleMask, MergableHorizOp(HorizOpcode))) {
- SDValue HorizBinOp =
- DAG.getNode(HorizOpcode, SDLoc(N), VT, DAG.getBitcast(VT, LHS),
- DAG.getBitcast(VT, RHS));
- if (!PostShuffleMask.empty())
- HorizBinOp = DAG.getVectorShuffle(VT, SDLoc(HorizBinOp), HorizBinOp,
- DAG.getUNDEF(VT), PostShuffleMask);
- return HorizBinOp;
+ SDLoc DL(N);
+ MVT HorizVT = CanonicalizeHorizOps(DL, LHS, RHS);
+ SDValue HorizBinOp = DAG.getNode(HorizOpcode, DL, HorizVT, LHS, RHS);
+ HorizBinOp = DAG.getVectorShuffle(HorizVT, DL, HorizBinOp, HorizBinOp,
+ PostShuffleMask);
+ return DAG.getInsertSubvector(DL, DAG.getUNDEF(VT), HorizBinOp, 0);
}
}
break;
@@ -55164,13 +55175,13 @@ static SDValue combineToHorizontalAddSub(SDNode *N, SelectionDAG &DAG,
ArrayRef<SDValue> Ops) {
return DAG.getNode(HorizOpcode, DL, Ops[0].getValueType(), Ops);
};
- SDValue HorizBinOp = SplitOpsAndApply(
- DAG, Subtarget, SDLoc(N), VT,
- {DAG.getBitcast(VT, LHS), DAG.getBitcast(VT, RHS)}, HOpBuilder);
- if (!PostShuffleMask.empty())
- HorizBinOp = DAG.getVectorShuffle(VT, SDLoc(HorizBinOp), HorizBinOp,
- DAG.getUNDEF(VT), PostShuffleMask);
- return HorizBinOp;
+ SDLoc DL(N);
+ MVT HorizVT = CanonicalizeHorizOps(DL, LHS, RHS);
+ SDValue HorizBinOp = SplitOpsAndApply(DAG, Subtarget, DL, HorizVT,
+ {LHS, RHS}, HOpBuilder);
+ HorizBinOp = DAG.getVectorShuffle(HorizVT, DL, HorizBinOp, HorizBinOp,
+ PostShuffleMask);
+ return DAG.getInsertSubvector(DL, DAG.getUNDEF(VT), HorizBinOp, 0);
}
}
break;
diff --git a/llvm/test/CodeGen/X86/haddsub-undef.ll b/llvm/test/CodeGen/X86/haddsub-undef.ll
index 2b5325b7ec180..657556fcea688 100644
--- a/llvm/test/CodeGen/X86/haddsub-undef.ll
+++ b/llvm/test/CodeGen/X86/haddsub-undef.ll
@@ -193,7 +193,6 @@ define <4 x float> @add_v4f32_0uu3(<4 x float> %a, <4 x float> %b) {
ret <4 x float> %r
}
-; FIXME: Use lower xmm only
define <8 x float> @add_v8f32_0uu3uuuu(<8 x float> %a, <8 x float> %b) {
; SSE-LABEL: add_v8f32_0uu3uuuu:
; SSE: # %bb.0:
@@ -202,7 +201,7 @@ define <8 x float> @add_v8f32_0uu3uuuu(<8 x float> %a, <8 x float> %b) {
;
; AVX-LABEL: add_v8f32_0uu3uuuu:
; AVX: # %bb.0:
-; AVX-NEXT: vhaddps %ymm1, %ymm0, %ymm0
+; AVX-NEXT: vhaddps %xmm1, %xmm0, %xmm0
; AVX-NEXT: retq
%x = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 0, i32 poison, i32 poison, i32 10, i32 poison, i32 poison, i32 poison, i32 poison>
%y = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 1, i32 poison, i32 poison, i32 11, i32 poison, i32 poison, i32 poison, i32 poison>
@@ -236,7 +235,6 @@ define <8 x float> @add_v8f32_0uuuuu6u(<8 x float> %a, <8 x float> %b) {
ret <8 x float> %r
}
-; FIXME: Use lower xmm only
define <8 x float> @add_v8f32_01uuuuuu(<8 x float> %a, <8 x float> %b) {
; SSE-LABEL: add_v8f32_01uuuuuu:
; SSE: # %bb.0:
@@ -245,7 +243,7 @@ define <8 x float> @add_v8f32_01uuuuuu(<8 x float> %a, <8 x float> %b) {
;
; AVX-LABEL: add_v8f32_01uuuuuu:
; AVX: # %bb.0:
-; AVX-NEXT: vhaddps %ymm0, %ymm0, %ymm0
+; AVX-NEXT: vhaddps %xmm0, %xmm0, %xmm0
; AVX-NEXT: retq
%x = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 0, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
%y = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 1, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
diff --git a/llvm/test/CodeGen/X86/phaddsub-undef.ll b/llvm/test/CodeGen/X86/phaddsub-undef.ll
index f7d1fc3688472..fabf9e3a45062 100644
--- a/llvm/test/CodeGen/X86/phaddsub-undef.ll
+++ b/llvm/test/CodeGen/X86/phaddsub-undef.ll
@@ -10,32 +10,16 @@
; Verify that we correctly fold horizontal binop even in the presence of UNDEF/POISON.
-; FIXME: Use lower xmm only
define <8 x i32> @add_v8i32_0uu3uuuu(<8 x i32> %a, <8 x i32> %b) {
; SSE-LABEL: add_v8i32_0uu3uuuu:
; SSE: # %bb.0:
; SSE-NEXT: phaddd %xmm2, %xmm0
-; SSE-NEXT: phaddd %xmm3, %xmm1
; SSE-NEXT: retq
;
-; AVX1-LABEL: add_v8i32_0uu3uuuu:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT: vphaddd %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vphaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT: retq
-;
-; AVX2-LABEL: add_v8i32_0uu3uuuu:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vphaddd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: retq
-;
-; AVX512-LABEL: add_v8i32_0uu3uuuu:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vphaddd %ymm1, %ymm0, %ymm0
-; AVX512-NEXT: retq
+; AVX-LABEL: add_v8i32_0uu3uuuu:
+; AVX: # %bb.0:
+; AVX-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX-NEXT: retq
%x = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 0, i32 poison, i32 poison, i32 10, i32 poison, i32 poison, i32 poison, i32 poison>
%y = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 1, i32 poison, i32 poison, i32 11, i32 poison, i32 poison, i32 poison, i32 poison>
%h = add <8 x i32> %x, %y
@@ -107,31 +91,16 @@ define <8 x i32> @add_v8i32_uuuu4uu7(<8 x i32> %a, <8 x i32> %b) {
ret <8 x i32> %h
}
-; FIXME: Use lower xmm only
define <8 x i32> @add_v8i32_01uuuuuu(<8 x i32> %a, <8 x i32> %b) {
; SSE-LABEL: add_v8i32_01uuuuuu:
; SSE: # %bb.0:
; SSE-NEXT: phaddd %xmm0, %xmm0
-; SSE-NEXT: phaddd %xmm1, %xmm1
; SSE-NEXT: retq
;
-; AVX1-LABEL: add_v8i32_01uuuuuu:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vphaddd %xmm0, %xmm0, %xmm1
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX1-NEXT: retq
-;
-; AVX2-LABEL: add_v8i32_01uuuuuu:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vphaddd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT: retq
-;
-; AVX512-LABEL: add_v8i32_01uuuuuu:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vphaddd %ymm0, %ymm0, %ymm0
-; AVX512-NEXT: retq
+; AVX-LABEL: add_v8i32_01uuuuuu:
+; AVX: # %bb.0:
+; AVX-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX-NEXT: retq
%x = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 1, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
%y = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 0, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
%h = add <8 x i32> %x, %y
>From 3df6ca69c9e2b22687315a3074857383eefa93b8 Mon Sep 17 00:00:00 2001
From: Quentin Colombet <quentin.colombet at gmail.com>
Date: Mon, 6 Jul 2026 20:40:53 +0200
Subject: [PATCH 44/46] [AMDGPU][test] Use mir test for regalloc issue
(#197363)
Use the newly introduced split-from flag to produce a more robust test
case for the hoistSpillInsideBB live-range update issue.
NFC
---
.../regalloc-hoist-spill-live-range-upd.ll | 2870 -----------------
.../regalloc-hoist-spill-live-range-upd.mir | 71 +
2 files changed, 71 insertions(+), 2870 deletions(-)
delete mode 100644 llvm/test/CodeGen/AMDGPU/regalloc-hoist-spill-live-range-upd.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/regalloc-hoist-spill-live-range-upd.mir
diff --git a/llvm/test/CodeGen/AMDGPU/regalloc-hoist-spill-live-range-upd.ll b/llvm/test/CodeGen/AMDGPU/regalloc-hoist-spill-live-range-upd.ll
deleted file mode 100644
index a8d04981e08c6..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/regalloc-hoist-spill-live-range-upd.ll
+++ /dev/null
@@ -1,2870 +0,0 @@
-; RUN: llc -mcpu=gfx950 -o - -verify-regalloc %s | FileCheck %s
-;
-; This test is extremely fragile and chances are it won't test what it
-; was meant to pretty quickly.
-;
-; The gist of it is we want something that triggers the hoistSpillInsideBB
-; spilling optimization placement, while spilling a value that is defined
-; by a copy inserted by live-range splitting within the prologue of the basic
-; block (i.e., where we do our exec masks shenanigans).
-; Since the spiller cannot insert the spill code within that prologue, it
-; needs to extend the live-range through the prologue region and that test
-; is supposed to capture that.
-; The verify-regalloc flag makes sure the live-ranges are properly updated.
-; Before it would complain about missing live segment.
-
-; CHECK-LABEL: foo:
-
-
-target triple = "amdgcn-amd-amdhsa"
-
- at global_smem = external addrspace(3) global [0 x i8]
-
-define amdgpu_kernel void @foo(ptr addrspace(1) inreg readonly captures(none) %arg, ptr addrspace(1) inreg readonly %arg1, ptr addrspace(1) inreg readonly %arg2, ptr addrspace(1) inreg %arg3, i32 inreg %arg4, i32 inreg %arg5, i32 inreg %arg6, i32 inreg %arg7, i32 inreg %arg8, i32 inreg %arg9, i32 %arg10, i32 %arg11, i32 %arg12, i32 %arg13, i32 %.frozen, i32 %arg14, i32 %arg15, i32 %arg16, i32 %arg17, i32 %arg18, i32 %arg19, i32 %arg20, i64 %arg21, i32 %arg22, i64 %arg23, i64 %arg24, i32 %arg25, i64 %arg26, i64 %arg27, i64 %arg28, i64 %arg29, i64 %arg30, i32 %arg31, i64 %arg32, i64 %arg33, ptr addrspace(8) %arg34, i32 %.decomposed, i64 %arg35, i32 %arg36, i32 %arg37, i32 %arg38, i32 %arg39, i32 %arg40, i32 %arg41, i32 %arg42, i32 %arg43, i32 %arg44, i32 %arg45, i32 %arg46, i32 %arg47, i32 %arg48, i32 %arg49, i32 %arg50, i32 %arg51, i32 %arg52, i32 %arg53, i32 %arg54, i32 %arg55, i32 %arg56, i32 %arg57, i32 %arg58, i32 %arg59, i32 %arg60, i32 %arg61, i32 %arg62, i32 %arg63, i32 %arg64, i32 %arg65, i32 %arg66, i32 %arg67, i32 %arg68, i32 %arg69, i32 %arg70, i32 %arg71, i32 %arg72, i32 %arg73, i32 %arg74, i32 %arg75, i32 %arg76, i32 %arg77, i32 %arg78, i32 %arg79, i32 %arg80, i32 %arg81, i32 %arg82, i32 %arg83, i32 %arg84, i32 %arg85, i32 %arg86, i32 %arg87, i32 %arg88, i32 %arg89, i32 %arg90, i32 %arg91, i32 %arg92, i32 %arg93, i32 %arg94, i32 %arg95, i32 %arg96, i32 %arg97, i32 %arg98, i32 %arg99, i32 %arg100, i32 %arg101, i32 %arg102, i32 %arg103, i32 %arg104, i32 %arg105, i32 %arg106, i32 %arg107, i32 %arg108, i32 %arg109, i32 %arg110, i32 %arg111, i32 %arg112, i32 %arg113, i32 %arg114, i32 %arg115, i32 %arg116, i32 %arg117, i32 %arg118, i32 %arg119, i32 %arg120, i32 %arg121, i32 %arg122, i32 %arg123, i32 %arg124, i32 %arg125, i32 %arg126, i32 %arg127, i32 %arg128, i32 %arg129, i32 %arg130, i32 %arg131, i32 %arg132, i32 %arg133, i32 %arg134, i32 %arg135, i32 %arg136, i32 %arg137, i32 %arg138, i32 %arg139, i32 %arg140, i32 %arg141, i32 %arg142, i32 %arg143, i32 %arg144, i32 %arg145, i32 %arg146, i64 %arg147, i64 %arg148, i64 %arg149, i64 %arg150, i64 %arg151, i64 %arg152, i64 %arg153, i64 %arg154, i64 %arg155, i64 %arg156, i64 %arg157, i64 %arg158, i64 %arg159, i64 %arg160, i64 %arg161, i64 %arg162, i64 %arg163, i64 %arg164, i64 %arg165, i64 %arg166, i64 %arg167, i64 %arg168, i64 %arg169, i64 %arg170, i64 %arg171, i64 %arg172, i64 %arg173, i64 %arg174, i64 %arg175, i64 %arg176, i64 %arg177, i64 %arg178, i64 %arg179, i64 %arg180, i64 %arg181, i64 %arg182, i64 %arg183, i64 %arg184, i64 %arg185, i64 %arg186, i64 %arg187, i64 %arg188, i64 %arg189, i64 %arg190, i64 %arg191, i64 %arg192, i64 %arg193, i64 %arg194, i64 %arg195, i64 %arg196, i64 %arg197, i64 %arg198, i64 %arg199, i64 %arg200, i64 %arg201, i64 %arg202, i64 %arg203, i64 %arg204, i64 %arg205, i64 %arg206, i64 %arg207, i64 %arg208, i64 %arg209, i64 %arg210, i64 %arg211, i64 %arg212, i64 %arg213, i64 %arg214, i64 %arg215, i64 %arg216, i64 %arg217, i64 %arg218, i64 %arg219, i64 %arg220, i64 %arg221, i64 %arg222, i64 %arg223, i64 %arg224, i64 %arg225, i64 %arg226, i64 %arg227, i64 %arg228, i64 %arg229, i64 %arg230, i64 %arg231, i64 %arg232, i64 %arg233, i64 %arg234, i64 %arg235, i64 %arg236, i64 %arg237, i64 %arg238, i64 %arg239, i64 %arg240, i64 %arg241, i64 %arg242, i64 %arg243, i64 %arg244, i64 %arg245, i64 %arg246, i64 %arg247, i64 %arg248, i64 %arg249, i64 %arg250, i64 %arg251, i64 %arg252, i64 %arg253, i64 %arg254, i64 %arg255, i64 %arg256, i64 %arg257, i64 %arg258, i64 %arg259, i64 %arg260, i64 %arg261, i64 %arg262, i64 %arg263, i64 %arg264, i64 %arg265, i64 %arg266, i64 %arg267, i64 %arg268, i64 %arg269, i64 %arg270, i64 %arg271, i64 %arg272, i64 %arg273, i64 %arg274, i64 %arg275, i64 %arg276, i64 %arg277, i64 %arg278, i64 %arg279, i64 %arg280, i64 %arg281, i64 %arg282, i64 %arg283, i64 %arg284, i64 %arg285, i64 %arg286, i64 %arg287, i64 %arg288, i64 %arg289, i64 %arg290, i64 %arg291, i64 %arg292, i64 %arg293, i64 %arg294, i64 %arg295, i64 %arg296, i64 %arg297, i64 %arg298, i64 %arg299, i64 %arg300, i64 %arg301, i64 %arg302, i64 %arg303, i64 %arg304, i64 %arg305, i64 %arg306, i64 %arg307, i64 %arg308, i64 %arg309, i64 %arg310, i64 %arg311, i64 %arg312, i64 %arg313, i64 %arg314, i64 %arg315, i64 %arg316, i64 %arg317, i64 %arg318, i64 %arg319, i64 %arg320, i64 %arg321, i64 %arg322, i64 %arg323, i64 %arg324, i64 %arg325, i64 %arg326, i64 %arg327, i64 %arg328, i64 %arg329, i64 %arg330, i64 %arg331, i64 %arg332, i64 %arg333, i64 %arg334, i64 %arg335, i64 %arg336, i64 %arg337, i64 %arg338, i64 %arg339, i64 %arg340, i64 %arg341, i64 %arg342, i64 %arg343, i64 %arg344, i64 %arg345, i64 %arg346, i64 %arg347, i64 %arg348, i64 %arg349, i64 %arg350, i64 %arg351, i64 %arg352, i64 %arg353, i64 %arg354, i64 %arg355, i64 %arg356, i64 %arg357, i64 %arg358, i64 %arg359, i64 %arg360, i64 %arg361, i64 %arg362, i64 %arg363, i64 %arg364, i64 %arg365, i64 %arg366, i64 %arg367, i64 %arg368, i64 %arg369, i64 %arg370, i64 %arg371, i64 %arg372, i64 %arg373, i64 %arg374, i64 %arg375, i64 %arg376, i64 %arg377, i64 %arg378, i64 %arg379, i64 %arg380, i64 %arg381, i64 %arg382, i64 %arg383, i64 %arg384, i64 %arg385, i64 %arg386, i64 %arg387, i64 %arg388, i64 %arg389, i64 %arg390, i64 %arg391, i64 %arg392, i64 %arg393, i64 %arg394, i64 %arg395, i64 %arg396, i64 %arg397, i64 %arg398, i64 %arg399, i64 %arg400, i64 %arg401, i64 %arg402, i64 %arg403, i64 %arg404, i64 %arg405, i64 %arg406, i64 %arg407, i64 %arg408, i64 %arg409, i64 %arg410, i64 %arg411, i64 %arg412, i64 %arg413, i64 %arg414, i64 %arg415, i64 %arg416, i64 %arg417, i64 %arg418, i64 %arg419, i64 %arg420, i64 %arg421, i64 %arg422, i64 %arg423, i64 %arg424, i64 %arg425, i64 %arg426, i64 %arg427, i64 %arg428, i64 %arg429, i64 %arg430, i64 %arg431, i64 %arg432, i64 %arg433, i64 %arg434, i64 %arg435, i64 %arg436, i64 %arg437, i64 %arg438, i64 %arg439, i64 %arg440, i64 %arg441, i64 %arg442, i64 %arg443, i64 %arg444, i64 %arg445, i64 %arg446, i64 %arg447, i64 %arg448, i64 %arg449, i64 %arg450, i64 %arg451, i64 %arg452, i64 %arg453, i64 %arg454, i64 %arg455, i64 %arg456, i64 %arg457, i64 %arg458, i64 %arg459, i64 %arg460, i64 %arg461, i64 %arg462, i64 %arg463, i64 %arg464, i64 %arg465, i64 %arg466, i64 %arg467, i64 %arg468, i64 %arg469, i64 %arg470, i64 %arg471, i64 %arg472, i64 %arg473, i64 %arg474, i64 %arg475, i64 %arg476, i64 %arg477, i64 %arg478, i64 %arg479, i64 %arg480, i64 %arg481, i64 %arg482, i64 %arg483, i64 %arg484, i64 %arg485, i64 %arg486, i64 %arg487, i64 %arg488, i64 %arg489, i64 %arg490, i64 %arg491, i64 %arg492, i64 %arg493, i64 %arg494, i64 %arg495, i64 %arg496, i64 %arg497, i64 %arg498, i64 %arg499, i64 %arg500, i64 %arg501, i64 %arg502, i64 %arg503, i64 %arg504, i64 %arg505, i64 %arg506, i64 %arg507, i64 %arg508, i64 %arg509, i64 %arg510, i64 %arg511, i64 %arg512, i64 %arg513, i64 %arg514, i64 %arg515, i64 %arg516, i64 %arg517, i64 %arg518, i64 %arg519, i64 %arg520, i64 %arg521, i64 %arg522, i64 %arg523, i64 %arg524, i64 %arg525, i64 %arg526, i64 %arg527, i64 %arg528, i64 %arg529, i64 %arg530, i64 %arg531, i64 %arg532, i64 %arg533, i64 %arg534, i64 %arg535, i64 %arg536, i64 %arg537, i64 %arg538, i64 %arg539, i64 %arg540, i64 %arg541, i64 %arg542, i64 %arg543, i64 %arg544, i64 %arg545, i64 %arg546, i64 %arg547, i64 %arg548, i64 %arg549, i64 %arg550, i64 %arg551, i64 %arg552, i64 %arg553, i64 %arg554, i64 %arg555, i64 %arg556, i64 %arg557, i64 %arg558, i64 %arg559, i64 %arg560, i64 %arg561, i64 %arg562, i64 %arg563, i64 %arg564, i64 %arg565, i64 %arg566, i64 %arg567, i64 %arg568, i64 %arg569, i64 %arg570, i64 %arg571, i64 %arg572, i64 %arg573, i64 %arg574, i64 %arg575, i64 %arg576, i64 %arg577, i64 %arg578, i64 %arg579, i1 %arg580, i1 %arg581, i1 %arg582, i1 %arg583, i1 %arg584, i1 %arg585, i1 %arg586, i1 %arg587, i1 %arg588, i1 %arg589, i1 %arg590, i1 %arg591, i1 %arg592, i1 %arg593, i1 %arg594, i1 %arg595, i1 %arg596, i1 %arg597, i1 %arg598, i1 %arg599, i1 %arg600, i1 %arg601, i1 %arg602, i1 %arg603, i1 %arg604, i1 %arg605, i1 %arg606, i1 %arg607, i1 %arg608, i1 %arg609, i1 %arg610, i1 %arg611, i1 %arg612, i1 %arg613, i1 %arg614, i1 %arg615, i1 %arg616, i1 %arg617, i1 %arg618, i1 %arg619, i1 %arg620, i1 %arg621, i1 %arg622, i1 %arg623, i1 %arg624, i1 %arg625, i1 %arg626, i1 %arg627, i1 %arg628, i1 %arg629, i1 %arg630, i1 %arg631, i1 %arg632, i1 %arg633, i1 %arg634, i1 %arg635, i1 %arg636, i1 %arg637, i1 %arg638, i1 %arg639, i1 %arg640, i1 %arg641, i1 %arg642, i1 %arg643, i1 %arg644, i1 %arg645, i1 %arg646, i1 %arg647, i1 %arg648, i1 %arg649, i1 %arg650, i1 %arg651, i1 %arg652, i1 %arg653, i1 %arg654, i1 %arg655, i1 %arg656, i1 %arg657, i1 %arg658, i1 %arg659, i1 %arg660, i1 %arg661, i1 %arg662, i1 %arg663, i1 %arg664, i1 %arg665, i1 %arg666, i1 %arg667, i1 %arg668, i1 %arg669, i1 %arg670, i1 %arg671, i1 %arg672, i1 %arg673, i1 %arg674, i1 %arg675, i1 %arg676, i1 %arg677, i1 %arg678, i1 %arg679, i1 %arg680, i1 %arg681, i1 %arg682, i1 %arg683, i1 %arg684, i32 %arg685, i1 %arg686, i32 %arg687, ptr addrspace(8) %arg688, i32 %arg689, i32 %arg690, i1 %arg691, i32 %arg692, i32 %arg693, i1 %arg694, i32 %arg695, i32 %arg696, i1 %arg697, i32 %arg698, i32 %arg699, i32 %arg700, i1 %arg701, i32 %arg702, i32 %arg703, i32 %arg704, i1 %arg705, i32 %arg706, i32 %arg707, i1 %arg708, i32 %arg709, i32 %arg710, i32 %arg711, i1 %arg712, i32 %arg713, i32 %arg714, i32 %arg715, i1 %arg716, i32 %arg717, i32 %arg718, i1 %arg719, i32 %arg720, i32 %arg721, i32 %arg722, i1 %arg723, i32 %arg724, i32 %arg725, i32 %arg726, i1 %arg727, i32 %arg728, i32 %arg729, i32 %arg730, i32 %arg731, i32 %arg732, i32 %arg733, i32 %arg734, i1 %arg735, i32 %arg736, i32 %arg737, i32 %arg738, i1 %arg739, i32 %arg740, i32 %arg741, i32 %arg742, i1 %arg743, i32 %arg744, i32 %arg745, i32 %arg746, i1 %arg747, i32 %arg748, i32 %arg749, i32 %arg750, i1 %arg751, i32 %arg752, i32 %arg753, i32 %arg754, i32 %arg755, i32 %arg756, i1 %arg757, i32 %arg758, i32 %arg759, i32 %arg760, i1 %arg761, i32 %arg762, i32 %arg763, i32 %arg764, i1 %arg765, i32 %arg766, i32 %arg767, i32 %arg768, i32 %arg769, i32 %arg770, i1 %arg771, i1 %arg772, i32 %arg773, i1 %arg774, i1 %arg775, i1 %arg776, i1 %arg777, i1 %arg778, i32 %arg779, i1 %arg780, i32 %arg781, i32 %arg782, i32 %arg783, i1 %arg784, i32 %arg785, i32 %arg786, i32 %arg787, i1 %arg788, i32 %arg789, i32 %arg790, i32 %arg791, i1 %arg792, i32 %arg793, i32 %arg794, i32 %arg795, i1 %arg796, i32 %arg797, i32 %arg798, i32 %arg799, i1 %arg800, i32 %arg801, i32 %arg802, i32 %arg803, i1 %arg804, i32 %arg805, i32 %arg806, i32 %arg807, i1 %arg808, i32 %arg809, i32 %arg810, i32 %arg811, i1 %arg812, i32 %arg813, i32 %arg814, i32 %arg815, i1 %arg816, i32 %arg817, i32 %arg818, i32 %arg819, i1 %arg820, i32 %arg821, i32 %arg822, i32 %arg823, i1 %arg824, i32 %arg825, i32 %arg826, i32 %arg827, i1 %arg828, i32 %arg829, i32 %arg830, i32 %arg831, ptr addrspace(3) %global_smem, i32 %arg832, i16 %arg833, i32 %arg834, ptr addrspace(3) %arg835, i16 %arg836, i16 %arg837, i16 %arg838, i16 %arg839, i16 %arg840, i16 %arg841, i16 %arg842, i32 %arg843, i16 %arg844, i32 %arg845, i16 %arg846, ptr addrspace(3) %arg847, i16 %arg848, i32 %arg849, i16 %arg850, ptr addrspace(3) %arg851, i16 %arg852, i32 %arg853, i16 %arg854, ptr addrspace(3) %arg855, i32 %arg856, i16 %arg857, i16 %arg858, i16 %arg859, i32 %arg860, i32 %arg861, i16 %arg862, ptr addrspace(3) %arg863, i16 %arg864, i16 %arg865, i16 %arg866, i32 %arg867, i16 %arg868, i32 %arg869, i16 %arg870, ptr addrspace(3) %arg871, i16 %arg872, i32 %arg873, i16 %arg874, ptr addrspace(3) %arg875, i16 %arg876, i32 %arg877, i16 %arg878, ptr addrspace(3) %arg879, i16 %arg880, i16 %arg881, i16 %arg882, i32 %arg883, i16 %arg884, ptr addrspace(3) %arg885, i16 %arg886, i16 %arg887, i16 %arg888, i16 %arg889, i32 %arg890, i16 %arg891, ptr addrspace(3) %arg892, i16 %arg893, i32 %arg894, i16 %arg895, ptr addrspace(3) %arg896, i16 %arg897, i32 %arg898, i16 %arg899, ptr addrspace(3) %arg900, i16 %arg901, i32 %arg902, i1 %arg903, i32 %arg904, i32 %arg905, i32 %arg906, i32 %arg907, i32 %arg908, i32 %arg909, i32 %arg910, i32 %arg911, i32 %arg912, i32 %arg913, i32 %arg914, i32 %arg915, i32 %arg916, i32 %arg917, i32 %arg918, i32 %arg919, i32 %arg920, i32 %arg921, i32 %arg922, i32 %arg923, i32 %arg924, i32 %arg925, i32 %arg926, i32 %arg927, i32 %arg928, i32 %arg929, i32 %arg930, i32 %arg931, i32 %arg932, i32 %arg933, i32 %arg934, i32 %arg935, i32 %arg936, i32 %arg937, i32 %arg938, i32 %arg939, i32 %arg940, i32 %arg941, i32 %arg942, i32 %arg943, i32 %arg944, i32 %arg945, i32 %arg946, i32 %arg947, i32 %arg948, i32 %arg949, i32 %arg950, i32 %arg951, i32 %arg952, i32 %arg953, i32 %arg954, i32 %arg955, i32 %arg956, i32 %arg957, i32 %arg958, i32 %arg959, i32 %arg960, i32 %arg961, i32 %arg962, i32 %arg963, i32 %arg964, i32 %arg965, i32 %arg966, i32 %arg967, i32 %arg968, i32 %arg969, i32 %arg970, i32 %arg971, i32 %arg972, i32 %arg973, i32 %arg974, i32 %arg975, i32 %arg976, i32 %arg977, i32 %arg978, i32 %arg979, i32 %arg980, i32 %arg981, i32 %arg982, i32 %arg983, i32 %arg984, i32 %arg985, i32 %arg986, i32 %arg987, i32 %arg988, i32 %arg989, i32 %arg990, i32 %arg991, i32 %arg992, i32 %arg993, i32 %arg994, i32 %arg995, i32 %arg996, i32 %arg997, i32 %arg998, i32 %arg999, float %arg1000, float %arg1001, i32 %arg1002, i32 %arg1003, i32 %arg1004, i32 %.pn3661, i32 %.pn261662, i32 %.pn259663, i32 %.pn257664, i32 %.pn253666, i32 %.pn249668, i32 %.pn247669, i32 %.pn245670, i32 %.pn243671, i32 %.pn241672, i32 %.pn239673, i32 %.pn237674, i32 %.pn233676, i32 %.pn231677, i32 %.pn223681, i32 %.pn105740, i32 %.pn103741, i32 %.pn101742, i32 %.pn99743, i32 %.pn93746, i32 %.pn91747, i32 %.pn89748, i32 %.pn87749, i32 %.pn85750, i32 %.pn83751, i32 %.pn81752, i32 %.pn79753, i32 %.pn77754, i32 %.pn75755, i32 %.pn73756, i32 %.pn71757, i32 %.pn69758, i32 %.pn67759, i32 %.pn65760, i32 %.pn63761, i32 %.pn61762, i32 %.pn59763, i32 %.pn57764, i32 %.pn55765, i32 %.pn53766, i32 %.pn51767, i32 %.pn49768, i32 %.pn41772, i1 %arg1005, i1 %arg1006, i1 %arg1007, i1 %arg1008, i1 %arg1009, i32 %arg1010, i32 %arg1011, i1 %arg1012, i32 %arg1013, i32 %arg1014, i1 %arg1015, i32 %arg1016, i32 %arg1017, i1 %arg1018, i32 %arg1019, i32 %arg1020, i1 %arg1021, i32 %arg1022, i32 %arg1023, i32 %arg1024, i1 %arg1025, i32 %arg1026, i32 %arg1027, i1 %arg1028, i32 %arg1029, i32 %arg1030, i1 %arg1031, i32 %arg1032, i32 %arg1033, i1 %arg1034, i32 %arg1035, i1 %arg1036, i32 %arg1037, i1 %arg1038, i32 %arg1039, i1 %arg1040, i32 %arg1041, i32 %arg1042, i1 %arg1043, i32 %arg1044, i1 %arg1045, i32 %arg1046, i1 %arg1047, i32 %arg1048, i1 %arg1049, i32 %arg1050, i1 %arg1051, i32 %arg1052, i1 %arg1053, i1 %arg1054, i32 %arg1055, i32 %arg1056, i1 %arg1057, i32 %arg1058, i1 %arg1059, i32 %arg1060, i32 %arg1061, float %arg1062, <4 x float> %arg1063, float %arg1064, <4 x float> %arg1065, <4 x float> %arg1066, <4 x float> %arg1067, <4 x float> %arg1068, <4 x float> %arg1069, <4 x float> %arg1070, <4 x float> %arg1071, <4 x float> %arg1072, <4 x float> %arg1073, <4 x float> %arg1074, ptr addrspace(3) %arg1075, i32 %arg1076, i32 %arg1077, ptr addrspace(3) %arg1078, i32 %arg1079, ptr addrspace(3) %arg1080, ptr addrspace(3) %arg1081, i32 %arg1082, ptr addrspace(3) %arg1083, i32 %arg1084, i32 %arg1085, i32 %arg1086, ptr addrspace(3) %arg1087, i32 %arg1088, i32 %arg1089, i32 %arg1090, ptr addrspace(3) %arg1091, ptr addrspace(3) %arg1092, i32 %arg1093, i32 %arg1094, ptr addrspace(3) %arg1095, i32 %arg1096, i32 %arg1097, i32 %arg1098, ptr addrspace(3) %arg1099, ptr addrspace(3) %arg1100, i32 %arg1101, ptr addrspace(3) %arg1102, i32 %arg1103, i16 %arg1104, ptr addrspace(3) %arg1105, i32 %arg1106, i16 %arg1107, i16 %arg1108, i32 %arg1109, ptr addrspace(3) %arg1110, i32 %arg1111, i32 %arg1112, i32 %arg1113, i32 %arg1114, i32 %arg1115, i32 %arg1116, i32 %arg1117, i32 %arg1118, i32 %arg1119, i32 %arg1120, i32 %arg1121, i32 %arg1122, i32 %arg1123, i32 %arg1124, i32 %arg1125, i32 %arg1126, i32 %arg1127, i32 %arg1128, i32 %arg1129, i32 %arg1130, i32 %arg1131, i32 %arg1132, i32 %arg1133, i32 %arg1134, i32 %arg1135, i32 %arg1136, i32 %arg1137, i32 %arg1138, i32 %arg1139, i32 %arg1140, i32 %arg1141, i32 %arg1142, i32 %arg1143, i32 %arg1144, i32 %arg1145, i32 %arg1146, i32 %arg1147, i32 %arg1148, i32 %arg1149, i32 %arg1150, i32 %arg1151, i32 %arg1152, i32 %arg1153, i32 %arg1154, i32 %arg1155, i32 %arg1156, i32 %arg1157, i32 %arg1158, i32 %arg1159, i1 %arg1160, i1 %arg1161, i1 %arg1162, i1 %arg1163, i1 %arg1164, i1 %arg1165, i1 %arg1166, i1 %arg1167, i1 %arg1168, i1 %arg1169, i1 %arg1170, i1 %arg1171, i1 %arg1172, i1 %arg1173, i1 %arg1174, i1 %arg1175, i1 %arg1176, i1 %arg1177, i1 %arg1178, i1 %arg1179, i1 %arg1180, i1 %arg1181, i1 %arg1182, i1 %arg1183, i1 %arg1184, i1 %arg1185, i1 %arg1186, i1 %arg1187, i1 %arg1188, i1 %arg1189, i1 %arg1190, i1 %arg1191, i1 %arg1192, i1 %arg1193, i1 %arg1194, i1 %arg1195, i1 %arg1196, i1 %arg1197, i1 %arg1198, i1 %arg1199, i1 %arg1200, i1 %arg1201, i1 %arg1202, i1 %arg1203, i1 %arg1204, i1 %arg1205, i1 %arg1206, i1 %arg1207, ptr addrspace(8) %arg1208, i32 %arg1209, i1 %arg1210, i32 %arg1211, i1 %arg1212, i32 %arg1213, i1 %arg1214, i32 %arg1215, i1 %arg1216, i32 %arg1217, i1 %arg1218, i32 %arg1219, i1 %arg1220, i32 %arg1221, i1 %arg1222, i32 %arg1223, i1 %arg1224, i32 %arg1225, i1 %arg1226, i32 %arg1227, i1 %arg1228, i32 %arg1229, i1 %arg1230, i32 %arg1231, i1 %arg1232, i32 %arg1233, i1 %arg1234, i32 %arg1235, i1 %arg1236, i32 %arg1237, i1 %arg1238, i32 %arg1239, i1 %arg1240, i32 %arg1241, i1 %arg1242, i32 %arg1243, i1 %arg1244, i32 %arg1245, i1 %arg1246, i32 %arg1247, i1 %arg1248, i32 %arg1249, i1 %arg1250, i32 %arg1251, i1 %arg1252, i32 %arg1253, i1 %arg1254, i32 %arg1255, i1 %arg1256, i32 %arg1257, i1 %arg1258, i32 %arg1259, i1 %arg1260, i32 %arg1261, i1 %arg1262, i32 %arg1263, i1 %arg1264, i32 %arg1265, i1 %arg1266, i32 %arg1267, i1 %arg1268, i32 %arg1269, i1 %arg1270, i32 %arg1271, i1 %arg1272, i32 %arg1273, i1 %arg1274, i32 %arg1275, i1 %arg1276, i32 %arg1277, i1 %arg1278, i32 %arg1279, i1 %arg1280, i32 %arg1281, i1 %arg1282, i32 %arg1283, i1 %arg1284, i32 %arg1285, i1 %arg1286, i32 %arg1287, i1 %arg1288, i32 %arg1289, i1 %arg1290, i32 %arg1291, i1 %arg1292, i32 %arg1293, i1 %arg1294, i32 %arg1295, i1 %arg1296, i32 %arg1297, i1 %arg1298, i32 %arg1299, i32 %.idx, ptr addrspace(3) %arg1300, i16 %arg1301, ptr addrspace(3) %arg1302, i16 %arg1303, i16 %arg1304, ptr addrspace(3) %arg1305, i16 %arg1306, i16 %arg1307, i16 %arg1308, ptr addrspace(3) %arg1309, i16 %arg1310, i16 %arg1311, i32 %arg1312, ptr addrspace(3) %arg1313, i32 %arg1314, i16 %arg1315, ptr addrspace(3) %arg1316, i32 %arg1317, i16 %arg1318, ptr addrspace(3) %arg1319, i16 %arg1320, i32 %arg1321, i16 %arg1322, ptr addrspace(3) %arg1323, i16 %arg1324, ptr addrspace(3) %arg1325, i32 %arg1326, i16 %arg1327, ptr addrspace(3) %arg1328, i16 %arg1329, ptr addrspace(3) %arg1330, i16 %arg1331, i32 %arg1332, i16 %arg1333, i16 %arg1334, i16 %arg1335, ptr addrspace(3) %arg1336, i32 %arg1337, ptr addrspace(3) %arg1338, i32 %arg1339, ptr addrspace(3) %arg1340, i16 %arg1341, i32 %arg1342, i16 %arg1343, i16 %arg1344, i16 %arg1345, i32 %arg1346, i16 %arg1347, i16 %arg1348, i16 %arg1349) #0 {
-bb:
- %i = tail call i32 @llvm.amdgcn.workitem.id.x()
- %i1350 = tail call i32 @llvm.amdgcn.readfirstlane.i32(i32 %arg4)
- %i1351 = and i32 %arg4, 448
- %i1352 = tail call i32 @llvm.amdgcn.workgroup.id.x()
- %i1353 = add i32 0, 0
- %i1354 = sdiv i32 0, 0
- %.frozen1 = freeze i32 %arg4
- %i1355 = sdiv i32 %arg4, %arg4
- %i1356 = mul i32 %arg4, %arg4
- %.decomposed2 = sub i32 %arg4, %arg4
- %i1357 = and i32 %arg4, 63
- %i1358 = or disjoint i32 %arg4, %arg17
- %i1359 = and i32 %arg11, 256
- %i1360 = lshr exact i32 %arg18, 0
- %i1361 = or disjoint i32 %arg18, 2
- %i1362 = or disjoint i32 %arg4, 4
- %i1363 = or disjoint i32 %arg4, 6
- %i1364 = or disjoint i32 %arg4, 8
- %i1365 = or disjoint i32 %arg4, 10
- %i1366 = or disjoint i32 %arg4, 12
- %i1367 = or disjoint i32 0, 14
- %i1368 = or disjoint i32 %arg4, 16
- %i1369 = or disjoint i32 %arg4, 18
- %i1370 = or disjoint i32 0, 20
- %i1371 = or disjoint i32 %arg4, 22
- %i1372 = or disjoint i32 %arg4, 24
- %i1373 = or disjoint i32 %arg4, 26
- %i1374 = or disjoint i32 %arg4, 28
- %i1375 = or disjoint i32 0, 30
- %i1376 = or disjoint i32 %arg4, 32
- %i1377 = or disjoint i32 %arg4, 34
- %i1378 = or disjoint i32 0, 36
- %i1379 = or disjoint i32 0, 38
- %i1380 = or disjoint i32 %arg4, 40
- %i1381 = or disjoint i32 %arg4, 42
- %i1382 = or disjoint i32 %arg4, 44
- %i1383 = or disjoint i32 %arg4, 46
- %i1384 = or disjoint i32 %arg4, 48
- %i1385 = or disjoint i32 %arg4, 50
- %i1386 = or disjoint i32 %arg4, 52
- %i1387 = or disjoint i32 %arg4, 56
- %i1388 = or disjoint i32 %arg4, 58
- %i1389 = or disjoint i32 %arg4, 60
- %i1390 = or disjoint i32 %arg4, 62
- %i1391 = or disjoint i32 %arg4, 64
- %i1392 = or disjoint i32 %arg4, 66
- %i1393 = or disjoint i32 %arg4, 68
- %i1394 = or disjoint i32 %arg4, 70
- %i1395 = or disjoint i32 %arg4, 72
- %i1396 = or disjoint i32 %arg4, 74
- %i1397 = or disjoint i32 %arg4, 76
- %i1398 = or disjoint i32 %arg4, 78
- %i1399 = or disjoint i32 %arg4, 80
- %i1400 = or disjoint i32 %arg4, 82
- %i1401 = or disjoint i32 %arg4, 84
- %i1402 = or disjoint i32 %arg4, 86
- %i1403 = or disjoint i32 %arg4, 88
- %i1404 = or disjoint i32 %arg4, 90
- %i1405 = or disjoint i32 %arg4, 92
- %i1406 = or disjoint i32 %arg4, 94
- %i1407 = or disjoint i32 %arg4, 96
- %i1408 = or disjoint i32 %arg4, 98
- %i1409 = or disjoint i32 %arg4, 100
- %i1410 = or disjoint i32 %arg4, 102
- %i1411 = or disjoint i32 %arg4, 104
- %i1412 = or disjoint i32 %arg4, 106
- %i1413 = or disjoint i32 %arg4, 108
- %i1414 = or disjoint i32 %arg4, 110
- %i1415 = or disjoint i32 %arg4, 112
- %i1416 = or disjoint i32 %arg4, 114
- %i1417 = or disjoint i32 0, 116
- %i1418 = or disjoint i32 0, 118
- %i1419 = or disjoint i32 %arg4, 120
- %i1420 = or disjoint i32 %arg4, 122
- %i1421 = or disjoint i32 0, 0
- %i1422 = or disjoint i32 0, 1
- %i1423 = or disjoint i32 %arg4, 0
- %i1424 = or disjoint i32 0, 130
- %i1425 = or disjoint i32 0, 0
- %i1426 = or disjoint i32 0, 1
- %i1427 = or disjoint i32 0, 1
- %i1428 = or disjoint i32 0, 0
- %i1429 = or disjoint i32 0, 0
- %i1430 = or disjoint i32 0, 150
- %i1431 = or disjoint i32 0, 0
- %i1432 = or disjoint i32 %arg4, 154
- %i1433 = or disjoint i32 %arg4, 156
- %i1434 = or disjoint i32 0, 158
- %i1435 = or disjoint i32 %arg4, 160
- %i1436 = or disjoint i32 0, 162
- %i1437 = or disjoint i32 0, 164
- %i1438 = or disjoint i32 %arg4, 166
- %i1439 = or disjoint i32 %arg4, 168
- %i1440 = or disjoint i32 %arg4, 170
- %i1441 = or disjoint i32 %arg4, 172
- %i1442 = or disjoint i32 %arg4, 174
- %i1443 = or disjoint i32 %arg4, 176
- %i1444 = or disjoint i32 %arg4, 178
- %i1445 = or disjoint i32 %arg4, 180
- %i1446 = or disjoint i32 %arg4, 182
- %i1447 = or disjoint i32 %arg4, 184
- %i1448 = or disjoint i32 %arg4, 186
- %i1449 = or disjoint i32 %arg4, 188
- %i1450 = or disjoint i32 %arg4, 190
- %i1451 = or disjoint i32 %arg4, 192
- %i1452 = or disjoint i32 %arg4, 194
- %i1453 = or disjoint i32 %arg4, 196
- %i1454 = or disjoint i32 %arg4, 198
- %i1455 = or disjoint i32 %arg4, 200
- %i1456 = or disjoint i32 %arg4, 202
- %i1457 = or disjoint i32 %arg4, 204
- %i1458 = or disjoint i32 %arg4, 206
- %i1459 = or disjoint i32 %arg4, 208
- %i1460 = or disjoint i32 %arg4, 210
- %i1461 = or disjoint i32 %arg4, 212
- %i1462 = or disjoint i32 %arg4, 214
- %i1463 = or disjoint i32 %arg4, 216
- %i1464 = or disjoint i32 %arg4, 218
- %i1465 = or disjoint i32 %arg4, 1
- %i1466 = or disjoint i32 %arg4, 222
- %i1467 = or disjoint i32 %arg4, 224
- %i1468 = or disjoint i32 %arg4, 226
- %i1469 = or disjoint i32 %arg4, 228
- %i1470 = or disjoint i32 1, 230
- %i1471 = or disjoint i32 %arg4, 232
- %i1472 = or disjoint i32 %arg4, 1
- %i1473 = or disjoint i32 %arg4, 236
- %i1474 = or disjoint i32 %arg4, 238
- %i1475 = or disjoint i32 %arg4, 240
- %i1476 = or disjoint i32 %arg4, 242
- %i1477 = or disjoint i32 %arg4, 244
- %i1478 = or disjoint i32 %arg4, 246
- %i1479 = or disjoint i32 %arg4, 248
- %i1480 = or disjoint i32 1, 250
- %i1481 = or disjoint i32 %arg4, 252
- %i1482 = or disjoint i32 %arg19, 254
- %i1483 = and i32 %arg20, 255
- %i1484 = sext i32 %arg13 to i64
- %i1485 = shl nsw i64 %arg21, 0
- %i1486 = zext nneg i32 %arg4 to i64
- %i1487 = zext nneg i32 %arg4 to i64
- %i1488 = or disjoint i64 %arg21, %arg21
- %i1489 = zext i32 %arg4 to i64
- %i1490 = zext nneg i32 %arg4 to i64
- %i1491 = mul i64 %arg23, 1
- %i1492 = mul nuw nsw i64 %arg24, %arg26
- %i1493 = add i64 %arg27, 1
- %i1494 = add i64 %arg28, %arg29
- %i1495 = trunc i64 %arg30 to i32
- %i1496 = add i32 %arg6, 1
- %i1497 = sdiv i32 %arg31, 256
- %i1498 = icmp sgt i32 %arg31, 255
- %i1499 = sext i32 %arg4 to i64
- %i1500 = icmp slt i64 %arg32, %arg33
- %i1501 = icmp slt i32 %arg25, 1
- %i1502 = tail call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1(ptr addrspace(1) %arg, i16 0, i64 2147483646, i32 159744)
- %i1503 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 0, i32 0, i32 0)
- %i1504 = sext i32 %.decomposed to i64
- %i1505 = shl nsw i64 %arg35, 1
- %i1506 = zext nneg i32 %arg7 to i64
- %i1507 = zext nneg i32 %arg4 to i64
- %i1508 = zext nneg i32 %arg4 to i64
- %i1509 = zext nneg i32 %arg4 to i64
- %i1510 = zext nneg i32 %arg4 to i64
- %i1511 = zext nneg i32 %arg4 to i64
- %i1512 = zext nneg i32 %arg4 to i64
- %i1513 = zext nneg i32 %arg4 to i64
- %i1514 = zext nneg i32 %arg4 to i64
- %i1515 = zext nneg i32 %arg4 to i64
- %i1516 = zext nneg i32 %arg4 to i64
- %i1517 = zext nneg i32 %arg4 to i64
- %i1518 = zext nneg i32 %arg4 to i64
- %i1519 = zext nneg i32 %arg4 to i64
- %i1520 = zext nneg i32 %arg4 to i64
- %i1521 = zext nneg i32 %arg4 to i64
- %i1522 = zext nneg i32 %arg4 to i64
- %i1523 = zext nneg i32 %arg4 to i64
- %i1524 = zext nneg i32 %arg4 to i64
- %i1525 = zext nneg i32 %arg4 to i64
- %i1526 = zext nneg i32 %arg4 to i64
- %i1527 = zext nneg i32 %arg4 to i64
- %i1528 = zext nneg i32 %arg4 to i64
- %i1529 = zext nneg i32 %arg4 to i64
- %i1530 = zext nneg i32 %arg4 to i64
- %i1531 = zext nneg i32 %arg4 to i64
- %i1532 = zext nneg i32 %arg4 to i64
- %i1533 = zext nneg i32 %arg4 to i64
- %i1534 = zext nneg i32 %arg4 to i64
- %i1535 = zext nneg i32 %arg4 to i64
- %i1536 = zext nneg i32 %arg4 to i64
- %i1537 = zext nneg i32 %arg4 to i64
- %i1538 = zext nneg i32 %arg4 to i64
- %i1539 = zext nneg i32 %arg4 to i64
- %i1540 = zext nneg i32 %arg4 to i64
- %i1541 = zext nneg i32 %arg4 to i64
- %i1542 = zext nneg i32 %arg4 to i64
- %i1543 = zext nneg i32 %arg4 to i64
- %i1544 = zext nneg i32 %arg4 to i64
- %i1545 = zext nneg i32 %arg4 to i64
- %i1546 = zext nneg i32 %arg4 to i64
- %i1547 = zext nneg i32 %arg4 to i64
- %i1548 = zext nneg i32 %arg4 to i64
- %i1549 = zext nneg i32 %arg4 to i64
- %i1550 = zext nneg i32 %arg4 to i64
- %i1551 = zext nneg i32 %arg4 to i64
- %i1552 = zext nneg i32 %arg4 to i64
- %i1553 = zext nneg i32 %arg4 to i64
- %i1554 = zext nneg i32 %arg4 to i64
- %i1555 = zext nneg i32 %arg4 to i64
- %i1556 = zext nneg i32 %arg4 to i64
- %i1557 = zext nneg i32 %arg4 to i64
- %i1558 = zext nneg i32 %arg4 to i64
- %i1559 = zext nneg i32 %arg4 to i64
- %i1560 = zext nneg i32 %arg4 to i64
- %i1561 = zext nneg i32 %arg4 to i64
- %i1562 = zext nneg i32 %arg4 to i64
- %i1563 = zext nneg i32 %arg4 to i64
- %i1564 = zext nneg i32 %arg4 to i64
- %i1565 = zext nneg i32 0 to i64
- %i1566 = zext nneg i32 1 to i64
- %i1567 = zext nneg i32 %arg4 to i64
- %i1568 = zext nneg i32 %arg4 to i64
- %i1569 = zext nneg i32 %arg4 to i64
- %i1570 = zext nneg i32 %arg4 to i64
- %i1571 = zext nneg i32 %arg4 to i64
- %i1572 = zext nneg i32 %arg4 to i64
- %i1573 = zext nneg i32 %arg4 to i64
- %i1574 = zext nneg i32 %arg4 to i64
- %i1575 = zext nneg i32 %arg4 to i64
- %i1576 = zext nneg i32 %arg4 to i64
- %i1577 = zext nneg i32 %arg4 to i64
- %i1578 = zext nneg i32 %arg4 to i64
- %i1579 = zext nneg i32 %arg4 to i64
- %i1580 = zext nneg i32 %arg4 to i64
- %i1581 = zext nneg i32 %arg4 to i64
- %i1582 = zext nneg i32 %arg4 to i64
- %i1583 = zext nneg i32 %arg4 to i64
- %i1584 = zext nneg i32 %arg4 to i64
- %i1585 = zext nneg i32 %arg4 to i64
- %i1586 = zext nneg i32 %arg4 to i64
- %i1587 = zext nneg i32 %arg4 to i64
- %i1588 = zext nneg i32 %arg4 to i64
- %i1589 = zext nneg i32 %arg4 to i64
- %i1590 = zext nneg i32 %arg4 to i64
- %i1591 = zext nneg i32 %arg4 to i64
- %i1592 = zext nneg i32 %arg4 to i64
- %i1593 = zext nneg i32 %arg4 to i64
- %i1594 = zext nneg i32 %arg4 to i64
- %i1595 = zext nneg i32 %arg4 to i64
- %i1596 = zext nneg i32 %arg4 to i64
- %i1597 = zext nneg i32 %arg4 to i64
- %i1598 = zext nneg i32 %arg4 to i64
- %i1599 = zext nneg i32 %arg4 to i64
- %i1600 = zext nneg i32 %arg4 to i64
- %i1601 = zext nneg i32 %arg4 to i64
- %i1602 = zext nneg i32 %arg4 to i64
- %i1603 = zext nneg i32 %arg4 to i64
- %i1604 = zext nneg i32 %arg4 to i64
- %i1605 = zext nneg i32 %arg4 to i64
- %i1606 = zext nneg i32 %arg4 to i64
- %i1607 = zext nneg i32 %arg4 to i64
- %i1608 = zext nneg i32 %arg4 to i64
- %i1609 = zext nneg i32 %arg4 to i64
- %i1610 = zext nneg i32 %arg4 to i64
- %i1611 = zext nneg i32 %arg4 to i64
- %i1612 = zext nneg i32 %arg4 to i64
- %i1613 = zext nneg i32 %arg4 to i64
- %i1614 = zext nneg i32 %arg4 to i64
- %i1615 = zext nneg i32 %arg4 to i64
- %i1616 = zext nneg i32 %arg4 to i64
- %i1617 = zext nneg i32 %arg4 to i64
- %i1618 = zext nneg i32 %arg4 to i64
- %i1619 = or disjoint i64 %arg21, %arg21
- %i1620 = or disjoint i64 1, %arg21
- %i1621 = or disjoint i64 %arg21, %arg21
- %i1622 = or disjoint i64 %arg21, %arg21
- %i1623 = or disjoint i64 %arg21, %arg21
- %i1624 = or disjoint i64 1, %arg21
- %i1625 = or disjoint i64 %arg21, %arg21
- %i1626 = or disjoint i64 %arg21, %arg21
- %i1627 = or disjoint i64 1, %arg21
- %i1628 = or disjoint i64 %arg21, %arg21
- %i1629 = or disjoint i64 %arg21, %arg21
- %i1630 = or disjoint i64 %arg21, %arg21
- %i1631 = or disjoint i64 %arg21, %arg21
- %i1632 = or disjoint i64 %arg21, %arg21
- %i1633 = or disjoint i64 %arg21, %arg21
- %i1634 = or disjoint i64 %arg21, %arg21
- %i1635 = or disjoint i64 %arg21, %arg21
- %i1636 = or disjoint i64 %arg21, %arg21
- %i1637 = or disjoint i64 %arg21, %arg21
- %i1638 = or disjoint i64 %arg21, %arg21
- %i1639 = or disjoint i64 %arg21, %arg21
- %i1640 = or disjoint i64 %arg21, %arg21
- %i1641 = or disjoint i64 %arg21, %arg21
- %i1642 = or disjoint i64 1, %arg21
- %i1643 = or disjoint i64 1, %arg21
- %i1644 = or disjoint i64 %arg21, %arg21
- %i1645 = or disjoint i64 %arg21, %arg21
- %i1646 = or disjoint i64 %arg21, %arg21
- %i1647 = or disjoint i64 %arg21, %arg21
- %i1648 = or disjoint i64 %arg21, %arg21
- %i1649 = or disjoint i64 %arg21, %arg21
- %i1650 = or disjoint i64 %arg21, %arg21
- %i1651 = or disjoint i64 %arg21, %arg21
- %i1652 = or disjoint i64 %arg21, %arg21
- %i1653 = or disjoint i64 %arg21, %arg21
- %i1654 = or disjoint i64 %arg21, %arg21
- %i1655 = or disjoint i64 %arg21, %arg21
- %i1656 = or disjoint i64 %arg21, %arg21
- %i1657 = or disjoint i64 %arg21, %arg21
- %i1658 = or disjoint i64 %arg21, %arg21
- %i1659 = or disjoint i64 %arg21, %arg21
- %i1660 = or disjoint i64 %arg21, %arg21
- %i1661 = or disjoint i64 %arg21, %arg21
- %i1662 = or disjoint i64 %arg21, %arg21
- %i1663 = or disjoint i64 %arg21, %arg21
- %i1664 = or disjoint i64 %arg21, %arg21
- %i1665 = or disjoint i64 1, %arg21
- %i1666 = or disjoint i64 %arg21, %arg21
- %i1667 = or disjoint i64 %arg21, %arg21
- %i1668 = or disjoint i64 %arg21, %arg21
- %i1669 = or disjoint i64 %arg21, %arg21
- %i1670 = or disjoint i64 %arg21, %arg21
- %i1671 = or disjoint i64 %arg21, %arg21
- %i1672 = or disjoint i64 %arg21, %arg21
- %i1673 = or disjoint i64 %arg21, %arg21
- %i1674 = or disjoint i64 %arg21, %arg21
- %i1675 = or disjoint i64 %arg21, %arg21
- %i1676 = or disjoint i64 %arg21, %arg21
- %i1677 = or disjoint i64 %arg21, %arg21
- %i1678 = or disjoint i64 0, %arg21
- %i1679 = or disjoint i64 0, %arg21
- %i1680 = or disjoint i64 0, 0
- %i1681 = or disjoint i64 1, 1
- %i1682 = or disjoint i64 %arg21, %arg21
- %i1683 = or disjoint i64 %arg21, %arg21
- %i1684 = or disjoint i64 %arg21, %arg21
- %i1685 = or disjoint i64 %arg21, %arg21
- %i1686 = or disjoint i64 %arg21, %arg21
- %i1687 = or disjoint i64 %arg21, %arg21
- %i1688 = or disjoint i64 %arg21, %arg21
- %i1689 = or disjoint i64 %arg21, %arg21
- %i1690 = or disjoint i64 0, %arg21
- %i1691 = or disjoint i64 %arg21, %arg21
- %i1692 = or disjoint i64 %arg21, %arg21
- %i1693 = or disjoint i64 1, %arg21
- %i1694 = or disjoint i64 1, %arg21
- %i1695 = or disjoint i64 1, %arg21
- %i1696 = or disjoint i64 1, %arg21
- %i1697 = or disjoint i64 1, %arg21
- %i1698 = or disjoint i64 1, %arg21
- %i1699 = or disjoint i64 1, %arg21
- %i1700 = or disjoint i64 1, %arg21
- %i1701 = or disjoint i64 1, %arg21
- %i1702 = or disjoint i64 1, %arg21
- %i1703 = or disjoint i64 1, %arg21
- %i1704 = or disjoint i64 0, %arg21
- %i1705 = or disjoint i64 %arg21, %arg21
- %i1706 = or disjoint i64 0, %arg21
- %i1707 = or disjoint i64 %arg21, %arg21
- %i1708 = or disjoint i64 1, %arg21
- %i1709 = or disjoint i64 %arg21, 0
- %i1710 = or disjoint i64 0, %arg21
- %i1711 = or disjoint i64 1, %arg21
- %i1712 = or disjoint i64 %arg21, %arg21
- %i1713 = or disjoint i64 %arg21, 1
- %i1714 = or disjoint i64 1, %arg21
- %i1715 = or disjoint i64 1, %arg21
- %i1716 = or disjoint i64 %arg21, 1
- %i1717 = or disjoint i64 1, %arg21
- %i1718 = or disjoint i64 1, %arg21
- %i1719 = or disjoint i64 1, %arg21
- %i1720 = or disjoint i64 1, %arg21
- %i1721 = or disjoint i64 1, %arg21
- %i1722 = or disjoint i64 0, %arg21
- %i1723 = or disjoint i64 0, %arg21
- %i1724 = or disjoint i64 1, %arg21
- %i1725 = or disjoint i64 1, %arg21
- %i1726 = or disjoint i64 0, %arg21
- %i1727 = or disjoint i64 0, 0
- %i1728 = or disjoint i64 1, %arg21
- %i1729 = or disjoint i64 0, %arg21
- %i1730 = or disjoint i64 0, %arg21
- %i1731 = or disjoint i64 0, %arg21
- %i1732 = or disjoint i64 0, %arg21
- %i1733 = or disjoint i64 0, %arg21
- %i1734 = or disjoint i64 %arg21, %arg21
- %i1735 = zext i32 %arg4 to i64
- %i1736 = mul nsw i64 %arg21, %arg21
- %i1737 = mul nuw nsw i64 %arg21, %arg21
- %i1738 = mul nuw nsw i64 %arg21, %arg21
- %i1739 = mul nuw nsw i64 %arg21, %arg21
- %i1740 = mul nuw nsw i64 %arg21, %arg21
- %i1741 = mul nuw nsw i64 %arg21, %arg21
- %i1742 = mul nuw nsw i64 %arg21, %arg21
- %i1743 = mul nuw nsw i64 %arg21, %arg21
- %i1744 = mul nuw nsw i64 %arg21, %arg21
- %i1745 = mul nuw nsw i64 %arg21, %arg21
- %i1746 = mul nuw nsw i64 %arg21, %arg21
- %i1747 = mul nuw nsw i64 %arg21, %arg21
- %i1748 = mul nuw nsw i64 %arg21, %arg21
- %i1749 = mul nuw nsw i64 %arg21, %arg21
- %i1750 = mul nuw nsw i64 %arg21, %arg21
- %i1751 = mul nuw nsw i64 %arg21, %arg21
- %i1752 = mul nuw nsw i64 %arg21, %arg21
- %i1753 = mul nuw nsw i64 %arg21, %arg21
- %i1754 = mul nuw nsw i64 0, 0
- %i1755 = mul nuw nsw i64 0, 0
- %i1756 = mul nuw nsw i64 %arg21, %arg21
- %i1757 = mul nuw nsw i64 %arg21, 1
- %i1758 = mul nuw nsw i64 0, 0
- %i1759 = mul nuw nsw i64 %arg21, %arg21
- %i1760 = mul nuw nsw i64 %arg21, 1
- %i1761 = mul nuw nsw i64 %arg21, %arg21
- %i1762 = mul nuw nsw i64 %arg21, %arg21
- %i1763 = mul nuw nsw i64 %arg21, %arg21
- %i1764 = mul nuw nsw i64 %arg21, %arg21
- %i1765 = mul nuw nsw i64 %arg21, %arg21
- %i1766 = mul nuw nsw i64 %arg21, %arg21
- %i1767 = mul nuw nsw i64 %arg21, %arg21
- %i1768 = mul nuw nsw i64 %arg21, %arg21
- %i1769 = mul nuw nsw i64 %arg21, %arg21
- %i1770 = mul nuw nsw i64 %arg21, %arg21
- %i1771 = mul nuw nsw i64 %arg21, %arg21
- %i1772 = mul nuw nsw i64 %arg21, %arg21
- %i1773 = mul nuw nsw i64 %arg21, %arg21
- %i1774 = mul nuw nsw i64 %arg21, %arg21
- %i1775 = mul nuw nsw i64 %arg21, %arg21
- %i1776 = mul nuw nsw i64 %arg21, %arg21
- %i1777 = mul nuw nsw i64 %arg21, %arg21
- %i1778 = mul nuw nsw i64 %arg21, %arg21
- %i1779 = mul nuw nsw i64 %arg21, %arg21
- %i1780 = mul nuw nsw i64 %arg21, %arg21
- %i1781 = mul nuw nsw i64 %arg21, %arg21
- %i1782 = mul nuw nsw i64 %arg21, %arg21
- %i1783 = mul nuw nsw i64 %arg21, 1
- %i1784 = mul nuw nsw i64 %arg21, %arg21
- %i1785 = mul nuw nsw i64 %arg21, %arg21
- %i1786 = mul nuw nsw i64 %arg21, %arg21
- %i1787 = mul nuw nsw i64 %arg21, 1
- %i1788 = mul nuw nsw i64 %arg21, %arg21
- %i1789 = mul nuw nsw i64 %arg21, %arg21
- %i1790 = mul nuw nsw i64 %arg21, %arg21
- %i1791 = mul nuw nsw i64 %arg21, 1
- %i1792 = mul nuw nsw i64 1, %arg21
- %i1793 = mul nuw nsw i64 1, %arg21
- %i1794 = mul nuw nsw i64 %arg21, %arg21
- %i1795 = mul nuw nsw i64 %arg21, %arg21
- %i1796 = mul nuw nsw i64 0, 0
- %i1797 = mul nuw nsw i64 %arg21, %arg21
- %i1798 = mul nuw nsw i64 %arg21, %arg21
- %i1799 = mul nuw nsw i64 %arg21, %arg21
- %i1800 = mul nuw nsw i64 %arg21, %arg21
- %i1801 = mul nuw nsw i64 %arg21, %arg21
- %i1802 = mul nuw nsw i64 %arg21, %arg21
- %i1803 = mul nuw nsw i64 %arg21, %arg21
- %i1804 = mul nuw nsw i64 %arg21, %arg21
- %i1805 = mul nuw nsw i64 %arg21, %arg21
- %i1806 = mul nuw nsw i64 %arg21, %arg21
- %i1807 = mul nuw nsw i64 %arg21, %arg21
- %i1808 = mul nuw nsw i64 %arg21, %arg21
- %i1809 = mul nuw nsw i64 %arg21, %arg21
- %i1810 = mul nuw nsw i64 %arg21, %arg21
- %i1811 = mul nuw nsw i64 %arg21, %arg21
- %i1812 = mul nuw nsw i64 %arg21, %arg21
- %i1813 = mul nuw nsw i64 %arg21, %arg21
- %i1814 = mul nuw nsw i64 %arg21, %arg21
- %i1815 = mul nuw nsw i64 %arg21, %arg21
- %i1816 = mul nuw nsw i64 %arg21, %arg21
- %i1817 = mul nuw nsw i64 %arg21, %arg21
- %i1818 = mul nuw nsw i64 %arg21, %arg21
- %i1819 = mul nuw nsw i64 %arg21, %arg21
- %i1820 = mul nuw nsw i64 %arg21, %arg21
- %i1821 = mul nuw nsw i64 %arg21, %arg21
- %i1822 = mul nuw nsw i64 %arg21, %arg21
- %i1823 = mul nuw nsw i64 %arg21, 1
- %i1824 = mul nuw nsw i64 %arg21, %arg21
- %i1825 = mul nuw nsw i64 %arg21, %arg21
- %i1826 = mul nuw nsw i64 %arg21, %arg21
- %i1827 = mul nuw nsw i64 %arg21, %arg21
- %i1828 = mul nuw nsw i64 %arg21, %arg21
- %i1829 = mul nuw nsw i64 %arg21, %arg21
- %i1830 = mul nuw nsw i64 %arg21, %arg21
- %i1831 = mul nuw nsw i64 %arg21, %arg21
- %i1832 = mul nuw nsw i64 %arg21, %arg21
- %i1833 = mul nuw nsw i64 %arg21, 1
- %i1834 = mul nuw nsw i64 %arg21, %arg21
- %i1835 = mul nuw nsw i64 %arg21, %arg21
- %i1836 = mul nuw nsw i64 %arg21, %arg21
- %i1837 = mul nuw nsw i64 %arg21, %arg21
- %i1838 = mul nuw nsw i64 %arg21, %arg21
- %i1839 = mul nuw nsw i64 %arg21, %arg21
- %i1840 = mul nuw nsw i64 %arg21, %arg21
- %i1841 = mul nuw nsw i64 %arg21, %arg21
- %i1842 = or disjoint i64 %arg21, %arg21
- %i1843 = add i64 %arg21, %arg21
- %i1844 = add i64 %arg21, %arg21
- %i1845 = add i64 %arg21, %arg21
- %i1846 = add i64 %arg21, %arg21
- %i1847 = add i64 %arg21, %arg21
- %i1848 = add i64 0, %arg21
- %i1849 = add i64 %arg21, %arg21
- %i1850 = add i64 %arg21, %arg21
- %i1851 = add i64 %arg21, %arg21
- %i1852 = add i64 %arg21, %arg21
- %i1853 = add i64 %arg21, %arg21
- %i1854 = add i64 %arg21, %arg21
- %i1855 = add i64 0, %arg21
- %i1856 = add i64 0, %arg21
- %i1857 = add i64 %arg21, %arg21
- %i1858 = add i64 %arg21, %arg21
- %i1859 = add i64 0, %arg21
- %i1860 = add i64 0, 0
- %i1861 = add i64 0, 0
- %i1862 = add i64 %arg21, %arg21
- %i1863 = add i64 0, %arg21
- %i1864 = add i64 %arg21, 0
- %i1865 = add i64 0, %arg21
- %i1866 = add i64 0, %arg21
- %i1867 = add i64 0, %arg21
- %i1868 = add i64 0, %arg21
- %i1869 = add i64 0, %arg21
- %i1870 = add i64 0, %arg21
- %i1871 = add i64 0, %arg21
- %i1872 = add i64 0, %arg21
- %i1873 = add i64 0, %arg21
- %i1874 = add i64 0, %arg21
- %i1875 = add i64 0, %arg21
- %i1876 = add i64 0, %arg21
- %i1877 = add i64 0, %arg21
- %i1878 = add i64 0, %arg21
- %i1879 = add i64 0, %arg21
- %i1880 = add i64 0, %arg21
- %i1881 = add i64 0, %arg21
- %i1882 = add i64 0, %arg21
- %i1883 = add i64 0, %arg21
- %i1884 = add i64 0, %arg21
- %i1885 = add i64 0, %arg21
- %i1886 = add i64 0, %arg21
- %i1887 = add i64 0, %arg21
- %i1888 = add i64 0, %arg21
- %i1889 = add i64 0, %arg21
- %i1890 = add i64 0, %arg21
- %i1891 = add i64 0, %arg21
- %i1892 = add i64 0, %arg21
- %i1893 = add i64 0, %arg21
- %i1894 = add i64 0, %arg21
- %i1895 = add i64 0, %arg21
- %i1896 = add i64 0, %arg21
- %i1897 = add i64 0, %arg21
- %i1898 = add i64 0, %arg21
- %i1899 = add i64 %arg21, %arg21
- %i1900 = add i64 %arg21, %arg21
- %i1901 = add i64 %arg21, %arg21
- %i1902 = add i64 0, 0
- %i1903 = add i64 1, %arg21
- %i1904 = add i64 1, %arg21
- %i1905 = add i64 %arg21, %arg21
- %i1906 = add i64 %arg21, %arg21
- %i1907 = add i64 %arg21, %arg21
- %i1908 = add i64 %arg21, %arg21
- %i1909 = add i64 %arg21, %arg21
- %i1910 = add i64 %arg21, %arg21
- %i1911 = add i64 %arg21, %arg21
- %i1912 = add i64 %arg21, %arg21
- %i1913 = add i64 %arg21, %arg21
- %i1914 = add i64 %arg21, %arg21
- %i1915 = add i64 %arg21, %arg21
- %i1916 = add i64 %arg21, %arg21
- %i1917 = add i64 %arg21, %arg21
- %i1918 = add i64 %arg21, %arg21
- %i1919 = add i64 %arg21, %arg21
- %i1920 = add i64 %arg21, %arg21
- %i1921 = add i64 %arg21, %arg21
- %i1922 = add i64 %arg21, %arg21
- %i1923 = add i64 %arg21, %arg21
- %i1924 = add i64 %arg21, %arg21
- %i1925 = add i64 %arg21, %arg21
- %i1926 = add i64 1, %arg21
- %i1927 = add i64 0, %arg21
- %i1928 = add i64 0, %arg21
- %i1929 = add i64 0, %arg21
- %i1930 = add i64 %arg21, %arg21
- %i1931 = add i64 %arg21, %arg21
- %i1932 = add i64 %arg21, %arg21
- %i1933 = add i64 %arg21, %arg21
- %i1934 = add i64 %arg21, %arg21
- %i1935 = add i64 %arg21, %arg21
- %i1936 = add i64 0, %arg21
- %i1937 = add i64 1, %arg21
- %i1938 = add i64 %arg21, %arg21
- %i1939 = add i64 %arg21, %arg21
- %i1940 = add i64 %arg21, %arg21
- %i1941 = add i64 %arg21, %arg21
- %i1942 = add i64 %arg21, %arg21
- %i1943 = add i64 0, %arg21
- %i1944 = add i64 0, %arg21
- %i1945 = add i64 0, %arg21
- %i1946 = add i64 0, %arg21
- %i1947 = add i64 0, %arg21
- %i1948 = trunc i64 %arg365 to i32
- %i1949 = trunc i64 %arg366 to i32
- %i1950 = trunc i64 %arg367 to i32
- %i1951 = trunc i64 %arg368 to i32
- %i1952 = trunc i64 %arg369 to i32
- %i1953 = trunc i64 %arg370 to i32
- %i1954 = trunc i64 %arg371 to i32
- %i1955 = trunc i64 %arg372 to i32
- %i1956 = trunc i64 %arg373 to i32
- %i1957 = trunc i64 %arg374 to i32
- %i1958 = trunc i64 %arg155 to i32
- %i1959 = trunc i64 %arg21 to i32
- %i1960 = trunc i64 %arg21 to i32
- %i1961 = trunc i64 %arg21 to i32
- %i1962 = trunc i64 %arg21 to i32
- %i1963 = trunc i64 %arg21 to i32
- %i1964 = trunc i64 %arg21 to i32
- %i1965 = trunc i64 0 to i32
- %i1966 = trunc i64 0 to i32
- %i1967 = trunc i64 %arg21 to i32
- %i1968 = trunc i64 %arg21 to i32
- %i1969 = trunc i64 %arg21 to i32
- %i1970 = trunc i64 %arg21 to i32
- %i1971 = trunc i64 %arg21 to i32
- %i1972 = trunc i64 %arg387 to i32
- %i1973 = trunc i64 %arg388 to i32
- %i1974 = trunc i64 %arg389 to i32
- %i1975 = trunc i64 %arg390 to i32
- %i1976 = trunc i64 %arg23 to i32
- %i1977 = trunc i64 %arg21 to i32
- %i1978 = trunc i64 %arg21 to i32
- %i1979 = trunc i64 %arg394 to i32
- %i1980 = trunc i64 %arg395 to i32
- %i1981 = trunc i64 %arg396 to i32
- %i1982 = trunc i64 %arg397 to i32
- %i1983 = trunc i64 %arg30 to i32
- %i1984 = trunc i64 %arg21 to i32
- %i1985 = trunc i64 %arg21 to i32
- %i1986 = trunc i64 %arg21 to i32
- %i1987 = trunc i64 %arg21 to i32
- %i1988 = trunc i64 %arg21 to i32
- %i1989 = trunc i64 %arg21 to i32
- %i1990 = trunc i64 %arg21 to i32
- %i1991 = trunc i64 %arg21 to i32
- %i1992 = trunc i64 %arg21 to i32
- %i1993 = trunc i64 %arg21 to i32
- %i1994 = trunc i64 %arg21 to i32
- %i1995 = trunc i64 %arg21 to i32
- %i1996 = trunc i64 %arg411 to i32
- %i1997 = trunc i64 %arg410 to i32
- %i1998 = trunc i64 %arg21 to i32
- %i1999 = trunc i64 %arg21 to i32
- %i2000 = trunc i64 %arg21 to i32
- %i2001 = trunc i64 %arg21 to i32
- %i2002 = trunc i64 %arg21 to i32
- %i2003 = trunc i64 %arg21 to i32
- %i2004 = trunc i64 %arg21 to i32
- %i2005 = trunc i64 %arg21 to i32
- %i2006 = trunc i64 %arg21 to i32
- %i2007 = trunc i64 %arg21 to i32
- %i2008 = trunc i64 0 to i32
- %i2009 = trunc i64 %arg21 to i32
- %i2010 = trunc i64 %arg21 to i32
- %i2011 = trunc i64 %arg424 to i32
- %i2012 = trunc i64 %arg425 to i32
- %i2013 = trunc i64 %arg426 to i32
- %i2014 = trunc i64 %arg427 to i32
- %i2015 = trunc i64 %arg428 to i32
- %i2016 = trunc i64 %arg429 to i32
- %i2017 = trunc i64 %arg430 to i32
- %i2018 = trunc i64 %arg431 to i32
- %i2019 = trunc i64 %arg432 to i32
- %i2020 = trunc i64 %arg433 to i32
- %i2021 = trunc i64 %arg434 to i32
- %i2022 = trunc i64 %arg435 to i32
- %i2023 = trunc i64 %arg436 to i32
- %i2024 = trunc i64 %arg437 to i32
- %i2025 = trunc i64 %arg438 to i32
- %i2026 = trunc i64 %arg439 to i32
- %i2027 = trunc i64 %arg179 to i32
- %i2028 = trunc i64 %arg21 to i32
- %i2029 = trunc i64 %arg21 to i32
- %i2030 = trunc i64 %arg21 to i32
- %i2031 = trunc i64 %arg21 to i32
- %i2032 = trunc i64 %arg21 to i32
- %i2033 = trunc i64 %arg21 to i32
- %i2034 = trunc i64 %arg21 to i32
- %i2035 = trunc i64 %arg21 to i32
- %i2036 = trunc i64 %arg21 to i32
- %i2037 = trunc i64 %arg21 to i32
- %i2038 = trunc i64 %arg21 to i32
- %i2039 = trunc i64 %arg21 to i32
- %i2040 = trunc i64 %arg21 to i32
- %i2041 = trunc i64 %arg21 to i32
- %i2042 = trunc i64 %arg455 to i32
- %i2043 = trunc i64 %arg456 to i32
- %i2044 = trunc i64 %arg457 to i32
- %i2045 = trunc i64 %arg458 to i32
- %i2046 = trunc i64 %arg459 to i32
- %i2047 = trunc i64 %arg460 to i32
- %i2048 = trunc i64 %arg461 to i32
- %i2049 = trunc i64 %arg462 to i32
- %i2050 = trunc i64 %arg180 to i32
- %i2051 = trunc i64 %arg21 to i32
- %i2052 = trunc i64 %arg21 to i32
- %i2053 = trunc i64 %arg21 to i32
- %i2054 = sext i32 %arg4 to i64
- %i2055 = icmp slt i64 %arg21, %arg468
- %i2056 = icmp slt i64 %arg469, %arg234
- %i2057 = icmp slt i64 %arg470, %arg172
- %i2058 = icmp slt i64 %arg471, %arg160
- %i2059 = icmp slt i64 %arg472, %arg35
- %i2060 = icmp slt i64 %arg473, %arg232
- %i2061 = icmp slt i64 %arg474, %arg166
- %i2062 = icmp slt i64 %arg475, %arg199
- %i2063 = icmp slt i64 %arg21, %arg21
- %i2064 = icmp slt i64 %arg21, %arg21
- %i2065 = icmp slt i64 %arg21, %arg21
- %i2066 = icmp slt i64 %arg21, %arg21
- %i2067 = icmp slt i64 %arg21, %arg21
- %i2068 = icmp slt i64 %arg21, %arg21
- %i2069 = icmp slt i64 %arg21, %arg21
- %i2070 = icmp slt i64 %arg21, %arg21
- %i2071 = icmp slt i64 %arg21, %arg21
- %i2072 = icmp slt i64 %arg21, %arg21
- %i2073 = icmp slt i64 %arg21, %arg21
- %i2074 = icmp slt i64 %arg21, %arg21
- %i2075 = icmp slt i64 %arg21, %arg21
- %i2076 = icmp slt i64 %arg21, %arg21
- %i2077 = icmp slt i64 %arg21, %arg21
- %i2078 = icmp slt i64 %arg21, %arg21
- %i2079 = icmp slt i64 %arg21, %arg21
- %i2080 = icmp slt i64 %arg21, %arg21
- %i2081 = icmp slt i64 %arg149, %arg21
- %i2082 = icmp slt i64 %arg164, %arg21
- %i2083 = icmp slt i64 %arg177, %arg21
- %i2084 = icmp slt i64 %arg189, %arg21
- %i2085 = icmp slt i64 %arg151, %arg21
- %i2086 = icmp slt i64 %arg21, %arg468
- %i2087 = icmp slt i64 %arg171, %arg21
- %i2088 = icmp slt i64 %arg501, %arg468
- %i2089 = icmp slt i64 %arg502, %arg468
- %i2090 = icmp slt i64 %arg503, %arg468
- %i2091 = icmp slt i64 %arg504, %arg233
- %i2092 = icmp slt i64 %arg505, %arg21
- %i2093 = icmp slt i64 %arg506, %arg239
- %i2094 = icmp slt i64 %arg507, %arg468
- %i2095 = icmp slt i64 %arg508, %arg28
- %i2096 = icmp slt i64 %arg212, %arg21
- %i2097 = icmp slt i64 %arg211, %arg26
- %i2098 = icmp slt i64 %arg21, %arg21
- %i2099 = icmp slt i64 %arg512, %arg184
- %i2100 = icmp slt i64 %arg513, %arg179
- %i2101 = icmp slt i64 %arg514, %arg224
- %i2102 = icmp slt i64 %arg515, %arg468
- %i2103 = icmp slt i64 %arg516, %arg468
- %i2104 = icmp slt i64 %arg517, %arg468
- %i2105 = icmp slt i64 %arg518, %arg468
- %i2106 = icmp slt i64 %arg519, %arg468
- %i2107 = icmp slt i64 %arg520, %arg170
- %i2108 = icmp slt i64 %arg521, %arg158
- %i2109 = icmp slt i64 %arg522, %arg194
- %i2110 = icmp slt i64 %arg523, %arg468
- %i2111 = icmp slt i64 %arg524, %arg468
- %i2112 = icmp slt i64 %arg525, %arg468
- %i2113 = icmp slt i64 %arg526, %arg468
- %i2114 = icmp slt i64 %arg527, %arg468
- %i2115 = icmp slt i64 %arg528, %arg30
- %i2116 = icmp slt i64 0, %arg468
- %i2117 = icmp slt i64 1, %arg184
- %i2118 = icmp slt i64 %arg529, %arg468
- %i2119 = icmp slt i64 0, 0
- %i2120 = icmp slt i64 %arg530, %arg192
- %i2121 = icmp slt i64 %arg531, %arg170
- %i2122 = icmp slt i64 %arg532, %arg35
- %i2123 = icmp slt i64 %arg533, %arg223
- %i2124 = icmp slt i64 %arg534, %arg147
- %i2125 = icmp slt i64 %arg535, %arg170
- %i2126 = icmp slt i64 %arg536, %arg30
- %i2127 = icmp slt i64 %arg537, %arg468
- %i2128 = icmp slt i64 %arg538, %arg185
- %i2129 = icmp slt i64 %arg539, %arg205
- %i2130 = icmp slt i64 %arg540, %arg224
- %i2131 = icmp slt i64 %arg541, %arg468
- %i2132 = icmp slt i64 %arg542, %arg468
- %i2133 = icmp slt i64 %arg543, %arg468
- %i2134 = icmp slt i64 %arg544, %arg468
- %i2135 = icmp slt i64 %arg545, %arg468
- %i2136 = icmp slt i64 %arg546, %arg166
- %i2137 = icmp slt i64 %arg547, %arg228
- %i2138 = icmp slt i64 %arg548, %arg468
- %i2139 = icmp slt i64 %arg549, %arg468
- %i2140 = icmp slt i64 %arg550, %arg468
- %i2141 = icmp slt i64 %arg551, %arg468
- %i2142 = icmp slt i64 %arg552, %arg468
- %i2143 = icmp slt i64 %arg553, %arg468
- %i2144 = icmp slt i64 %arg554, %arg182
- %i2145 = icmp slt i64 %arg555, %arg21
- %i2146 = icmp slt i64 %arg556, %arg26
- %i2147 = icmp slt i64 %arg557, %arg27
- %i2148 = icmp slt i64 %arg558, %arg28
- %i2149 = icmp slt i64 %arg559, %arg28
- %i2150 = icmp slt i64 %arg162, 1
- %i2151 = icmp slt i64 %arg21, %arg21
- %i2152 = icmp slt i64 %arg21, %arg21
- %i2153 = icmp slt i64 %arg21, %arg21
- %i2154 = icmp slt i64 %arg564, %arg156
- %i2155 = icmp slt i64 %arg565, %arg250
- %i2156 = icmp slt i64 %arg566, %arg30
- %i2157 = icmp slt i64 %arg567, %arg185
- %i2158 = icmp slt i64 %arg568, %arg227
- %i2159 = icmp slt i64 %arg569, %arg468
- %i2160 = icmp slt i64 %arg570, %arg154
- %i2161 = icmp slt i64 %arg571, %arg192
- %i2162 = icmp slt i64 %arg572, %arg179
- %i2163 = icmp slt i64 %arg573, %arg468
- %i2164 = icmp slt i64 0, %arg468
- %i2165 = icmp slt i64 %arg574, %arg468
- %i2166 = icmp slt i64 %arg575, %arg468
- %i2167 = icmp slt i64 %arg174, %arg468
- %i2168 = icmp slt i64 %arg157, %arg21
- %i2169 = icmp slt i64 %arg21, %arg21
- %i2170 = icmp slt i64 %arg21, %arg21
- %i2171 = and i1 %arg580, %arg581
- %i2172 = and i1 %arg580, %arg582
- %i2173 = and i1 %arg580, %arg583
- %i2174 = and i1 %arg580, %arg584
- %i2175 = and i1 %arg580, %arg585
- %i2176 = and i1 %arg580, %arg586
- %i2177 = and i1 %arg580, %arg587
- %i2178 = and i1 %arg580, %arg588
- %i2179 = and i1 %arg580, %arg589
- %i2180 = and i1 %arg580, %arg590
- %i2181 = and i1 %arg580, %arg591
- %i2182 = and i1 %arg580, %arg592
- %i2183 = and i1 %arg580, %arg593
- %i2184 = and i1 %arg580, %arg594
- %i2185 = and i1 %arg580, %arg595
- %i2186 = and i1 %arg580, %arg596
- %i2187 = and i1 %arg580, %arg597
- %i2188 = and i1 %arg580, %arg598
- %i2189 = and i1 %arg580, %arg599
- %i2190 = and i1 %arg580, %arg600
- %i2191 = and i1 %arg580, %arg601
- %i2192 = and i1 %arg580, %arg602
- %i2193 = and i1 %arg580, %arg603
- %i2194 = and i1 %arg580, %arg604
- %i2195 = and i1 %arg580, %arg605
- %i2196 = and i1 %arg580, %arg606
- %i2197 = and i1 %arg580, %arg607
- %i2198 = and i1 %arg580, %arg608
- %i2199 = and i1 %arg580, %arg609
- %i2200 = and i1 %arg580, %arg610
- %i2201 = and i1 %i1501, %i2088
- %i2202 = and i1 %i1501, %i2089
- %i2203 = and i1 %i1501, %i2090
- %i2204 = and i1 %arg580, %arg608
- %i2205 = and i1 %arg580, %arg580
- %i2206 = and i1 %arg580, %arg580
- %i2207 = and i1 %i1501, %i2094
- %i2208 = and i1 %arg580, %i2095
- %i2209 = and i1 %arg1028, %arg580
- %i2210 = and i1 %arg580, %arg580
- %i2211 = and i1 %arg580, %arg580
- %i2212 = and i1 %arg580, %arg580
- %i2213 = and i1 %arg580, %arg580
- %i2214 = and i1 %arg580, %arg580
- %i2215 = and i1 %i1501, %i2102
- %i2216 = and i1 %i1501, %i2103
- %i2217 = and i1 %i1501, %i2104
- %i2218 = and i1 %i1501, %i2105
- %i2219 = and i1 %i1501, %i2106
- %i2220 = and i1 %arg580, %arg587
- %i2221 = and i1 %arg580, %arg580
- %i2222 = and i1 %arg580, %arg580
- %i2223 = and i1 %i1501, %i2110
- %i2224 = and i1 %i1501, %i2111
- %i2225 = and i1 %i1501, %i2112
- %i2226 = and i1 %i1501, %i2113
- %i2227 = and i1 %i1501, %i2114
- %i2228 = and i1 %arg580, %i2115
- %i2229 = and i1 %i1501, %i2116
- %i2230 = and i1 %arg580, %arg620
- %i2231 = and i1 %i1501, %i2118
- %i2232 = and i1 false, false
- %i2233 = and i1 %i1501, %i2127
- %i2234 = and i1 %arg580, %arg609
- %i2235 = and i1 %arg580, %arg580
- %i2236 = and i1 %arg580, %arg580
- %i2237 = and i1 %i1501, %i2131
- %i2238 = and i1 %i1501, %i2132
- %i2239 = and i1 %i1501, %i2133
- %i2240 = and i1 %i1501, %i2134
- %i2241 = and i1 %i1501, %i2135
- %i2242 = and i1 %arg580, %arg616
- %i2243 = and i1 %arg580, %arg617
- %i2244 = and i1 %arg580, %i2138
- %i2245 = and i1 %i1501, %i2139
- %i2246 = and i1 %i1501, %i2140
- %i2247 = and i1 %i1501, %i2141
- %i2248 = and i1 %i1501, %i2142
- %i2249 = and i1 %i1501, %i2143
- %i2250 = and i1 %arg580, %arg597
- %i2251 = and i1 %arg580, %arg580
- %i2252 = and i1 %arg580, %arg580
- %i2253 = and i1 true, %arg580
- %i2254 = and i1 %arg580, %arg580
- %i2255 = and i1 false, false
- %i2256 = and i1 %arg580, %arg580
- %i2257 = and i1 true, %arg580
- %i2258 = and i1 %arg580, %arg580
- %i2259 = and i1 true, %arg580
- %i2260 = and i1 %arg580, %arg580
- %i2261 = and i1 %arg580, %arg580
- %i2262 = and i1 %arg580, %arg580
- %i2263 = and i1 %arg580, %arg580
- %i2264 = and i1 %arg580, %arg630
- %i2265 = and i1 %i1501, %i2159
- %i2266 = and i1 %arg580, %arg591
- %i2267 = and i1 %arg580, %arg580
- %i2268 = and i1 %arg580, %arg580
- %i2269 = and i1 %i1501, %i2163
- %i2270 = and i1 %i1501, %i2164
- %i2271 = and i1 %i1501, %i2165
- %i2272 = and i1 %i1501, %i2166
- %i2273 = and i1 %i1501, %i2167
- %i2274 = and i1 %arg580, %arg604
- %i2275 = and i1 %arg580, %arg580
- %i2276 = and i1 %arg580, %arg580
- %i2277 = and i1 %i1498, %i2171
- %i2278 = and i1 %i1498, %i2172
- %i2279 = and i1 %i1498, %arg634
- %i2280 = and i1 %arg635, %i2174
- %i2281 = and i1 %i1498, %i2175
- %i2282 = and i1 %i1498, %i2176
- %i2283 = and i1 %arg595, %arg580
- %i2284 = and i1 %arg580, %arg580
- %i2285 = and i1 %i1498, %i2179
- %i2286 = and i1 %i1498, %i2180
- %i2287 = and i1 %i1498, %i2181
- %i2288 = and i1 %i1498, %i2182
- %i2289 = and i1 %i1498, %i2183
- %i2290 = and i1 %arg601, %arg580
- %i2291 = and i1 %arg580, %arg580
- %i2292 = and i1 %arg580, %arg580
- %i2293 = and i1 %i1498, %i2187
- %i2294 = and i1 %i1498, %i2188
- %i2295 = and i1 %arg611, %arg580
- %i2296 = and i1 %arg580, %arg580
- %i2297 = and i1 %arg580, %arg580
- %i2298 = and i1 %i1498, %i2192
- %i2299 = and i1 %i1498, %i2193
- %i2300 = and i1 %i1498, %i2194
- %i2301 = and i1 %i1498, %i2195
- %i2302 = and i1 %arg613, %arg580
- %i2303 = and i1 %arg580, %arg580
- %i2304 = and i1 %arg580, %arg580
- %i2305 = and i1 %i1498, %i2199
- %i2306 = and i1 %i1498, %i2200
- %i2307 = and i1 %i1498, %i2201
- %i2308 = and i1 %i1498, %i2202
- %i2309 = and i1 %i1498, %i2203
- %i2310 = and i1 %arg627, %arg580
- %i2311 = and i1 %arg580, %arg580
- %i2312 = and i1 %arg580, %arg580
- %i2313 = and i1 %i1498, %i2207
- %i2314 = and i1 %arg584, %i2208
- %i2315 = and i1 %i1498, %i2209
- %i2316 = and i1 %arg589, %arg580
- %i2317 = and i1 %arg580, %arg580
- %i2318 = and i1 %arg580, %arg580
- %i2319 = and i1 %arg580, %arg580
- %i2320 = and i1 %arg580, %arg580
- %i2321 = and i1 %i1498, %i2215
- %i2322 = and i1 %i1498, %i2216
- %i2323 = and i1 %i1498, %i2217
- %i2324 = and i1 %i1498, %i2218
- %i2325 = and i1 %i1498, %i2219
- %i2326 = and i1 %arg625, %arg580
- %i2327 = and i1 %arg580, %arg580
- %i2328 = and i1 %arg580, %arg580
- %i2329 = and i1 %i1498, %i2223
- %i2330 = and i1 %i1498, %i2224
- %i2331 = and i1 %i1498, %i2225
- %i2332 = and i1 %i1498, %i2226
- %i2333 = and i1 %i1498, %i2227
- %i2334 = and i1 %arg635, %i2228
- %i2335 = and i1 %arg584, %i2229
- %i2336 = and i1 %arg616, %arg580
- %i2337 = and i1 %i1498, %i2231
- %i2338 = and i1 false, false
- %i2339 = and i1 %i1498, %i2233
- %i2340 = and i1 %arg630, %arg580
- %i2341 = and i1 %arg580, %arg580
- %i2342 = and i1 %arg580, %arg580
- %i2343 = and i1 %i1498, %i2237
- %i2344 = and i1 %i1498, %i2238
- %i2345 = and i1 %i1498, %i2239
- %i2346 = and i1 %i1498, %i2240
- %i2347 = and i1 %i1498, %i2241
- %i2348 = and i1 %arg595, %arg580
- %i2349 = and i1 %arg580, %arg580
- %i2350 = and i1 %arg635, %i2244
- %i2351 = and i1 %i1498, %i2245
- %i2352 = and i1 %i1498, %i2246
- %i2353 = and i1 %i1498, %i2247
- %i2354 = and i1 %i1498, %i2248
- %i2355 = and i1 %i1498, %i2249
- %i2356 = and i1 %arg621, %arg580
- %i2357 = and i1 true, %arg580
- %i2358 = and i1 %arg580, %arg580
- %i2359 = and i1 %arg580, %arg580
- %i2360 = and i1 %arg580, %arg580
- %i2361 = and i1 false, false
- %i2362 = and i1 %arg580, %arg580
- %i2363 = and i1 %arg580, %arg580
- %i2364 = and i1 %arg580, %arg580
- %i2365 = and i1 %arg580, %arg580
- %i2366 = and i1 %arg580, %arg580
- %i2367 = and i1 %arg580, %arg580
- %i2368 = and i1 %arg580, %arg580
- %i2369 = and i1 %arg580, %arg580
- %i2370 = and i1 %i1498, %i2264
- %i2371 = and i1 %i1498, %i2265
- %i2372 = and i1 %arg602, %arg580
- %i2373 = and i1 %arg580, %arg580
- %i2374 = and i1 %arg580, %arg580
- %i2375 = and i1 %i1498, %i2269
- %i2376 = and i1 %i1498, %i2270
- %i2377 = and i1 %i1498, %i2271
- %i2378 = and i1 %i1498, %i2272
- %i2379 = and i1 %i1498, %i2273
- %i2380 = and i1 %arg613, %arg580
- %i2381 = and i1 %arg580, %arg580
- %i2382 = and i1 %arg580, %arg580
- %i2383 = tail call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1(ptr addrspace(1) %arg1, i16 0, i64 2147483646, i32 159744)
- %i2384 = shl i32 %i1948, 1
- %i2385 = select i1 %i2277, i32 %i2384, i32 0
- %i2386 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2385, i32 0, i32 0)
- %i2387 = shl i32 %i1949, 1
- %i2388 = select i1 %i2278, i32 %i2387, i32 0
- %i2389 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2388, i32 0, i32 0)
- %i2390 = shl i32 %i1950, 1
- %i2391 = select i1 %i2279, i32 %i2390, i32 0
- %i2392 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2391, i32 0, i32 0)
- %i2393 = shl i32 %i1951, 1
- %i2394 = select i1 %i2280, i32 %i2393, i32 0
- %i2395 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2394, i32 0, i32 0)
- %i2396 = shl i32 %i1952, 1
- %i2397 = select i1 %i2281, i32 %i2396, i32 0
- %i2398 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) null, i32 %i2397, i32 0, i32 0)
- %i2399 = shl i32 %i1953, 0
- %i2400 = select i1 %i2282, i32 %i2399, i32 0
- %i2401 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) null, i32 %i2400, i32 0, i32 0)
- %i2402 = shl i32 %i1954, 1
- %i2403 = select i1 %arg615, i32 %i2402, i32 0
- %i2404 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) null, i32 %i2403, i32 0, i32 0)
- %i2405 = shl i32 %i1955, 1
- %i2406 = select i1 %arg599, i32 %i2405, i32 0
- %i2407 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) null, i32 %i2406, i32 0, i32 0)
- %i2408 = shl i32 %i1956, 1
- %i2409 = select i1 %i2285, i32 %i2408, i32 0
- %i2410 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) null, i32 %i2409, i32 0, i32 0)
- %i2411 = shl i32 %i1957, 0
- %i2412 = select i1 %i2286, i32 %i2411, i32 0
- %i2413 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) null, i32 %i2412, i32 0, i32 0)
- %i2414 = shl i32 0, 0
- %i2415 = select i1 %i2287, i32 0, i32 1
- %i2416 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) null, i32 %i2415, i32 0, i32 0)
- %i2417 = shl i32 %arg44, 1
- %i2418 = select i1 %i2288, i32 %arg126, i32 0
- %i2419 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) null, i32 %i2418, i32 0, i32 0)
- %i2420 = shl i32 %arg5, 0
- %i2421 = select i1 %i2289, i32 %arg98, i32 0
- %i2422 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) null, i32 %i2421, i32 0, i32 0)
- %i2423 = shl i32 %arg38, 0
- %i2424 = select i1 %arg580, i32 %arg4, i32 0
- %i2425 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i2426 = shl i32 %arg4, 0
- %i2427 = select i1 %arg580, i32 %arg4, i32 0
- %i2428 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i2429 = select i1 %arg580, i32 0, i32 1
- %i2430 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i2431 = shl i32 %arg4, 0
- %i2432 = select i1 %i2293, i32 %arg105, i32 0
- %i2433 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2432, i32 0, i32 0)
- %i2434 = shl i32 %arg42, 0
- %i2435 = select i1 %i2294, i32 %arg74, i32 0
- %i2436 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2435, i32 0, i32 0)
- %i2437 = shl i32 %arg62, 0
- %i2438 = select i1 %arg580, i32 %arg4, i32 0
- %i2439 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i2440 = shl i32 %arg4, 0
- %i2441 = select i1 %arg580, i32 %arg4, i32 0
- %i2442 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i2443 = shl i32 %arg4, 0
- %i2444 = select i1 %arg580, i32 %arg4, i32 0
- %i2445 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i2446 = shl i32 %i1972, 0
- %i2447 = select i1 %i2298, i32 %i2446, i32 0
- %i2448 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2447, i32 0, i32 0)
- %i2449 = shl i32 %i1973, 0
- %i2450 = select i1 %i2299, i32 %i2449, i32 0
- %i2451 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2450, i32 0, i32 0)
- %i2452 = shl i32 %i1974, 0
- %i2453 = select i1 %i2300, i32 %i2452, i32 0
- %i2454 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2453, i32 0, i32 0)
- %i2455 = shl i32 %i1975, 0
- %i2456 = select i1 %i2301, i32 %i2455, i32 0
- %i2457 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2456, i32 0, i32 0)
- %i2458 = shl i32 %.frozen, 0
- %i2459 = select i1 %arg580, i32 %arg4, i32 0
- %i2460 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i2461 = shl i32 %arg4, 0
- %i2462 = select i1 %arg580, i32 %arg4, i32 0
- %i2463 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i2464 = shl i32 %arg4, 0
- %i2465 = select i1 %arg580, i32 %arg4, i32 0
- %i2466 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i2467 = shl i32 %i1979, 0
- %i2468 = select i1 %i2305, i32 %i2467, i32 0
- %i2469 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2468, i32 0, i32 0)
- %i2470 = shl i32 %i1980, 0
- %i2471 = select i1 %i2306, i32 %i2470, i32 0
- %i2472 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2471, i32 0, i32 0)
- %i2473 = shl i32 %i1981, 0
- %i2474 = select i1 %i2307, i32 %i2473, i32 0
- %i2475 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2474, i32 0, i32 0)
- %i2476 = shl i32 %i1982, 0
- %i2477 = select i1 %i2308, i32 %i2476, i32 0
- %i2478 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2477, i32 0, i32 0)
- %i2479 = shl i32 %arg46, 0
- %i2480 = select i1 %i2309, i32 %arg726, i32 0
- %i2481 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2480, i32 0, i32 0)
- %i2482 = shl i32 %arg62, 0
- %i2483 = select i1 %arg580, i32 %arg4, i32 0
- %i2484 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i2485 = shl i32 %arg4, 0
- %i2486 = select i1 %arg580, i32 %arg4, i32 0
- %i2487 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i2488 = shl i32 %arg4, 0
- %i2489 = select i1 %arg580, i32 %arg4, i32 0
- %i2490 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i2491 = shl i32 %arg4, 0
- %i2492 = select i1 %i2313, i32 %arg700, i32 0
- %i2493 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2492, i32 0, i32 0)
- %i2494 = shl i32 %arg45, 0
- %i2495 = select i1 %i2314, i32 %arg17, i32 0
- %i2496 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i2497 = shl i32 %arg4, 0
- %i2498 = select i1 %i2315, i32 %arg707, i32 0
- %i2499 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2498, i32 0, i32 0)
- %i2500 = shl i32 %arg50, 0
- %i2501 = select i1 %arg580, i32 %arg4, i32 0
- %i2502 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %arg737, i32 0, i32 0)
- %i2503 = shl i32 %arg15, 0
- %i2504 = select i1 %arg580, i32 %arg4, i32 0
- %i2505 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %arg713, i32 0, i32 0)
- %i2506 = shl i32 %arg4, 0
- %i2507 = select i1 %arg580, i32 %arg4, i32 0
- %i2508 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i2509 = shl i32 %arg4, 0
- %i2510 = select i1 %arg580, i32 %arg4, i32 0
- %i2511 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i2512 = shl i32 %arg4, 0
- %i2513 = select i1 %arg580, i32 %arg4, i32 0
- %i2514 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i2515 = shl i32 %arg4, 0
- %i2516 = select i1 %i2321, i32 %arg755, i32 0
- %i2517 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2516, i32 0, i32 0)
- %i2518 = shl i32 %i1996, 0
- %i2519 = select i1 %i2322, i32 %i2518, i32 0
- %i2520 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2519, i32 0, i32 0)
- %i2521 = shl i32 %i1997, 0
- %i2522 = select i1 %i2323, i32 %i2521, i32 0
- %i2523 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2522, i32 0, i32 0)
- %i2524 = shl i32 %arg13, 0
- %i2525 = select i1 %i2324, i32 %arg53, i32 0
- %i2526 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2525, i32 0, i32 0)
- %i2527 = shl i32 %arg39, 0
- %i2528 = select i1 %i2325, i32 %arg790, i32 0
- %i2529 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2528, i32 0, i32 0)
- %i2530 = shl i32 %arg56, 0
- %i2531 = select i1 %arg580, i32 %arg4, i32 0
- %i2532 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i2533 = shl i32 %arg4, 0
- %i2534 = select i1 %arg580, i32 %arg4, i32 0
- %i2535 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i2536 = shl i32 %arg4, 0
- %i2537 = select i1 %arg580, i32 %arg4, i32 0
- %i2538 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i2539 = shl i32 %arg4, 0
- %i2540 = select i1 %i2329, i32 %arg769, i32 0
- %i2541 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2540, i32 0, i32 0)
- %i2542 = select i1 %i2330, i32 0, i32 1
- %i2543 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2542, i32 0, i32 0)
- %i2544 = select i1 %i2331, i32 0, i32 1
- %i2545 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2544, i32 0, i32 0)
- %i2546 = select i1 %i2332, i32 0, i32 1
- %i2547 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2546, i32 0, i32 0)
- %i2548 = select i1 %i2333, i32 0, i32 1
- %i2549 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2548, i32 0, i32 0)
- %i2550 = select i1 %i2334, i32 0, i32 1
- %i2551 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %arg60, i32 0, i32 0)
- %i2552 = select i1 %arg580, i32 0, i32 0
- %i2553 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) null, i32 0, i32 0, i32 0)
- %i2554 = select i1 %arg580, i32 0, i32 1
- %i2555 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i2556 = select i1 %i2337, i32 0, i32 1
- %i2557 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2556, i32 0, i32 0)
- %i2558 = select i1 false, i32 0, i32 0
- %i2559 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) null, i32 0, i32 0, i32 0)
- %i2560 = select i1 %i2339, i32 0, i32 1
- %i2561 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2560, i32 0, i32 0)
- %i2562 = shl i32 %i2011, 1
- %i2563 = select i1 %arg581, i32 %i2562, i32 0
- %i2564 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %i2563, i32 0, i32 0)
- %i2565 = shl i32 %i2012, 1
- %i2566 = select i1 %arg591, i32 %i2565, i32 0
- %i2567 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %i2566, i32 0, i32 0)
- %i2568 = shl i32 %i2013, 1
- %i2569 = select i1 %arg599, i32 %i2568, i32 0
- %i2570 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %i2569, i32 0, i32 0)
- %i2571 = shl i32 %i2014, 1
- %i2572 = select i1 %i2343, i32 %i2571, i32 0
- %i2573 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2572, i32 0, i32 0)
- %i2574 = shl i32 %i2015, 1
- %i2575 = select i1 %i2344, i32 %i2574, i32 0
- %i2576 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2575, i32 0, i32 0)
- %i2577 = shl i32 %i2016, 1
- %i2578 = select i1 %i2345, i32 %i2577, i32 0
- %i2579 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2578, i32 0, i32 0)
- %i2580 = shl i32 %i2017, 1
- %i2581 = select i1 %i2346, i32 %i2580, i32 0
- %i2582 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2581, i32 0, i32 0)
- %i2583 = shl i32 %i2018, 1
- %i2584 = select i1 %i2347, i32 %i2583, i32 0
- %i2585 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2584, i32 0, i32 0)
- %i2586 = shl i32 %i2019, 1
- %i2587 = select i1 %arg591, i32 %i2586, i32 0
- %i2588 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %i2587, i32 0, i32 0)
- %i2589 = shl i32 %i2020, 1
- %i2590 = select i1 %arg597, i32 %i2589, i32 0
- %i2591 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %i2590, i32 0, i32 0)
- %i2592 = shl i32 %i2021, 1
- %i2593 = select i1 %i2350, i32 %i2592, i32 0
- %i2594 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %i2593, i32 0, i32 0)
- %i2595 = shl i32 %i2022, 1
- %i2596 = select i1 %i2351, i32 %i2595, i32 0
- %i2597 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2596, i32 0, i32 0)
- %i2598 = shl i32 %i2023, 1
- %i2599 = select i1 %i2352, i32 %i2598, i32 0
- %i2600 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2599, i32 0, i32 0)
- %i2601 = shl i32 %i2024, 1
- %i2602 = select i1 %i2353, i32 %i2601, i32 0
- %i2603 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2602, i32 0, i32 0)
- %i2604 = shl i32 %i2025, 1
- %i2605 = select i1 %i2354, i32 %i2604, i32 0
- %i2606 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %i2605, i32 0, i32 0)
- %i2607 = shl i32 %i2026, 1
- %i2608 = select i1 %i2355, i32 %i2607, i32 0
- %i2609 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %i2608, i32 0, i32 0)
- %i2610 = shl i32 %arg47, 1
- %i2611 = select i1 %arg580, i32 %arg4, i32 0
- %i2612 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i2613 = shl i32 %arg4, 1
- %i2614 = select i1 %arg580, i32 %arg4, i32 0
- %i2615 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i2616 = shl i32 %arg4, 1
- %i2617 = select i1 %arg580, i32 %arg4, i32 0
- %i2618 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i2619 = shl i32 %arg4, 1
- %i2620 = select i1 %arg580, i32 %arg4, i32 0
- %i2621 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %arg794, i32 0, i32 0)
- %i2622 = shl i32 %arg14, 1
- %i2623 = select i1 %arg580, i32 %arg4, i32 0
- %i2624 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %arg789, i32 0, i32 0)
- %i2625 = shl i32 0, 0
- %i2626 = select i1 false, i32 0, i32 0
- %i2627 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) null, i32 0, i32 0, i32 0)
- %i2628 = shl i32 %arg4, 0
- %i2629 = select i1 %arg580, i32 %arg4, i32 0
- %i2630 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %arg750, i32 0, i32 0)
- %i2631 = shl i32 %arg4, 0
- %i2632 = select i1 %arg580, i32 %arg4, i32 0
- %i2633 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %arg703, i32 0, i32 0)
- %i2634 = shl i32 %arg41, 0
- %i2635 = select i1 %arg580, i32 %arg4, i32 0
- %i2636 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i2637 = shl i32 %arg4, 1
- %i2638 = select i1 %arg580, i32 %arg4, i32 0
- %i2639 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i2640 = shl i32 %arg4, 1
- %i2641 = select i1 %arg580, i32 %arg4, i32 0
- %i2642 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i2643 = shl i32 %arg4, 1
- %i2644 = select i1 %arg580, i32 %arg4, i32 0
- %i2645 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %arg822, i32 0, i32 0)
- %i2646 = shl i32 %arg40, 1
- %i2647 = select i1 %arg580, i32 %arg4, i32 0
- %i2648 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %arg826, i32 0, i32 0)
- %i2649 = shl i32 %arg52, 1
- %i2650 = select i1 %arg580, i32 %arg4, i32 0
- %i2651 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %arg830, i32 0, i32 0)
- %i2652 = shl i32 %arg55, 1
- %i2653 = select i1 %i2370, i32 %arg53, i32 0
- %i2654 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %i2653, i32 0, i32 0)
- %i2655 = shl i32 %i2042, 0
- %i2656 = select i1 %i2371, i32 %i2655, i32 0
- %i2657 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %i2656, i32 0, i32 0)
- %i2658 = shl i32 %i2043, 1
- %i2659 = select i1 %arg626, i32 %i2658, i32 0
- %i2660 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %i2659, i32 0, i32 0)
- %i2661 = shl i32 %i2044, 1
- %i2662 = select i1 %arg599, i32 %i2661, i32 0
- %i2663 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %i2662, i32 0, i32 0)
- %i2664 = shl i32 %i2045, 1
- %i2665 = select i1 %arg623, i32 %i2664, i32 0
- %i2666 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %i2665, i32 0, i32 0)
- %i2667 = shl i32 %i2046, 1
- %i2668 = select i1 %i2375, i32 %i2667, i32 0
- %i2669 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %i2668, i32 0, i32 0)
- %i2670 = shl i32 %i2047, 1
- %i2671 = select i1 %i2376, i32 %i2670, i32 0
- %i2672 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %i2671, i32 0, i32 0)
- %i2673 = shl i32 %i2048, 1
- %i2674 = select i1 %i2377, i32 %i2673, i32 0
- %i2675 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %i2674, i32 0, i32 0)
- %i2676 = shl i32 %i2049, 0
- %i2677 = select i1 %i2378, i32 %i2676, i32 0
- %i2678 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %i2677, i32 0, i32 0)
- %i2679 = shl i32 %arg56, 0
- %i2680 = select i1 %i2379, i32 %arg36, i32 0
- %i2681 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %i2680, i32 0, i32 0)
- %i2682 = shl i32 %arg14, 0
- %i2683 = select i1 %arg580, i32 0, i32 0
- %i2684 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 0, i32 0, i32 0)
- %i2685 = shl i32 %arg4, 0
- %i2686 = select i1 %arg580, i32 0, i32 0
- %i2687 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 0, i32 0, i32 0)
- %i2688 = shl i32 %arg4, 0
- %i2689 = select i1 %arg580, i32 0, i32 0
- %i2690 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 0, i32 0, i32 0)
- %i2691 = shl nuw nsw i32 %i1483, 1
- %i2692 = icmp eq i32 %i1359, 0
- %i2693 = select i1 %i2692, i32 0, i32 528
- %i2694 = xor i32 %i2691, %i2693
- %i2695 = xor i32 %arg869, 1056
- %i2696 = getelementptr inbounds nuw i8, ptr addrspace(3) getelementptr (i8, ptr addrspace(3) @global_smem, i32 262144), i32 %i2695
- store i16 %i1503, ptr addrspace(3) %i2696, align 2
- %i2697 = getelementptr inbounds nuw i8, ptr addrspace(3) @global_smem, i32 %i2694
- store i16 %i2386, ptr addrspace(3) %i2697, align 2
- store i16 %i2410, ptr addrspace(3) null, align 2
- %i2698 = or disjoint i32 0, 1
- %i2699 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %i2698
- store i16 %i2433, ptr addrspace(3) %i2699, align 2
- store i16 %i2448, ptr addrspace(3) null, align 2
- store i16 %i2469, ptr addrspace(3) %global_smem, align 2
- %i2700 = or disjoint i32 1, 0
- store i16 %i2493, ptr addrspace(3) null, align 2
- store i16 %i2517, ptr addrspace(3) %global_smem, align 2
- store i16 %i2541, ptr addrspace(3) null, align 2
- %i2701 = or disjoint i32 %i2694, 65536
- %i2702 = getelementptr inbounds nuw i8, ptr addrspace(3) @global_smem, i32 %i2701
- store i16 %i2557, ptr addrspace(3) %i2702, align 2
- %i2703 = or disjoint i32 %arg718, 73728
- %i2704 = or disjoint i32 0, 81920
- store i16 %i2573, ptr addrspace(3) null, align 2
- %i2705 = or disjoint i32 %i2694, 98304
- %i2706 = getelementptr inbounds nuw i8, ptr addrspace(3) @global_smem, i32 %i2705
- store i16 %i2597, ptr addrspace(3) %i2706, align 2
- %i2707 = or disjoint i32 %arg53, 106496
- %i2708 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %i2707
- store i16 %i2621, ptr addrspace(3) null, align 2
- %i2709 = or disjoint i32 0, 114688
- %i2710 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %i2709
- store i16 %i2645, ptr addrspace(3) %i2710, align 2
- %i2711 = or disjoint i32 255, 122880
- store i16 %i2669, ptr addrspace(3) null, align 2
- %i2712 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %arg832
- store i16 %i2389, ptr addrspace(3) %i2712, align 2
- %i2713 = or disjoint i32 0, 0
- %i2714 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %i2713
- store i16 %i2413, ptr addrspace(3) %i2714, align 2
- %i2715 = or disjoint i32 1, 0
- store i16 %i2451, ptr addrspace(3) null, align 2
- store i16 %i2472, ptr addrspace(3) %global_smem, align 2
- store i16 %arg833, ptr addrspace(3) null, align 2
- store i16 %i2520, ptr addrspace(3) %global_smem, align 2
- store i16 %i2543, ptr addrspace(3) null, align 2
- %i2716 = or disjoint i32 %i2695, 65536
- %i2717 = getelementptr inbounds nuw i8, ptr addrspace(3) @global_smem, i32 %i2716
- store i16 0, ptr addrspace(3) %i2717, align 2
- %i2718 = or disjoint i32 0, 0
- %i2719 = or disjoint i32 %arg127, 81920
- store i16 %i2576, ptr addrspace(3) null, align 2
- %i2720 = or disjoint i32 %i2695, 98304
- %i2721 = getelementptr inbounds nuw i8, ptr addrspace(3) @global_smem, i32 %i2720
- store i16 %i2600, ptr addrspace(3) %i2721, align 2
- %i2722 = or disjoint i32 1, 1
- %i2723 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %i2722
- store i16 %i2624, ptr addrspace(3) %i2723, align 2
- %i2724 = or disjoint i32 %i2695, 114688
- %i2725 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %i2724
- store i16 %i2648, ptr addrspace(3) null, align 2
- %i2726 = or disjoint i32 %i2695, 122880
- %i2727 = getelementptr inbounds nuw i8, ptr addrspace(3) @global_smem, i32 %i2726
- store i16 %i2672, ptr addrspace(3) %i2727, align 2
- %i2728 = xor i32 %arg790, 2112
- store i16 %i2392, ptr addrspace(3) null, align 2
- %i2729 = or disjoint i32 %i2728, 0
- %i2730 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %i2729
- store i16 %i2416, ptr addrspace(3) %i2730, align 2
- store i16 %i2454, ptr addrspace(3) null, align 2
- %i2731 = or disjoint i32 1, 0
- %i2732 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %i2731
- store i16 %i2475, ptr addrspace(3) %i2732, align 2
- store i16 %i2499, ptr addrspace(3) null, align 2
- store i16 %i2523, ptr addrspace(3) %global_smem, align 2
- store i16 %i2545, ptr addrspace(3) null, align 2
- %i2733 = or disjoint i32 0, 81920
- %i2734 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %i2733
- store i16 0, ptr addrspace(3) %i2734, align 2
- %i2735 = or disjoint i32 0, 0
- %i2736 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %i2735
- store i16 %i2579, ptr addrspace(3) null, align 2
- %i2737 = or disjoint i32 0, 1
- %i2738 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %i2737
- store i16 %i2603, ptr addrspace(3) %i2738, align 2
- %i2739 = or disjoint i32 0, 0
- %i2740 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %i2739
- store i16 0, ptr addrspace(3) null, align 2
- %i2741 = or disjoint i32 0, 0
- %i2742 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %i2741
- store i16 %i2651, ptr addrspace(3) null, align 2
- %i2743 = or disjoint i32 0, 1
- %i2744 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %i2743
- store i16 %i2675, ptr addrspace(3) %i2744, align 2
- %i2745 = xor i32 %arg789, 3168
- store i16 %i2395, ptr addrspace(3) null, align 2
- %i2746 = or disjoint i32 0, 0
- %i2747 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %i2746
- store i16 %i2419, ptr addrspace(3) %i2747, align 2
- store i16 %i2478, ptr addrspace(3) null, align 2
- store i16 %i2502, ptr addrspace(3) %global_smem, align 2
- store i16 %i2526, ptr addrspace(3) null, align 2
- store i16 %i2547, ptr addrspace(3) %global_smem, align 2
- %i2748 = or disjoint i32 %arg823, 73728
- %i2749 = or disjoint i32 %arg19, 90112
- %i2750 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %i2749
- store i16 %i2582, ptr addrspace(3) null, align 2
- %i2751 = or disjoint i32 %i2745, 98304
- %i2752 = getelementptr inbounds nuw i8, ptr addrspace(3) @global_smem, i32 %i2751
- store i16 %i2606, ptr addrspace(3) %i2752, align 2
- %i2753 = or disjoint i32 %arg123, 106496
- %i2754 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %i2753
- store i16 %i2630, ptr addrspace(3) null, align 2
- %i2755 = or disjoint i32 %i2745, 114688
- %i2756 = getelementptr inbounds nuw i8, ptr addrspace(3) @global_smem, i32 %i2755
- store i16 %i2654, ptr addrspace(3) %i2756, align 2
- %i2757 = or disjoint i32 0, 0
- store i16 %i2678, ptr addrspace(3) null, align 2
- %i2758 = xor i32 %arg1146, 0
- %i2759 = getelementptr inbounds nuw i8, ptr addrspace(3) @global_smem, i32 %i2758
- store i16 %i2398, ptr addrspace(3) %i2759, align 2
- store i16 %i2422, ptr addrspace(3) null, align 2
- %i2760 = or disjoint i32 1, 0
- %i2761 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %i2760
- store i16 %i2436, ptr addrspace(3) %i2761, align 2
- store i16 %i2457, ptr addrspace(3) null, align 2
- %i2762 = or disjoint i32 1, 0
- %i2763 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %i2762
- store i16 %i2481, ptr addrspace(3) %i2763, align 2
- store i16 %i2505, ptr addrspace(3) null, align 2
- store i16 %i2529, ptr addrspace(3) %global_smem, align 2
- store i16 %i2549, ptr addrspace(3) null, align 2
- %i2764 = or disjoint i32 0, 0
- %i2765 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %i2764
- store i16 0, ptr addrspace(3) %i2765, align 2
- %i2766 = or disjoint i32 1, 0
- store i16 %i2561, ptr addrspace(3) null, align 2
- %i2767 = or disjoint i32 %arg89, 90112
- %i2768 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %arg834
- store i16 %i2585, ptr addrspace(3) %i2768, align 2
- %i2769 = or disjoint i32 0, 0
- %i2770 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %i2769
- store i16 %i2609, ptr addrspace(3) %i2770, align 2
- store i16 %i2633, ptr addrspace(3) null, align 2
- store i16 %i2657, ptr addrspace(3) %global_smem, align 2
- store i16 %i2681, ptr addrspace(3) null, align 2
- %i2771 = xor i32 %arg98, 5280
- %i2772 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %i2771
- store i16 %i2401, ptr addrspace(3) %arg835, align 2
- store i16 %arg836, ptr addrspace(3) %global_smem, align 2
- %i2773 = or disjoint i32 1, 0
- store i16 %arg837, ptr addrspace(3) null, align 2
- store i16 %arg838, ptr addrspace(3) %global_smem, align 2
- store i16 %arg839, ptr addrspace(3) null, align 2
- store i16 %arg840, ptr addrspace(3) %global_smem, align 2
- store i16 %arg841, ptr addrspace(3) null, align 2
- store i16 %arg842, ptr addrspace(3) %global_smem, align 2
- %i2774 = or disjoint i32 1, 0
- %i2775 = or disjoint i32 %arg93, 73728
- store i16 %arg844, ptr addrspace(3) null, align 2
- %i2776 = or disjoint i32 %arg843, 90112
- %i2777 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %arg845
- store i16 %arg846, ptr addrspace(3) %arg847, align 2
- %i2778 = or disjoint i32 %arg750, 98304
- store i16 %arg848, ptr addrspace(3) null, align 2
- %i2779 = or disjoint i32 %arg146, 106496
- %i2780 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %arg849
- store i16 %arg850, ptr addrspace(3) %arg851, align 2
- %i2781 = or disjoint i32 0, 0
- store i16 %arg852, ptr addrspace(3) null, align 2
- %i2782 = or disjoint i32 %arg843, 122880
- %i2783 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %arg853
- store i16 %arg854, ptr addrspace(3) %arg855, align 2
- %i2784 = xor i32 %arg856, 1
- store i16 %arg857, ptr addrspace(3) null, align 2
- store i16 %arg858, ptr addrspace(3) %global_smem, align 2
- store i16 %arg859, ptr addrspace(3) null, align 2
- %i2785 = or disjoint i32 %arg860, 0
- %i2786 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %arg861
- store i16 %arg862, ptr addrspace(3) %arg863, align 2
- store i16 %arg864, ptr addrspace(3) %global_smem, align 2
- store i16 %arg865, ptr addrspace(3) null, align 2
- store i16 %arg866, ptr addrspace(3) %global_smem, align 2
- store i16 0, ptr addrspace(3) null, align 2
- %i2787 = or disjoint i32 %arg860, 73728
- %i2788 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %arg867
- store i16 0, ptr addrspace(3) null, align 2
- %i2789 = or disjoint i32 %arg715, 81920
- store i16 %arg868, ptr addrspace(3) null, align 2
- %i2790 = or disjoint i32 %arg103, 90112
- %i2791 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %arg869
- store i16 %arg870, ptr addrspace(3) %arg871, align 2
- %i2792 = or disjoint i32 %arg19, 98304
- store i16 %arg872, ptr addrspace(3) null, align 2
- %i2793 = or disjoint i32 0, 0
- %i2794 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %arg873
- store i16 %arg874, ptr addrspace(3) %arg875, align 2
- %i2795 = or disjoint i32 %arg22, 114688
- store i16 %arg876, ptr addrspace(3) null, align 2
- %i2796 = or disjoint i32 %arg113, 122880
- %i2797 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %arg877
- store i16 %arg878, ptr addrspace(3) %arg879, align 2
- %i2798 = xor i32 1, 0
- store i16 %arg880, ptr addrspace(3) null, align 2
- store i16 %arg881, ptr addrspace(3) %global_smem, align 2
- store i16 %arg882, ptr addrspace(3) null, align 2
- %i2799 = or disjoint i32 1, 0
- %i2800 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %arg883
- store i16 %arg884, ptr addrspace(3) %arg885, align 2
- store i16 %arg886, ptr addrspace(3) null, align 2
- store i16 %arg887, ptr addrspace(3) %global_smem, align 2
- store i16 %arg888, ptr addrspace(3) null, align 2
- store i16 %arg889, ptr addrspace(3) %global_smem, align 2
- %i2801 = or disjoint i32 1, 0
- %i2802 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %arg890
- store i16 %arg891, ptr addrspace(3) %arg892, align 2
- %i2803 = or disjoint i32 0, 0
- store i16 %arg893, ptr addrspace(3) null, align 2
- %i2804 = or disjoint i32 0, 0
- %i2805 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %arg894
- store i16 %arg895, ptr addrspace(3) %arg896, align 2
- %i2806 = or disjoint i32 0, 0
- store i16 %arg897, ptr addrspace(3) null, align 2
- %i2807 = or disjoint i32 0, 0
- %i2808 = getelementptr inbounds nuw i8, ptr addrspace(3) %global_smem, i32 %arg898
- store i16 %arg899, ptr addrspace(3) %arg900, align 2
- store i16 %arg901, ptr addrspace(3) null, align 2
- %i2809 = add nsw i32 %arg15, 0
- %i2810 = icmp sgt i32 %arg4, 1
- br i1 %arg903, label %.lr.ph, label %.._crit_edge_crit_edge
-
-.._crit_edge_crit_edge: ; preds = %bb
- %.pre = and i32 %arg10, 1
- br label %._crit_edge
-
-.lr.ph: ; preds = %bb
- %i2811 = and i32 %arg10, 1
- %smax = tail call i32 @llvm.smax.i32(i32 %arg904, i32 0)
- br label %bb2812
-
-bb2812: ; preds = %bb2812, %.lr.ph
- %.pn17784 = phi i32 [ %arg905, %.lr.ph ], [ %arg906, %bb2812 ]
- %.pn19783 = phi i32 [ %arg907, %.lr.ph ], [ %arg908, %bb2812 ]
- %.pn21782 = phi i32 [ %arg909, %.lr.ph ], [ %arg910, %bb2812 ]
- %.pn23781 = phi i32 [ %arg911, %.lr.ph ], [ %arg912, %bb2812 ]
- %.pn25780 = phi i32 [ %arg913, %.lr.ph ], [ %arg914, %bb2812 ]
- %.pn27779 = phi i32 [ %arg915, %.lr.ph ], [ %arg916, %bb2812 ]
- %.pn31777 = phi i32 [ %arg831, %.lr.ph ], [ %arg917, %bb2812 ]
- %.pn33776 = phi i32 [ %arg827, %.lr.ph ], [ %arg918, %bb2812 ]
- %.pn35775 = phi i32 [ %arg823, %.lr.ph ], [ %arg919, %bb2812 ]
- %.pn37774 = phi i32 [ %arg41, %.lr.ph ], [ %arg920, %bb2812 ]
- %.pn39773 = phi i32 [ %arg92, %.lr.ph ], [ %arg921, %bb2812 ]
- %.pn417723 = phi i32 [ %arg45, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn497684 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn517675 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn537666 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn557657 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn577648 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn597639 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn6176210 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn6376111 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn6576012 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn6775913 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn6975814 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn7175715 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn7375616 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn7575517 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn7775418 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn7975319 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn8175220 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn8375121 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn8575022 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn8774923 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn8974824 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn9174725 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn9374626 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn95745 = phi i32 [ %arg4, %.lr.ph ], [ 0, %bb2812 ]
- %.pn97744 = phi i32 [ 0, %.lr.ph ], [ 0, %bb2812 ]
- %.pn9974327 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn10174228 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn10374129 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn10574030 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn219683 = phi i32 [ 0, %.lr.ph ], [ 0, %bb2812 ]
- %.pn221682 = phi i32 [ 0, %.lr.ph ], [ 0, %bb2812 ]
- %.pn22368131 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn225680 = phi i32 [ 0, %.lr.ph ], [ 0, %bb2812 ]
- %.pn227679 = phi i32 [ 0, %.lr.ph ], [ 0, %bb2812 ]
- %.pn229678 = phi i32 [ 0, %.lr.ph ], [ 0, %bb2812 ]
- %.pn23167732 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn23367633 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn235675 = phi i32 [ 0, %.lr.ph ], [ 0, %bb2812 ]
- %.pn23767434 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn23967335 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn24167236 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn24367137 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn24567038 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn24766939 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn24966840 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn251667 = phi i32 [ 0, %.lr.ph ], [ 0, %bb2812 ]
- %.pn25366641 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn255665 = phi i32 [ 1, %.lr.ph ], [ 1, %bb2812 ]
- %.pn25766442 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn25966343 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %.pn26166244 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %i2813 = phi float [ 0.000000e+00, %.lr.ph ], [ %arg1000, %bb2812 ]
- %i2814 = phi float [ 0.000000e+00, %.lr.ph ], [ %arg1000, %bb2812 ]
- %.pn366145 = phi i32 [ %arg4, %.lr.ph ], [ %arg4, %bb2812 ]
- %i2815 = phi i32 [ 0, %.lr.ph ], [ %arg1004, %bb2812 ]
- %i2816 = add i32 %.pn3661, 1
- %i2817 = add i32 %.pn261662, 1
- %i2818 = add i32 %arg61, 1
- %i2819 = add i32 %.pn257664, 1
- %i2820 = add i32 1, 256
- %i2821 = add i32 %arg48, 1
- %i2822 = add i32 0, 0
- %i2823 = add i32 %.pn249668, 1
- %i2824 = add i32 %arg20, 256
- %i2825 = add i32 %arg4, 1
- %i2826 = add i32 %arg4, 1
- %i2827 = add i32 %arg4, 256
- %i2828 = add i32 %arg4, 1
- %i2829 = add i32 %arg4, 0
- %i2830 = add i32 0, 256
- %i2831 = add i32 %arg4, 1
- %i2832 = add i32 %arg4, 256
- %i2833 = add i32 0, 0
- %i2834 = add i32 0, 0
- %i2835 = add i32 0, 0
- %i2836 = add i32 %arg4, 1
- %i2837 = add i32 0, 0
- %i2838 = add i32 0, 0
- %i2839 = add i32 %.pn105740, 1
- %i2840 = add i32 %.pn103741, 1
- %i2841 = add i32 %.pn101742, 1
- %i2842 = add i32 %.pn99743, 256
- %i2843 = add i32 0, 256
- %i2844 = add i32 0, 256
- %i2845 = add i32 %.pn93746, 1
- %i2846 = add i32 %.pn91747, 1
- %i2847 = add i32 %.pn89748, 1
- %i2848 = add i32 %.pn87749, 1
- %i2849 = add i32 %.pn85750, 1
- %i2850 = add i32 %.pn83751, 1
- %i2851 = add i32 %.pn81752, 1
- %i2852 = add i32 %.pn79753, 1
- %i2853 = add i32 %.pn77754, 1
- %i2854 = add i32 %.pn75755, 1
- %i2855 = add i32 %.pn73756, 1
- %i2856 = add i32 %arg14, 1
- %i2857 = add i32 %arg4, 1
- %i2858 = add i32 %arg942, 1
- %i2859 = add i32 %arg122, 1
- %i2860 = add i32 %.decomposed, 1
- %i2861 = add i32 %arg48, 1
- %i2862 = add i32 %.pn59763, 1
- %i2863 = add i32 %arg720, 1
- %i2864 = add i32 %arg4, 1
- %i2865 = add i32 %arg4, 1
- %i2866 = add i32 %arg4, 1
- %i2867 = add i32 %arg4, 1
- %i2868 = add i32 %arg4, 1
- %i2869 = add i32 %.pn39773, 1
- %i2870 = add i32 %.pn37774, 1
- %i2871 = add i32 %.pn35775, 1
- %i2872 = add i32 %.pn33776, 1
- %i2873 = add i32 %.pn31777, 1
- %i2874 = add i32 %.pn27779, 1
- %i2875 = add i32 %.pn25780, 1
- %i2876 = add i32 %.pn23781, 1
- %i2877 = add i32 %.pn21782, 1
- %i2878 = add i32 %.pn19783, 1
- %i2879 = add i32 %.pn17784, 1
- %i2880 = add nuw nsw i32 %i2815, 1
- %i2881 = shl i32 %i2880, 0
- %i2882 = sub i32 %arg6, %i2881
- %i2883 = icmp slt i32 %i1483, %i2882
- %i2884 = and i1 %i1500, %i2883
- %i2885 = shl i32 %i2816, 1
- %i2886 = select i1 %i2884, i32 %i2885, i32 0
- %i2887 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i1502, i32 %i2886, i32 0, i32 0)
- tail call void @llvm.amdgcn.s.waitcnt(i32 0)
- %i2888 = and i1 %i2055, true
- %i2889 = and i1 %i2056, %i2883
- %i2890 = and i1 %i2057, %i2883
- %i2891 = and i1 %i2058, %i2883
- %i2892 = and i1 %i2059, %i2883
- %i2893 = and i1 %i2060, %i2883
- %i2894 = and i1 %i2061, %i2883
- %i2895 = and i1 %i2062, %i2883
- %i2896 = and i1 %arg589, %arg617
- %i2897 = and i1 %arg580, %arg580
- %i2898 = and i1 %arg580, %arg580
- %i2899 = and i1 %arg580, %arg580
- %i2900 = and i1 %arg580, %arg580
- %i2901 = and i1 %arg580, %arg580
- %i2902 = and i1 %arg580, %arg580
- %i2903 = and i1 %arg580, %arg580
- %i2904 = and i1 %arg580, %arg580
- %i2905 = and i1 %arg580, %arg580
- %i2906 = and i1 %arg580, %arg580
- %i2907 = and i1 %arg580, %arg580
- %i2908 = and i1 %arg580, %arg580
- %i2909 = and i1 %arg580, %arg580
- %i2910 = and i1 %arg580, %arg580
- %i2911 = and i1 %arg580, %arg580
- %i2912 = and i1 %arg580, %arg580
- %i2913 = and i1 %arg580, %arg580
- %i2914 = and i1 %i2081, %i2883
- %i2915 = and i1 %i2082, %i2883
- %i2916 = and i1 %i2083, %i2883
- %i2917 = and i1 %i2084, %i2883
- %i2918 = and i1 %i2085, %i2883
- %i2919 = and i1 %arg663, %i2883
- %i2920 = and i1 %i2088, %i2883
- %i2921 = and i1 %i2089, %i2883
- %i2922 = and i1 %i2090, %i2883
- %i2923 = and i1 %i2091, %i2883
- %i2924 = and i1 %i2092, %i2883
- %i2925 = and i1 %i2093, %i2883
- %i2926 = and i1 %i2094, %i2883
- %i2927 = and i1 %arg617, %i2883
- %i2928 = and i1 %i2097, %i2883
- %i2929 = and i1 %arg611, %arg586
- %i2930 = and i1 %i2099, %i2883
- %i2931 = and i1 %i2100, %i2883
- %i2932 = and i1 %i2101, %i2883
- %i2933 = and i1 %i2102, %i2883
- %i2934 = and i1 %i2104, %i2883
- %i2935 = and i1 %i2105, %i2883
- %i2936 = and i1 %i2106, %i2883
- %i2937 = and i1 %i2107, %i2883
- %i2938 = and i1 %i2108, %i2883
- %i2939 = and i1 %i2109, %i2883
- %i2940 = and i1 %i2110, %i2883
- %i2941 = and i1 %i2111, %i2883
- %i2942 = and i1 %i2120, %i2883
- %i2943 = and i1 %i2121, %i2883
- %i2944 = and i1 %i2122, %i2883
- %i2945 = and i1 %i2123, %i2883
- %i2946 = and i1 %i2124, %i2883
- %i2947 = and i1 %i2125, %i2883
- %i2948 = and i1 %i2126, %i2883
- %i2949 = and i1 %i2127, %i2883
- %i2950 = and i1 %i2128, %i2883
- %i2951 = and i1 %i2129, %i2883
- %i2952 = and i1 %i2130, %i2883
- %i2953 = and i1 %i2131, %i2883
- %i2954 = and i1 %i2132, %i2883
- %i2955 = and i1 %i2133, %i2883
- %i2956 = and i1 %i2134, %i2883
- %i2957 = and i1 %i2135, %i2883
- %i2958 = and i1 %i2136, %i2883
- %i2959 = and i1 %i2137, %i2883
- %i2960 = and i1 %i2138, %i2883
- %i2961 = and i1 %i2139, %i2883
- %i2962 = and i1 %i2140, %i2883
- %i2963 = and i1 %i2141, %i2883
- %i2964 = and i1 %i2142, %i2883
- %i2965 = and i1 %i2143, %i2883
- %i2966 = and i1 %i2144, %i2883
- %i2967 = and i1 %i2145, %i2883
- %i2968 = and i1 %i2146, %i2883
- %i2969 = and i1 %i2147, %i2883
- %i2970 = and i1 %i2148, %i2883
- %i2971 = and i1 %i2149, %i2883
- %i2972 = and i1 %i2150, %i2883
- %i2973 = and i1 %arg583, %arg580
- %i2974 = and i1 %arg580, %arg580
- %i2975 = and i1 %i2154, %i2883
- %i2976 = and i1 %i2155, %i2883
- %i2977 = and i1 %i2156, %i2883
- %i2978 = and i1 %i2157, %i2883
- %i2979 = and i1 %i2158, %i2883
- %i2980 = and i1 %i2160, %i2883
- %i2981 = and i1 %i2161, %i2883
- %i2982 = and i1 %i2162, %i2883
- %i2983 = and i1 %i2163, %i2883
- %i2984 = and i1 %i2164, %i2883
- %i2985 = and i1 %i2165, %i2883
- %i2986 = shl i32 %i2817, 1
- %i2987 = select i1 %i2888, i32 %i2986, i32 0
- %i2988 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2987, i32 0, i32 0)
- %i2989 = shl i32 %arg25, 1
- %i2990 = select i1 %i2889, i32 %arg715, i32 0
- %i2991 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) null, i32 %i2990, i32 0, i32 0)
- %i2992 = shl i32 %i2819, 1
- %i2993 = select i1 %i2890, i32 %i2992, i32 0
- %i2994 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2993, i32 0, i32 0)
- %i2995 = shl i32 %arg869, 1
- %i2996 = select i1 %i2891, i32 %i2995, i32 0
- %i2997 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2996, i32 0, i32 0)
- %i2998 = shl i32 %.pn253666, 1
- %i2999 = select i1 %i2892, i32 %i2998, i32 0
- %i3000 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i2999, i32 0, i32 0)
- %i3001 = shl i32 0, 0
- %i3002 = select i1 %i2893, i32 0, i32 -2147483648
- %i3003 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3002, i32 0, i32 0)
- %i3004 = shl i32 %i2823, 1
- %i3005 = select i1 %i2894, i32 %i3004, i32 0
- %i3006 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3005, i32 0, i32 0)
- %i3007 = shl i32 %arg787, 1
- %i3008 = select i1 %i2895, i32 %i3007, i32 0
- %i3009 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3008, i32 0, i32 0)
- %i3010 = shl i32 %arg18, 1
- %i3011 = select i1 %arg580, i32 %arg4, i32 0
- %i3012 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %arg1011, i32 0, i32 0)
- %i3013 = shl i32 %arg49, 1
- %i3014 = select i1 %arg580, i32 %arg4, i32 0
- %i3015 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %arg1013, i32 0, i32 0)
- %i3016 = shl i32 %arg4, 1
- %i3017 = select i1 %arg580, i32 %arg4, i32 0
- %i3018 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) null, i32 %arg1017, i32 0, i32 0)
- %i3019 = shl i32 %arg15, 1
- %i3020 = select i1 %arg580, i32 %arg4, i32 0
- %i3021 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %arg1020, i32 0, i32 0)
- %i3022 = shl i32 %arg36, 1
- %i3023 = select i1 %arg580, i32 %arg4, i32 0
- %i3024 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %arg1023, i32 0, i32 0)
- %i3025 = shl i32 %arg42, 0
- %i3026 = select i1 %arg580, i32 %arg4, i32 0
- %i3027 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %arg908, i32 0, i32 0)
- %i3028 = shl i32 %arg4, 1
- %i3029 = select i1 %arg580, i32 %arg4, i32 0
- %i3030 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %arg721, i32 0, i32 0)
- %i3031 = shl i32 %arg4, 1
- %i3032 = select i1 %arg580, i32 %arg4, i32 0
- %i3033 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) null, i32 %arg4, i32 0, i32 0)
- %i3034 = shl i32 0, 0
- %i3035 = select i1 %arg580, i32 0, i32 -2147483648
- %i3036 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %arg1035, i32 0, i32 0)
- %i3037 = shl i32 0, 0
- %i3038 = select i1 %arg600, i32 0, i32 -2147483648
- %i3039 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %arg1030, i32 0, i32 0)
- %i3040 = shl i32 0, 0
- %i3041 = select i1 %arg580, i32 0, i32 -2147483648
- %i3042 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %arg1039, i32 0, i32 0)
- %i3043 = shl i32 %.decomposed, 1
- %i3044 = select i1 %arg580, i32 %arg4, i32 0
- %i3045 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %arg1042, i32 0, i32 0)
- %i3046 = shl i32 0, 0
- %i3047 = select i1 %arg581, i32 0, i32 -2147483648
- %i3048 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %arg1044, i32 0, i32 0)
- %i3049 = shl i32 0, 0
- %i3050 = select i1 %arg602, i32 0, i32 -2147483648
- %i3051 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %arg1046, i32 0, i32 0)
- %i3052 = select i1 %arg619, i32 0, i32 -2147483648
- %i3053 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3054 = select i1 %arg580, i32 0, i32 -2147483648
- %i3055 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3056 = select i1 %arg580, i32 0, i32 -2147483648
- %i3057 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %arg67, i32 0, i32 0)
- %i3058 = select i1 %arg1053, i32 0, i32 -2147483648
- %i3059 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3058, i32 0, i32 0)
- %i3060 = select i1 %i2914, i32 0, i32 -2147483648
- %i3061 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3060, i32 0, i32 0)
- %i3062 = select i1 %i2915, i32 0, i32 -2147483648
- %i3063 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3062, i32 0, i32 0)
- %i3064 = select i1 %i2916, i32 0, i32 -2147483648
- %i3065 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3064, i32 0, i32 0)
- %i3066 = select i1 %i2917, i32 0, i32 -2147483648
- %i3067 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3066, i32 0, i32 0)
- %i3068 = select i1 %i2918, i32 0, i32 -2147483648
- %i3069 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3068, i32 0, i32 0)
- %i3070 = select i1 %i2919, i32 0, i32 -2147483648
- %i3071 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3070, i32 0, i32 0)
- %i3072 = select i1 %i2920, i32 0, i32 -2147483648
- %i3073 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3072, i32 0, i32 0)
- %i3074 = select i1 %i2921, i32 0, i32 -2147483648
- %i3075 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3074, i32 0, i32 0)
- %i3076 = select i1 %i2922, i32 0, i32 -2147483648
- %i3077 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3076, i32 0, i32 0)
- %i3078 = select i1 %i2923, i32 0, i32 -2147483648
- %i3079 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3078, i32 0, i32 0)
- %i3080 = select i1 %i2924, i32 0, i32 -2147483648
- %i3081 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3080, i32 0, i32 0)
- %i3082 = select i1 %i2925, i32 0, i32 -2147483648
- %i3083 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3082, i32 0, i32 0)
- %i3084 = select i1 %i2926, i32 0, i32 -2147483648
- %i3085 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3084, i32 0, i32 0)
- %i3086 = select i1 %i2927, i32 0, i32 -2147483648
- %i3087 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3086, i32 0, i32 0)
- %i3088 = select i1 %i2928, i32 0, i32 -2147483648
- %i3089 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3088, i32 0, i32 0)
- %i3090 = select i1 %arg608, i32 0, i32 -2147483648
- %i3091 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %arg1055, i32 0, i32 0)
- %i3092 = select i1 %i2930, i32 0, i32 -2147483648
- %i3093 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3092, i32 0, i32 0)
- %i3094 = select i1 %i2931, i32 0, i32 -2147483648
- %i3095 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3094, i32 0, i32 0)
- %i3096 = select i1 %i2932, i32 0, i32 -2147483648
- %i3097 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3096, i32 0, i32 0)
- %i3098 = select i1 %i2933, i32 0, i32 -2147483648
- %i3099 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3098, i32 0, i32 0)
- %i3100 = select i1 %i2934, i32 0, i32 -2147483648
- %i3101 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3100, i32 0, i32 0)
- %i3102 = select i1 %i2935, i32 0, i32 -2147483648
- %i3103 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3102, i32 0, i32 0)
- %i3104 = select i1 %i2936, i32 0, i32 -2147483648
- %i3105 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3104, i32 0, i32 0)
- %i3106 = select i1 %i2937, i32 0, i32 -2147483648
- %i3107 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3106, i32 0, i32 0)
- %i3108 = select i1 %i2938, i32 0, i32 -2147483648
- %i3109 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3108, i32 0, i32 0)
- %i3110 = select i1 %i2939, i32 0, i32 -2147483648
- %i3111 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3110, i32 0, i32 0)
- %i3112 = select i1 %i2940, i32 0, i32 -2147483648
- %i3113 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3112, i32 0, i32 0)
- %i3114 = select i1 %i2941, i32 0, i32 -2147483648
- %i3115 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3114, i32 0, i32 0)
- %i3116 = select i1 %i2942, i32 0, i32 -2147483648
- %i3117 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3116, i32 0, i32 0)
- %i3118 = shl i32 %i2839, 1
- %i3119 = select i1 %i2943, i32 %i3118, i32 0
- %i3120 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3119, i32 0, i32 0)
- %i3121 = shl i32 %i2840, 1
- %i3122 = select i1 %i2944, i32 %i3121, i32 0
- %i3123 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3122, i32 0, i32 0)
- %i3124 = shl i32 %i2841, 1
- %i3125 = select i1 %i2945, i32 %i3124, i32 0
- %i3126 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3125, i32 0, i32 0)
- %i3127 = shl i32 %i2842, 1
- %i3128 = select i1 %i2946, i32 %i3127, i32 0
- %i3129 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3128, i32 0, i32 0)
- %i3130 = shl i32 %arg910, 0
- %i3131 = select i1 %i2947, i32 %i3130, i32 0
- %i3132 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3131, i32 0, i32 0)
- %i3133 = shl i32 %arg31, 1
- %i3134 = select i1 %i2948, i32 %arg4, i32 0
- %i3135 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3134, i32 0, i32 0)
- %i3136 = shl i32 %i2845, 1
- %i3137 = select i1 %i2949, i32 %i3136, i32 0
- %i3138 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3137, i32 0, i32 0)
- %i3139 = shl i32 %i2846, 1
- %i3140 = select i1 %i2950, i32 %i3139, i32 0
- %i3141 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3140, i32 0, i32 0)
- %i3142 = shl i32 %i2847, 1
- %i3143 = select i1 %i2951, i32 %i3142, i32 0
- %i3144 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3143, i32 0, i32 0)
- %i3145 = shl i32 %i2848, 1
- %i3146 = select i1 %i2952, i32 %i3145, i32 0
- %i3147 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3146, i32 0, i32 0)
- %i3148 = shl i32 %i2849, 1
- %i3149 = select i1 %i2953, i32 %i3148, i32 0
- %i3150 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3149, i32 0, i32 0)
- %i3151 = shl i32 %i2850, 1
- %i3152 = select i1 %i2954, i32 %i3151, i32 0
- %i3153 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3152, i32 0, i32 0)
- %i3154 = shl i32 %i2851, 1
- %i3155 = select i1 %i2955, i32 %i3154, i32 0
- %i3156 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3155, i32 0, i32 0)
- %i3157 = shl i32 %i2852, 1
- %i3158 = select i1 %i2956, i32 %i3157, i32 0
- %i3159 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3158, i32 0, i32 0)
- %i3160 = shl i32 %i2853, 1
- %i3161 = select i1 %i2957, i32 %i3160, i32 0
- %i3162 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3161, i32 0, i32 0)
- %i3163 = shl i32 %i2854, 1
- %i3164 = select i1 %i2958, i32 %i3163, i32 0
- %i3165 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3164, i32 0, i32 0)
- %i3166 = shl i32 %i2855, 1
- %i3167 = select i1 %i2959, i32 %i3166, i32 0
- %i3168 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3167, i32 0, i32 0)
- %i3169 = shl i32 %.pn223681, 1
- %i3170 = select i1 %i2960, i32 %i3169, i32 0
- %i3171 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3170, i32 0, i32 0)
- %i3172 = shl i32 %arg46, 1
- %i3173 = select i1 %i2961, i32 %arg1056, i32 0
- %i3174 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %i3173, i32 0, i32 0)
- %i3175 = shl i32 %i2858, 1
- %i3176 = select i1 %i2962, i32 %i3175, i32 0
- %i3177 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3176, i32 0, i32 0)
- %i3178 = shl i32 %i2859, 1
- %i3179 = select i1 %i2963, i32 %i3178, i32 0
- %i3180 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3179, i32 0, i32 0)
- %i3181 = shl i32 %i2860, 1
- %i3182 = select i1 %i2964, i32 %i3181, i32 0
- %i3183 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3182, i32 0, i32 0)
- %i3184 = shl i32 %arg43, 1
- %i3185 = select i1 %i2965, i32 %arg1030, i32 0
- %i3186 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3185, i32 0, i32 0)
- %i3187 = shl i32 %i2862, 1
- %i3188 = select i1 %i2966, i32 %i3187, i32 0
- %i3189 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3188, i32 0, i32 0)
- %i3190 = shl i32 %i2863, 1
- %i3191 = select i1 %i2967, i32 %i3190, i32 0
- %i3192 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3191, i32 0, i32 0)
- %i3193 = shl i32 %i2864, 1
- %i3194 = select i1 %i2968, i32 %i3193, i32 0
- %i3195 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3194, i32 0, i32 0)
- %i3196 = shl i32 %arg144, 1
- %i3197 = select i1 %i2969, i32 %i3196, i32 0
- %i3198 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3197, i32 0, i32 0)
- %i3199 = shl i32 %arg31, 1
- %i3200 = select i1 %i2970, i32 %arg92, i32 0
- %i3201 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3200, i32 0, i32 0)
- %i3202 = shl i32 %arg76, 1
- %i3203 = select i1 %i2971, i32 %i3202, i32 0
- %i3204 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) null, i32 %i3203, i32 0, i32 0)
- %i3205 = select i1 %i2972, i32 0, i32 -2147483648
- %i3206 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3205, i32 0, i32 0)
- %i3207 = select i1 %arg609, i32 0, i32 -2147483648
- %i3208 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %arg1026, i32 0, i32 0)
- %i3209 = shl i32 %arg4, 1
- %i3210 = select i1 %arg580, i32 %arg4, i32 0
- %i3211 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg688, i32 %arg127, i32 0, i32 0)
- %i3212 = shl i32 %i2869, 1
- %i3213 = select i1 %i2975, i32 %i3212, i32 0
- %i3214 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3213, i32 0, i32 0)
- %i3215 = shl i32 %i2870, 1
- %i3216 = select i1 %i2976, i32 %i3215, i32 0
- %i3217 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3216, i32 0, i32 0)
- %i3218 = shl i32 %i2871, 1
- %i3219 = select i1 %i2977, i32 %i3218, i32 0
- %i3220 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3219, i32 0, i32 0)
- %i3221 = shl i32 %i2872, 1
- %i3222 = select i1 %i2978, i32 %i3221, i32 0
- %i3223 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3222, i32 0, i32 0)
- %i3224 = shl i32 %i2873, 1
- %i3225 = select i1 %i2979, i32 %i3224, i32 0
- %i3226 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3225, i32 0, i32 0)
- %i3227 = shl i32 %i2874, 1
- %i3228 = select i1 %i2980, i32 %i3227, i32 0
- %i3229 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) null, i32 %i3228, i32 0, i32 0)
- %i3230 = shl i32 %i2875, 1
- %i3231 = select i1 %i2981, i32 %i3230, i32 0
- %i3232 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3231, i32 0, i32 0)
- %i3233 = shl i32 %i2876, 1
- %i3234 = select i1 %i2982, i32 %i3233, i32 0
- %i3235 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) null, i32 %i3234, i32 0, i32 0)
- %i3236 = shl i32 %i2877, 1
- %i3237 = select i1 %i2983, i32 %i3236, i32 0
- %i3238 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3237, i32 0, i32 0)
- %i3239 = shl i32 %i2878, 1
- %i3240 = select i1 %i2984, i32 %i3239, i32 0
- %i3241 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3240, i32 0, i32 0)
- %i3242 = shl i32 %i2879, 1
- %i3243 = select i1 %i2985, i32 %i3242, i32 0
- %i3244 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i2383, i32 %i3243, i32 0, i32 0)
- %i3245 = insertelement <4 x float> zeroinitializer, float %arg1062, i64 0
- %i3246 = insertelement <4 x float> %arg1063, float %arg1064, i64 0
- %i3247 = insertelement <4 x float> %arg1065, float 0.000000e+00, i64 1
- %i3248 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %arg1066, i32 0, i32 0, i32 0)
- %i3249 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %arg1067, i32 0, i32 0, i32 0)
- %i3250 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %arg1068, i32 0, i32 0, i32 0)
- %i3251 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %arg1069, i32 0, i32 0, i32 0)
- %i3252 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %arg1070, i32 0, i32 0, i32 0)
- %i3253 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %arg1071, i32 0, i32 0, i32 0)
- %i3254 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %arg1072, i32 0, i32 0, i32 0)
- %i3255 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %arg1073, i32 0, i32 0, i32 0)
- %i3256 = extractelement <4 x float> %arg1074, i64 0
- %i3257 = extractelement <4 x float> %arg1074, i64 0
- %i3258 = getelementptr inbounds nuw i8, ptr addrspace(3) null, i32 %arg832
- store i16 %i2887, ptr addrspace(3) null, align 2
- %.idx655 = shl i32 0, 0
- %i3259 = getelementptr i8, ptr addrspace(3) @global_smem, i32 %.idx655
- %i3260 = getelementptr inbounds nuw i8, ptr addrspace(3) %i3259, i32 %i2694
- store i16 %i2988, ptr addrspace(3) %i3260, align 2
- store i16 %i3012, ptr addrspace(3) null, align 2
- %i3261 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %arg1076
- store i16 %i3036, ptr addrspace(3) %i3261, align 2
- store i16 %i3057, ptr addrspace(3) %arg1075, align 2
- store i16 %i3071, ptr addrspace(3) null, align 2
- %i3262 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %arg1077
- store i16 %i3085, ptr addrspace(3) %arg1078, align 2
- store i16 %i3099, ptr addrspace(3) %arg1075, align 2
- store i16 %i3113, ptr addrspace(3) null, align 2
- %i3263 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2701
- store i16 0, ptr addrspace(3) %i3263, align 2
- %i3264 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2703
- store i16 0, ptr addrspace(3) %i3264, align 2
- %i3265 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2704
- store i16 %i3126, ptr addrspace(3) %i3265, align 2
- store i16 %i3150, ptr addrspace(3) null, align 2
- %i3266 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2705
- store i16 %i3174, ptr addrspace(3) %i3266, align 2
- %i3267 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %arg1079
- store i16 %i3198, ptr addrspace(3) %arg1080, align 2
- %i3268 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2709
- store i16 %i3217, ptr addrspace(3) %arg1081, align 2
- %i3269 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %arg1082
- store i16 %i3238, ptr addrspace(3) %arg1083, align 2
- store i16 %i2991, ptr addrspace(3) null, align 2
- %i3270 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2713
- store i16 %i3015, ptr addrspace(3) %i3270, align 2
- store i16 %i3039, ptr addrspace(3) null, align 2
- store i16 %i3059, ptr addrspace(3) %arg1075, align 2
- store i16 %i3115, ptr addrspace(3) null, align 2
- %i3271 = getelementptr inbounds nuw i8, ptr addrspace(3) %i3259, i32 %i2716
- store i16 0, ptr addrspace(3) %i3271, align 2
- %i3272 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %arg1084
- store i16 0, ptr addrspace(3) null, align 2
- %i3273 = getelementptr inbounds nuw i8, ptr addrspace(3) %i3259, i32 %i2719
- store i16 %i3129, ptr addrspace(3) %i3273, align 2
- store i16 %i3153, ptr addrspace(3) null, align 2
- %i3274 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2720
- store i16 %i3177, ptr addrspace(3) %i3274, align 2
- %i3275 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %arg1085
- store i16 %i3201, ptr addrspace(3) null, align 2
- %i3276 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %arg1086
- store i16 %i3220, ptr addrspace(3) %arg1087, align 2
- %i3277 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2726
- store i16 %i3241, ptr addrspace(3) %i3277, align 2
- store i16 %i2994, ptr addrspace(3) null, align 2
- %i3278 = getelementptr inbounds nuw i8, ptr addrspace(3) %i3259, i32 %i2729
- store i16 %i3018, ptr addrspace(3) %i3278, align 2
- store i16 %i3042, ptr addrspace(3) %arg1075, align 2
- store i16 %i3061, ptr addrspace(3) null, align 2
- %i3279 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2731
- store i16 %i3073, ptr addrspace(3) %i3279, align 2
- store i16 %i3087, ptr addrspace(3) %arg1075, align 2
- store i16 %i3101, ptr addrspace(3) null, align 2
- %i3280 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %arg1088
- store i16 %i3132, ptr addrspace(3) %i3280, align 2
- %i3281 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %arg1089
- store i16 %i3156, ptr addrspace(3) null, align 2
- %i3282 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %arg1090
- store i16 %i3180, ptr addrspace(3) %arg1091, align 2
- %i3283 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2739
- store i16 %i3204, ptr addrspace(3) %arg1092, align 2
- %i3284 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %arg1093
- store i16 %i3223, ptr addrspace(3) null, align 2
- %i3285 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %arg1094
- store i16 %i3244, ptr addrspace(3) %arg1095, align 2
- %i3286 = getelementptr inbounds nuw i8, ptr addrspace(3) %i3259, i32 %i2745
- store i16 %i2997, ptr addrspace(3) %i3286, align 2
- store i16 %i3021, ptr addrspace(3) null, align 2
- store i16 %i3045, ptr addrspace(3) %arg1075, align 2
- store i16 %i3075, ptr addrspace(3) null, align 2
- store i16 %i3089, ptr addrspace(3) %arg1075, align 2
- store i16 %i3103, ptr addrspace(3) null, align 2
- store i16 0, ptr addrspace(3) %arg1075, align 2
- %i3287 = getelementptr inbounds nuw i8, ptr addrspace(3) %i3259, i32 %i2748
- store i16 0, ptr addrspace(3) %i3287, align 2
- store i16 %i3135, ptr addrspace(3) null, align 2
- %i3288 = getelementptr inbounds nuw i8, ptr addrspace(3) %i3259, i32 %i2749
- store i16 %i3159, ptr addrspace(3) %i3288, align 2
- %i3289 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %arg1096
- store i16 %i3183, ptr addrspace(3) null, align 2
- %i3290 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2753
- store i16 %i3206, ptr addrspace(3) %i3290, align 2
- %i3291 = getelementptr inbounds nuw i8, ptr addrspace(3) %i3259, i32 %i2755
- store i16 %i3226, ptr addrspace(3) %i3291, align 2
- %i3292 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %arg1097
- store i16 0, ptr addrspace(3) null, align 2
- %i3293 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %arg1098
- store i16 %i3000, ptr addrspace(3) %arg1099, align 2
- store i16 %i3024, ptr addrspace(3) null, align 2
- %i3294 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2760
- store i16 %i3048, ptr addrspace(3) %arg1100, align 2
- store i16 %i3063, ptr addrspace(3) %arg1075, align 2
- store i16 %i3077, ptr addrspace(3) null, align 2
- store i16 %i3091, ptr addrspace(3) %arg1075, align 2
- store i16 %i3105, ptr addrspace(3) null, align 2
- %i3295 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %arg1101
- store i16 0, ptr addrspace(3) %arg1102, align 2
- %i3296 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %arg1103
- store i16 0, ptr addrspace(3) null, align 2
- store i16 %i3138, ptr addrspace(3) null, align 2
- %i3297 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2767
- store i16 %i3162, ptr addrspace(3) %i3297, align 2
- %i3298 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2769
- store i16 %i3186, ptr addrspace(3) null, align 2
- %i3299 = getelementptr inbounds nuw i8, ptr addrspace(3) %i3259, i32 %i2771
- store i16 %i3003, ptr addrspace(3) %i3299, align 2
- store i16 %i3027, ptr addrspace(3) null, align 2
- %i3300 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2773
- store i16 %i3051, ptr addrspace(3) %i3300, align 2
- store i16 %i3065, ptr addrspace(3) %arg1075, align 2
- store i16 %i3079, ptr addrspace(3) null, align 2
- store i16 %i3093, ptr addrspace(3) %arg1075, align 2
- store i16 %i3107, ptr addrspace(3) null, align 2
- %i3301 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2774
- store i16 0, ptr addrspace(3) %i3301, align 2
- %i3302 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2775
- store i16 %i3117, ptr addrspace(3) %i3302, align 2
- store i16 %i3141, ptr addrspace(3) null, align 2
- %i3303 = getelementptr inbounds nuw i8, ptr addrspace(3) %i3259, i32 %i2776
- store i16 %i3165, ptr addrspace(3) %i3303, align 2
- %i3304 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2778
- store i16 %i3189, ptr addrspace(3) %i3304, align 2
- %i3305 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2779
- store i16 %i3208, ptr addrspace(3) %i3305, align 2
- %i3306 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2781
- store i16 %i3229, ptr addrspace(3) null, align 2
- %i3307 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %arg853
- store i16 0, ptr addrspace(3) %i3307, align 2
- store i16 %i3006, ptr addrspace(3) null, align 2
- store i16 %i3030, ptr addrspace(3) %arg1075, align 2
- store i16 %arg1104, ptr addrspace(3) null, align 2
- %i3308 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2785
- store i16 %i3067, ptr addrspace(3) %i3308, align 2
- store i16 %i3081, ptr addrspace(3) null, align 2
- store i16 %i3095, ptr addrspace(3) %arg1075, align 2
- store i16 %i3109, ptr addrspace(3) null, align 2
- %i3309 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2787
- store i16 %i3120, ptr addrspace(3) %i3309, align 2
- %i3310 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2789
- store i16 %i3144, ptr addrspace(3) %i3310, align 2
- %i3311 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2790
- store i16 %i3168, ptr addrspace(3) %i3311, align 2
- %i3312 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2792
- store i16 %i3192, ptr addrspace(3) %arg1105, align 2
- %i3313 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %arg873
- store i16 %i3211, ptr addrspace(3) null, align 2
- %i3314 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %arg1106
- store i16 %i3232, ptr addrspace(3) %i3314, align 2
- %i3315 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2796
- store i16 0, ptr addrspace(3) %i3315, align 2
- store i16 %i3009, ptr addrspace(3) null, align 2
- store i16 %arg1107, ptr addrspace(3) %arg1075, align 2
- store i16 %arg1108, ptr addrspace(3) null, align 2
- %i3316 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2799
- store i16 %i3069, ptr addrspace(3) %i3316, align 2
- store i16 %i3083, ptr addrspace(3) %arg1075, align 2
- store i16 %i3097, ptr addrspace(3) null, align 2
- store i16 %i3111, ptr addrspace(3) %arg1075, align 2
- store i16 %i3123, ptr addrspace(3) null, align 2
- %i3317 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %arg890
- store i16 %i3147, ptr addrspace(3) %i3317, align 2
- %i3318 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %arg1109
- store i16 %i3171, ptr addrspace(3) %i3318, align 2
- %i3319 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %arg894
- store i16 %i3195, ptr addrspace(3) null, align 2
- %i3320 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %i2806
- store i16 %i3214, ptr addrspace(3) %arg1110, align 2
- %i3321 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1075, i32 %arg898
- store i16 %i3235, ptr addrspace(3) null, align 2
- %exitcond.not = icmp eq i32 %i2880, %smax
- br i1 %exitcond.not, label %._crit_edge.loopexit, label %bb2812
-
-._crit_edge.loopexit: ; preds = %bb2812
- br label %._crit_edge
-
-._crit_edge: ; preds = %._crit_edge.loopexit, %.._crit_edge_crit_edge
- %.pre-phi943 = phi i32 [ %.pre, %.._crit_edge_crit_edge ], [ %.decomposed, %._crit_edge.loopexit ]
- %i3322 = xor i32 %.pre-phi943, 64
- %i3323 = or disjoint i32 %i3322, 1
- %i3324 = xor i32 %.pre-phi943, 1
- %i3325 = or disjoint i32 %i3324, 1
- br i1 %i1498, label %bb3326, label %._crit_edge._crit_edge
-
-bb3326: ; preds = %._crit_edge
- %i3327 = getelementptr inbounds nuw i8, ptr addrspace(3) null, i32 %i3325
- %i3328 = load <8 x half>, ptr addrspace(3) %i3327, align 16
- %i3329 = getelementptr inbounds nuw i8, ptr addrspace(3) null, i32 %i3323
- %i3330 = load <8 x half>, ptr addrspace(3) %i3329, align 16
- ret void
-
-._crit_edge._crit_edge: ; preds = %._crit_edge
- %i3331 = icmp slt i64 %i1734, %i2054
- %i3332 = tail call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1(ptr addrspace(1) %arg2, i16 0, i64 1, i32 1)
- br i1 %arg903, label %.lr.ph923, label %._crit_edge924
-
-.lr.ph923: ; preds = %._crit_edge._crit_edge
- br label %bb3333
-
-bb3333: ; preds = %bb3333, %.lr.ph923
- %i3334 = phi i32 [ 0, %.lr.ph923 ], [ %arg139, %bb3333 ]
- %i3335 = add i32 %i3334, 1
- %.not = icmp slt i32 0, %i1497
- %i3336 = select i1 %.not, i32 %i3335, i32 0
- %i3337 = shl i32 %i3336, 1
- %i3338 = or disjoint i32 %arg45, %arg4
- %i3339 = or disjoint i32 %arg4, %arg22
- %i3340 = or disjoint i32 %i3337, %i1364
- %i3341 = or disjoint i32 %i3337, %arg39
- %i3342 = or disjoint i32 %arg7, %arg4
- %i3343 = or disjoint i32 %arg4, %arg4
- %i3344 = or disjoint i32 %arg4, %arg43
- %i3345 = or disjoint i32 %i3337, %arg46
- %i3346 = or disjoint i32 %i3337, %arg47
- %i3347 = or disjoint i32 %arg51, %arg4
- %i3348 = or disjoint i32 0, %arg4
- %i3349 = or disjoint i32 %i3337, %i1377
- %i3350 = or disjoint i32 %i3337, %arg1213
- %i3351 = or disjoint i32 %i3337, %arg48
- %i3352 = or disjoint i32 %arg4, %arg4
- %i3353 = or disjoint i32 %arg4, %arg58
- %i3354 = or disjoint i32 %i3337, %arg56
- %i3355 = or disjoint i32 %i3337, %arg928
- %i3356 = or disjoint i32 %i3337, %arg130
- %i3357 = or disjoint i32 %arg4, %arg4
- %i3358 = or disjoint i32 %i3337, %arg120
- %i3359 = or disjoint i32 %i3337, %arg766
- %i3360 = or disjoint i32 %i3337, %arg45
- %i3361 = or disjoint i32 %i3337, %arg82
- %i3362 = or disjoint i32 %arg4, %arg4
- %i3363 = or disjoint i32 %i3337, %arg63
- %i3364 = or disjoint i32 %i3337, %arg51
- %i3365 = or disjoint i32 %arg4, %arg4
- %i3366 = or disjoint i32 %i3337, %arg79
- %i3367 = or disjoint i32 %arg54, %arg4
- %i3368 = or disjoint i32 %arg4, %arg4
- %i3369 = or disjoint i32 %i3337, %arg65
- %i3370 = or disjoint i32 %i3337, %arg86
- %i3371 = or disjoint i32 %arg20, %arg4
- %i3372 = or disjoint i32 %arg4, %arg4
- %i3373 = or disjoint i32 %arg4, %arg4
- %i3374 = or disjoint i32 %arg4, %arg4
- %i3375 = or disjoint i32 0, 0
- %i3376 = or disjoint i32 %arg4, %arg4
- %i3377 = or disjoint i32 0, 0
- %i3378 = or disjoint i32 %arg4, 1
- %i3379 = or disjoint i32 %i3337, 1
- %i3380 = or disjoint i32 %arg25, 0
- %i3381 = or disjoint i32 0, 0
- %i3382 = or disjoint i32 %i3337, %arg963
- %i3383 = or disjoint i32 %arg4, 0
- %i3384 = or disjoint i32 %i3337, %arg96
- %i3385 = or disjoint i32 %arg40, %arg4
- %i3386 = or disjoint i32 %arg4, %arg4
- %i3387 = or disjoint i32 %arg4, %arg4
- %i3388 = or disjoint i32 %arg4, %arg4
- %i3389 = or disjoint i32 %arg4, %arg4
- %i3390 = or disjoint i32 %arg4, %arg4
- %i3391 = or disjoint i32 %arg4, %arg4
- %i3392 = or disjoint i32 %arg4, %arg4
- %i3393 = or disjoint i32 %arg4, %arg4
- %i3394 = or disjoint i32 0, 0
- %i3395 = or disjoint i32 %arg4, %arg4
- %i3396 = or disjoint i32 %arg4, %arg4
- %i3397 = or disjoint i32 %arg4, %arg4
- %i3398 = or disjoint i32 %arg4, %arg4
- %i3399 = or disjoint i32 %arg4, %arg4
- %i3400 = or disjoint i32 %arg4, %arg4
- %i3401 = or disjoint i32 %arg4, %arg4
- %i3402 = or disjoint i32 %arg4, %arg4
- %i3403 = or disjoint i32 %arg4, %arg4
- %i3404 = or disjoint i32 %arg4, %arg4
- %i3405 = or disjoint i32 %arg4, %arg4
- %i3406 = or disjoint i32 %arg4, %arg4
- %i3407 = or disjoint i32 %arg4, %arg4
- %i3408 = or disjoint i32 %arg4, %arg4
- %i3409 = or disjoint i32 %arg4, %arg4
- %i3410 = or disjoint i32 %arg4, %arg4
- %i3411 = or disjoint i32 1, 1
- %i3412 = icmp slt i32 %arg4, %arg4
- %i3413 = icmp slt i32 %arg4, %arg6
- %i3414 = icmp slt i32 %i3340, %arg6
- %i3415 = icmp slt i32 %i3341, %arg6
- %i3416 = icmp slt i32 %arg9, %arg4
- %i3417 = icmp slt i32 %arg4, %arg4
- %i3418 = icmp slt i32 %arg4, %arg6
- %i3419 = icmp slt i32 %i3345, %arg6
- %i3420 = icmp slt i32 %i3346, 1
- %i3421 = icmp slt i32 %arg16, %arg4
- %i3422 = icmp slt i32 %arg4, %arg4
- %i3423 = icmp slt i32 %i3349, %arg6
- %i3424 = icmp slt i32 %i3350, %arg6
- %i3425 = icmp slt i32 %i3351, %arg6
- %i3426 = icmp slt i32 %arg12, %arg4
- %i3427 = icmp slt i32 %arg4, %arg6
- %i3428 = icmp slt i32 %i3354, %arg6
- %i3429 = icmp slt i32 %i3355, %arg6
- %i3430 = icmp slt i32 %i3356, %arg6
- %i3431 = icmp slt i32 %arg61, %arg6
- %i3432 = icmp slt i32 %i3358, %arg6
- %i3433 = icmp slt i32 %i3359, %arg6
- %i3434 = icmp slt i32 %i3360, %arg6
- %i3435 = icmp slt i32 %i3361, 1
- %i3436 = icmp slt i32 %arg44, %arg6
- %i3437 = icmp slt i32 %i3363, %arg6
- %i3438 = icmp slt i32 %i3364, %arg6
- %i3439 = icmp slt i32 %arg57, %arg6
- %i3440 = icmp slt i32 %i3366, %arg6
- %i3441 = icmp slt i32 %arg25, %arg4
- %i3442 = icmp slt i32 %arg4, %arg6
- %i3443 = icmp slt i32 %i3369, %arg6
- %i3444 = icmp slt i32 %i3370, %arg6
- %i3445 = icmp slt i32 %arg18, %arg4
- %i3446 = icmp slt i32 %arg4, %arg4
- %i3447 = icmp slt i32 %arg4, %arg4
- %i3448 = icmp slt i32 %arg4, 0
- %i3449 = icmp slt i32 0, 0
- %i3450 = icmp slt i32 %arg4, 0
- %i3451 = icmp slt i32 0, 0
- %i3452 = icmp slt i32 %arg4, 0
- %i3453 = icmp slt i32 %i3379, %arg6
- %i3454 = icmp slt i32 %arg7, 0
- %i3455 = icmp slt i32 0, 0
- %i3456 = icmp slt i32 %i3382, 0
- %i3457 = icmp slt i32 %arg39, %arg6
- %i3458 = icmp slt i32 %i3384, %arg6
- %i3459 = icmp slt i32 %arg54, 0
- %i3460 = icmp slt i32 %arg4, %arg4
- %i3461 = icmp slt i32 %arg4, 0
- %i3462 = icmp slt i32 %arg4, 0
- %i3463 = icmp slt i32 %arg4, %arg4
- %i3464 = icmp slt i32 %arg4, 0
- %i3465 = icmp slt i32 %arg4, %arg4
- %i3466 = icmp slt i32 %arg4, 0
- %i3467 = icmp slt i32 %arg4, 0
- %i3468 = icmp slt i32 0, 0
- %i3469 = icmp slt i32 %arg4, %arg4
- %i3470 = icmp slt i32 %arg4, 0
- %i3471 = icmp slt i32 %arg4, %arg4
- %i3472 = icmp slt i32 %arg4, 0
- %i3473 = icmp slt i32 %arg4, 0
- %i3474 = icmp slt i32 %arg4, 0
- %i3475 = icmp slt i32 %arg4, 0
- %i3476 = icmp slt i32 %arg4, %arg4
- %i3477 = icmp slt i32 %arg4, 0
- %i3478 = icmp slt i32 %arg4, 0
- %i3479 = icmp slt i32 %arg4, 0
- %i3480 = icmp slt i32 %arg4, 0
- %i3481 = icmp slt i32 %arg4, %arg4
- %i3482 = icmp slt i32 %arg4, 0
- %i3483 = icmp slt i32 %arg4, %arg4
- %i3484 = icmp slt i32 %arg4, 0
- %i3485 = icmp slt i32 1, 0
- %i3486 = and i1 %arg580, %arg580
- %i3487 = and i1 %arg580, %arg580
- %i3488 = and i1 %i3331, %i3414
- %i3489 = and i1 %arg1160, %i3415
- %i3490 = and i1 %arg1160, %arg621
- %i3491 = and i1 %arg580, %arg580
- %i3492 = and i1 %arg580, %arg580
- %i3493 = and i1 %arg1160, %i3419
- %i3494 = and i1 %arg1160, %i3420
- %i3495 = and i1 %arg605, %arg580
- %i3496 = and i1 true, %arg580
- %i3497 = and i1 %arg1160, %i3423
- %i3498 = and i1 %arg1160, %i3424
- %i3499 = and i1 %arg1160, %i3425
- %i3500 = and i1 %arg585, %arg580
- %i3501 = and i1 %arg580, %arg580
- %i3502 = and i1 %arg1160, %i3428
- %i3503 = and i1 %arg1160, %i3429
- %i3504 = and i1 %arg1160, %i3430
- %i3505 = and i1 %arg590, %arg580
- %i3506 = and i1 %arg1160, %i3432
- %i3507 = and i1 %arg1160, %i3433
- %i3508 = and i1 %arg1160, %i3434
- %i3509 = and i1 %arg1160, %i3435
- %i3510 = and i1 %arg609, %arg580
- %i3511 = and i1 %arg1160, %i3437
- %i3512 = and i1 %arg1160, %i3438
- %i3513 = and i1 %arg593, %arg580
- %i3514 = and i1 %arg1160, %i3440
- %i3515 = and i1 %arg609, %arg580
- %i3516 = and i1 %arg580, %arg580
- %i3517 = and i1 %arg1160, %i3443
- %i3518 = and i1 %arg1160, %i3444
- %i3519 = and i1 %arg588, %arg580
- %i3520 = and i1 %arg580, %arg580
- %i3521 = and i1 %arg580, %arg580
- %i3522 = and i1 %arg580, %arg580
- %i3523 = and i1 false, false
- %i3524 = and i1 %arg580, %arg580
- %i3525 = and i1 false, false
- %i3526 = and i1 %arg580, %arg580
- %i3527 = and i1 %arg1160, %i3453
- %i3528 = and i1 %arg619, %arg580
- %i3529 = and i1 false, false
- %i3530 = and i1 %arg1160, %i3456
- %i3531 = and i1 %arg626, %arg580
- %i3532 = and i1 %arg1160, %i3458
- %i3533 = and i1 %arg582, %arg580
- %i3534 = and i1 %arg580, %arg580
- %i3535 = and i1 %arg580, %arg580
- %i3536 = and i1 %arg580, %arg580
- %i3537 = and i1 %arg580, %arg580
- %i3538 = and i1 %arg580, %arg580
- %i3539 = and i1 %arg580, %arg580
- %i3540 = and i1 %arg580, %arg580
- %i3541 = and i1 %arg580, %arg580
- %i3542 = and i1 false, false
- %i3543 = and i1 %arg580, %arg580
- %i3544 = and i1 %arg580, %arg580
- %i3545 = and i1 %arg580, %arg580
- %i3546 = and i1 %arg580, %arg580
- %i3547 = and i1 %arg580, %arg580
- %i3548 = and i1 %arg580, %arg580
- %i3549 = and i1 %arg580, %arg580
- %i3550 = and i1 %arg580, %arg580
- %i3551 = and i1 %arg580, %arg580
- %i3552 = and i1 %arg580, %arg580
- %i3553 = and i1 %arg580, %arg580
- %i3554 = and i1 %arg580, %arg580
- %i3555 = and i1 %arg580, %arg580
- %i3556 = and i1 %arg580, %arg580
- %i3557 = and i1 %arg580, %arg580
- %i3558 = and i1 %arg580, %arg580
- %i3559 = and i1 true, true
- %i3560 = select i1 %arg580, i32 0, i32 -2147483648
- %i3561 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3562 = select i1 %arg580, i32 0, i32 -2147483648
- %i3563 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3564 = select i1 %i3488, i32 0, i32 -2147483648
- %i3565 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i3332, i32 %i3564, i32 0, i32 0)
- %i3566 = select i1 %i3489, i32 0, i32 -2147483648
- %i3567 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i3332, i32 %i3566, i32 0, i32 0)
- %i3568 = select i1 %i3490, i32 0, i32 -2147483648
- %i3569 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i3332, i32 %i3568, i32 0, i32 0)
- %i3570 = select i1 %arg620, i32 0, i32 -2147483648
- %i3571 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3572 = select i1 %arg580, i32 0, i32 -2147483648
- %i3573 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3574 = select i1 %i3493, i32 0, i32 -2147483648
- %i3575 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i3332, i32 %i3574, i32 0, i32 0)
- %i3576 = select i1 %i3494, i32 0, i32 -2147483648
- %i3577 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i3332, i32 %i3576, i32 0, i32 0)
- %i3578 = select i1 %arg621, i32 0, i32 -2147483648
- %i3579 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg1208, i32 %arg22, i32 0, i32 0)
- %i3580 = select i1 %arg580, i32 0, i32 1
- %i3581 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) null, i32 %arg4, i32 0, i32 0)
- %i3582 = select i1 %i3497, i32 0, i32 -2147483648
- %i3583 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i3332, i32 %i3582, i32 0, i32 0)
- %i3584 = select i1 %i3498, i32 0, i32 -2147483648
- %i3585 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i3332, i32 %i3584, i32 0, i32 0)
- %i3586 = select i1 %i3499, i32 0, i32 -2147483648
- %i3587 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i3332, i32 %i3586, i32 0, i32 0)
- %i3588 = select i1 %arg582, i32 0, i32 1
- %i3589 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3590 = select i1 %arg580, i32 0, i32 -2147483648
- %i3591 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3592 = select i1 %i3502, i32 0, i32 -2147483648
- %i3593 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i3332, i32 %i3592, i32 0, i32 0)
- %i3594 = select i1 %i3503, i32 0, i32 -2147483648
- %i3595 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i3332, i32 %i3594, i32 0, i32 0)
- %i3596 = select i1 %i3504, i32 0, i32 -2147483648
- %i3597 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i3332, i32 %i3596, i32 0, i32 0)
- %i3598 = select i1 %arg583, i32 0, i32 -2147483648
- %i3599 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3600 = select i1 %i3506, i32 0, i32 -2147483648
- %i3601 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i3332, i32 %i3600, i32 0, i32 0)
- %i3602 = select i1 %i3507, i32 0, i32 -2147483648
- %i3603 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i3332, i32 %i3602, i32 0, i32 0)
- %i3604 = select i1 %i3508, i32 0, i32 -2147483648
- %i3605 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i3332, i32 %i3604, i32 0, i32 0)
- %i3606 = select i1 %i3509, i32 0, i32 -2147483648
- %i3607 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i3332, i32 %i3606, i32 0, i32 0)
- %i3608 = select i1 %arg626, i32 0, i32 -2147483648
- %i3609 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3610 = select i1 %i3511, i32 0, i32 -2147483648
- %i3611 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i3332, i32 %i3610, i32 0, i32 0)
- %i3612 = select i1 %i3512, i32 0, i32 -2147483648
- %i3613 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i3332, i32 %i3612, i32 0, i32 0)
- %i3614 = select i1 %arg582, i32 0, i32 -2147483648
- %i3615 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3616 = select i1 %i3514, i32 0, i32 -2147483648
- %i3617 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i3332, i32 %i3616, i32 0, i32 0)
- %i3618 = select i1 %arg619, i32 0, i32 -2147483648
- %i3619 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3620 = select i1 %arg580, i32 0, i32 -2147483648
- %i3621 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3622 = select i1 %i3517, i32 0, i32 -2147483648
- %i3623 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i3332, i32 %i3622, i32 0, i32 0)
- %i3624 = select i1 %i3518, i32 0, i32 -2147483648
- %i3625 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i3332, i32 %i3624, i32 0, i32 0)
- %i3626 = select i1 %arg583, i32 0, i32 -2147483648
- %i3627 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3628 = select i1 %arg580, i32 0, i32 1
- %i3629 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3630 = select i1 %arg580, i32 0, i32 -2147483648
- %i3631 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg1208, i32 %arg101, i32 0, i32 0)
- %i3632 = select i1 %arg580, i32 0, i32 -2147483648
- %i3633 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg1208, i32 %arg56, i32 0, i32 0)
- %i3634 = select i1 false, i32 0, i32 0
- %i3635 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) null, i32 0, i32 0, i32 0)
- %i3636 = select i1 %arg580, i32 0, i32 -2147483648
- %i3637 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg1208, i32 %arg944, i32 0, i32 0)
- %i3638 = select i1 false, i32 0, i32 0
- %i3639 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) null, i32 0, i32 0, i32 0)
- %i3640 = select i1 %arg580, i32 0, i32 -2147483648
- %i3641 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg1208, i32 %arg60, i32 0, i32 0)
- %i3642 = select i1 %i3527, i32 0, i32 -2147483648
- %i3643 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i3332, i32 %i3642, i32 0, i32 0)
- %i3644 = select i1 %arg582, i32 0, i32 -2147483648
- %i3645 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3646 = select i1 false, i32 0, i32 0
- %i3647 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) null, i32 0, i32 0, i32 0)
- %i3648 = select i1 %i3530, i32 0, i32 -2147483648
- %i3649 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i3332, i32 %i3648, i32 0, i32 0)
- %i3650 = select i1 %arg600, i32 0, i32 -2147483648
- %i3651 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg1208, i32 %arg96, i32 0, i32 0)
- %i3652 = select i1 %i3532, i32 0, i32 -2147483648
- %i3653 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %i3332, i32 %i3652, i32 0, i32 0)
- %i3654 = select i1 %arg616, i32 0, i32 -2147483648
- %i3655 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3656 = select i1 %arg580, i32 0, i32 -2147483648
- %i3657 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3658 = select i1 %arg580, i32 0, i32 -2147483648
- %i3659 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg1208, i32 %arg752, i32 0, i32 0)
- %i3660 = select i1 %arg580, i32 0, i32 -2147483648
- %i3661 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg1208, i32 %arg31, i32 0, i32 0)
- %i3662 = select i1 %arg580, i32 0, i32 -2147483648
- %i3663 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3664 = select i1 %arg580, i32 0, i32 -2147483648
- %i3665 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg1208, i32 %arg107, i32 0, i32 0)
- %i3666 = select i1 %arg580, i32 0, i32 -2147483648
- %i3667 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3668 = select i1 %arg580, i32 0, i32 -2147483648
- %i3669 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg1208, i32 %arg70, i32 0, i32 0)
- %i3670 = select i1 %arg580, i32 0, i32 1
- %i3671 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg1208, i32 %arg49, i32 0, i32 0)
- %i3672 = select i1 false, i32 0, i32 0
- %i3673 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) null, i32 0, i32 0, i32 0)
- %i3674 = select i1 %arg580, i32 0, i32 -2147483648
- %i3675 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg1208, i32 %arg15, i32 0, i32 0)
- %i3676 = select i1 %arg580, i32 0, i32 -2147483648
- %i3677 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3678 = select i1 %arg580, i32 0, i32 -2147483648
- %i3679 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3680 = select i1 %arg580, i32 0, i32 -2147483648
- %i3681 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg1208, i32 %arg95, i32 0, i32 0)
- %i3682 = select i1 %arg580, i32 0, i32 -2147483648
- %i3683 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg1208, i32 %arg4, i32 0, i32 0)
- %i3684 = select i1 %arg580, i32 0, i32 -2147483648
- %i3685 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3686 = select i1 %arg580, i32 0, i32 -2147483648
- %i3687 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3688 = select i1 %arg580, i32 0, i32 -2147483648
- %i3689 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg1208, i32 %arg59, i32 0, i32 0)
- %i3690 = select i1 %arg580, i32 0, i32 -2147483648
- %i3691 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg1208, i32 %arg71, i32 0, i32 0)
- %i3692 = select i1 %arg580, i32 0, i32 -2147483648
- %i3693 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg1208, i32 %arg19, i32 0, i32 0)
- %i3694 = select i1 %arg580, i32 0, i32 -2147483648
- %i3695 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3696 = select i1 %arg580, i32 0, i32 -2147483648
- %i3697 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3698 = select i1 %arg580, i32 0, i32 -2147483648
- %i3699 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg1208, i32 %arg95, i32 0, i32 0)
- %i3700 = select i1 %arg580, i32 0, i32 -2147483648
- %i3701 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg1208, i32 %arg96, i32 0, i32 0)
- %i3702 = select i1 %arg580, i32 0, i32 -2147483648
- %i3703 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg1208, i32 %arg95, i32 0, i32 0)
- %i3704 = select i1 %arg580, i32 0, i32 -2147483648
- %i3705 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg34, i32 %arg4, i32 0, i32 0)
- %i3706 = select i1 true, i32 0, i32 1
- %i3707 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %arg1208, i32 1, i32 0, i32 0)
- %i3708 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> zeroinitializer, i32 0, i32 0, i32 0)
- %i3709 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> zeroinitializer, i32 0, i32 0, i32 0)
- %i3710 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> zeroinitializer, i32 0, i32 0, i32 0)
- %i3711 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> zeroinitializer, i32 0, i32 0, i32 0)
- %i3712 = shufflevector <4 x float> zeroinitializer, <4 x float> zeroinitializer, <2 x i32> <i32 0, i32 1>
- %i3713 = fptrunc <2 x float> zeroinitializer to <2 x half>
- %i3714 = shufflevector <2 x half> zeroinitializer, <2 x half> zeroinitializer, <2 x i32> <i32 0, i32 2>
- store <2 x half> zeroinitializer, ptr addrspace(3) null, align 4
- %.idx46 = shl i32 0, 0
- %i3715 = getelementptr i8, ptr addrspace(3) %global_smem, i32 %.idx
- %i3716 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %arg856
- store i16 %arg1301, ptr addrspace(3) %arg1302, align 2
- store i16 %arg1303, ptr addrspace(3) null, align 2
- %i3717 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %arg1076
- store i16 %arg1304, ptr addrspace(3) %arg1305, align 2
- store i16 %arg1306, ptr addrspace(3) %arg1300, align 2
- store i16 %arg1307, ptr addrspace(3) null, align 2
- %i3718 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %arg1077
- store i16 %arg1308, ptr addrspace(3) %arg1309, align 2
- store i16 %arg1310, ptr addrspace(3) %arg1300, align 2
- store i16 %arg1311, ptr addrspace(3) null, align 2
- %i3719 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %arg1312
- store i16 0, ptr addrspace(3) %arg1313, align 2
- %i3720 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %arg1314
- store i16 %arg1315, ptr addrspace(3) %arg1316, align 2
- %i3721 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %arg1317
- store i16 %arg1318, ptr addrspace(3) %arg1319, align 2
- store i16 %arg1320, ptr addrspace(3) null, align 2
- %i3722 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %arg1321
- store i16 %arg1322, ptr addrspace(3) %arg1323, align 2
- %i3723 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %arg1079
- store i16 %arg1324, ptr addrspace(3) %arg1325, align 2
- %i3724 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %arg1326
- store i16 %arg1327, ptr addrspace(3) null, align 2
- %i3725 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %arg1082
- store i16 1, ptr addrspace(3) %arg1328, align 2
- %i3726 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %arg832
- store i16 %arg1329, ptr addrspace(3) %arg1330, align 2
- store i16 %arg1331, ptr addrspace(3) null, align 2
- %i3727 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %arg1332
- store i16 %i3583, ptr addrspace(3) %i3727, align 2
- store i16 %i3593, ptr addrspace(3) %i3715, align 2
- store i16 %i3601, ptr addrspace(3) null, align 2
- store i16 %i3611, ptr addrspace(3) %arg1300, align 2
- store i16 %arg1333, ptr addrspace(3) null, align 2
- store i16 %arg1334, ptr addrspace(3) %arg1300, align 2
- %i3728 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %i2716
- store i16 %i3637, ptr addrspace(3) %i3728, align 2
- store i16 0, ptr addrspace(3) null, align 2
- %i3729 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %i2719
- store i16 %i3659, ptr addrspace(3) %i3729, align 2
- store i16 %i3669, ptr addrspace(3) %arg1300, align 2
- store i16 %arg1335, ptr addrspace(3) null, align 2
- %i3730 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %arg1085
- store i16 %i3689, ptr addrspace(3) %arg1336, align 2
- %i3731 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %i2724
- store i16 %i3699, ptr addrspace(3) %i3731, align 2
- store i16 0, ptr addrspace(3) null, align 2
- %i3732 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %i2748
- store i16 %i3649, ptr addrspace(3) %i3732, align 2
- store i16 %i3661, ptr addrspace(3) null, align 2
- %i3733 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %i2749
- store i16 %i3671, ptr addrspace(3) %i3733, align 2
- %i3734 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %i2751
- store i16 %i3681, ptr addrspace(3) %i3734, align 2
- store i16 %i3691, ptr addrspace(3) null, align 2
- %i3735 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %i2755
- store i16 %i3701, ptr addrspace(3) %i3735, align 2
- store i16 %i3565, ptr addrspace(3) null, align 2
- store i16 %i3575, ptr addrspace(3) %arg1300, align 2
- %i3736 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %arg1337
- store i16 %i3585, ptr addrspace(3) %arg1338, align 2
- store i16 %i3595, ptr addrspace(3) null, align 2
- %i3737 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %arg1339
- store i16 %i3603, ptr addrspace(3) %arg1340, align 2
- store i16 %i3613, ptr addrspace(3) %arg1300, align 2
- store i16 %i3623, ptr addrspace(3) null, align 2
- store i16 %i3631, ptr addrspace(3) %arg1300, align 2
- store i16 %i3641, ptr addrspace(3) null, align 2
- %i3738 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %arg1103
- store i16 %i3651, ptr addrspace(3) %i3738, align 2
- store i16 %arg1341, ptr addrspace(3) %arg1300, align 2
- store i16 0, ptr addrspace(3) null, align 2
- %i3739 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %arg1342
- store i16 %i3683, ptr addrspace(3) null, align 2
- store i16 %i3693, ptr addrspace(3) %arg1300, align 2
- store i16 %i3703, ptr addrspace(3) null, align 2
- %i3740 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %i2771
- store i16 %i3567, ptr addrspace(3) %i3740, align 2
- store i16 %i3577, ptr addrspace(3) null, align 2
- %i3741 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %i2773
- store i16 %i3587, ptr addrspace(3) %i3741, align 2
- store i16 %i3597, ptr addrspace(3) %arg1300, align 2
- store i16 %i3605, ptr addrspace(3) null, align 2
- store i16 %arg1343, ptr addrspace(3) %arg1300, align 2
- store i16 %i3625, ptr addrspace(3) null, align 2
- store i16 %i3633, ptr addrspace(3) %arg1300, align 2
- %i3742 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %i2774
- store i16 %i3643, ptr addrspace(3) %i3742, align 2
- store i16 %i3653, ptr addrspace(3) null, align 2
- store i16 %i3665, ptr addrspace(3) %arg1300, align 2
- %i3743 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %i2776
- store i16 %i3675, ptr addrspace(3) %i3743, align 2
- store i16 %arg1344, ptr addrspace(3) null, align 2
- %i3744 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %arg849
- store i16 %arg1345, ptr addrspace(3) %i3744, align 2
- %i3745 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %arg1346
- store i16 %arg1347, ptr addrspace(3) null, align 2
- %i3746 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %i2782
- store i16 0, ptr addrspace(3) %i3746, align 2
- %i3747 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %i2784
- store i16 %i3569, ptr addrspace(3) %i3747, align 2
- store i16 %i3579, ptr addrspace(3) null, align 2
- store i16 %arg1348, ptr addrspace(3) %arg1300, align 2
- store i16 %i3607, ptr addrspace(3) null, align 2
- store i16 %i3617, ptr addrspace(3) %arg1300, align 2
- %i3748 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg1300, i32 %arg867
- store i16 %arg1349, ptr addrspace(3) null, align 2
- br label %bb3333
-
-._crit_edge924: ; preds = %._crit_edge._crit_edge
- ret void
-}
-
-; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
-declare noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x() #1
-
-; Function Attrs: convergent nocallback nocreateundeforpoison nofree nounwind willreturn memory(none)
-declare i32 @llvm.amdgcn.readfirstlane.i32(i32) #2
-
-; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
-declare noundef i32 @llvm.amdgcn.workgroup.id.x() #1
-
-; Function Attrs: nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none)
-declare ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1(ptr addrspace(1) readnone, i16, i64, i32) #3
-
-; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: read)
-declare i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) readonly captures(none), i32, i32, i32 immarg) #4
-
-; Function Attrs: nocallback nofree nounwind willreturn
-declare void @llvm.amdgcn.s.waitcnt(i32 immarg) #5
-
-; Function Attrs: convergent nocallback nofree nounwind willreturn
-declare void @llvm.amdgcn.s.barrier() #6
-
-; Function Attrs: convergent nocallback nocreateundeforpoison nofree nosync nounwind willreturn memory(none)
-declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half>, <8 x half>, <4 x float>, i32 immarg, i32 immarg, i32 immarg) #7
-
-; Function Attrs: nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none)
-declare float @llvm.amdgcn.exp2.f32(float) #3
-
-; Function Attrs: convergent nocallback nofree nounwind willreturn memory(argmem: read)
-declare <4 x half> @llvm.amdgcn.ds.read.tr16.b64.v4f16(ptr addrspace(3) captures(none)) #8
-
-; Function Attrs: convergent nocallback nofree nounwind willreturn memory(none)
-declare i32 @llvm.amdgcn.update.dpp.i32(i32, i32, i32 immarg, i32 immarg, i32 immarg, i1 immarg) #9
-
-declare i32 @llvm.smax.i32(i32, i32) #3
-
-attributes #0 = { nofree norecurse nounwind "amdgpu-agpr-alloc"="0" "amdgpu-cluster-dims"="1,1,1" "amdgpu-flat-work-group-size"="1,512" "amdgpu-no-cluster-id-x" "amdgpu-no-cluster-id-y" "amdgpu-no-cluster-id-z" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="3, 3" "uniform-work-group-size" }
-attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
-attributes #2 = { convergent nocallback nocreateundeforpoison nofree nounwind willreturn memory(none) }
-attributes #3 = { nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none) }
-attributes #4 = { nocallback nofree nosync nounwind willreturn memory(argmem: read) }
-attributes #5 = { nocallback nofree nounwind willreturn }
-attributes #6 = { convergent nocallback nofree nounwind willreturn }
-attributes #7 = { convergent nocallback nocreateundeforpoison nofree nosync nounwind willreturn memory(none) }
-attributes #8 = { convergent nocallback nofree nounwind willreturn memory(argmem: read) }
-attributes #9 = { convergent nocallback nofree nounwind willreturn memory(none) }
diff --git a/llvm/test/CodeGen/AMDGPU/regalloc-hoist-spill-live-range-upd.mir b/llvm/test/CodeGen/AMDGPU/regalloc-hoist-spill-live-range-upd.mir
new file mode 100644
index 0000000000000..30d78f93df40d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/regalloc-hoist-spill-live-range-upd.mir
@@ -0,0 +1,71 @@
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -passes='greedy<wwm>' -verify-regalloc -o - %s | FileCheck %s
+#
+# The gist of this test is we want something that triggers the
+# hoistSpillInsideBB spilling optimization placement, while spilling a value
+# that is defined by a copy inserted by live-range splitting within the
+# prologue of the basic block (i.e., where we do our exec masks shenanigans).
+#
+# Since the spiller cannot insert the spill code within that prologue, it
+# needs to extend the live-range through the prologue region and that test
+# captures that.
+#
+# The verify-regalloc flag makes sure the live-ranges are properly updated.
+# Before it would complain about missing live segment.
+#
+# The test looks like this at a high level:
+#
+# bb:
+# %1 = COPY %0 #copy in the prologue
+# ... # prologue instruction
+# ... # high pressure point
+# use %1
+#
+# And with hoistSpillInsideBB we will generate:
+# bb:
+# -- # %1 = COPY %0 gets removed
+# ... # prologue instruction
+# store %0 # %0 liveness is extended to this point.
+# ... # high pressure point
+# %1_reload = load
+# use %1_reload
+#
+# To be able to hit the problematic code in hoistSpillInsideBB, %0 needs to
+# be defined by a PHI (hence the diamond with two %0 defs in the test case)
+# and %0 and %1 need to be sibling registers (hence the %3 split-from).
+
+# CHECK-LABEL: name: prologue_kill_256
+# CHECK: SI_SPILL_WWM_V32_SAVE %0
+
+---
+name: prologue_kill_256
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ scratchRSrcReg: '$sgpr96_sgpr97_sgpr98_sgpr99'
+ stackPtrOffsetReg: '$sgpr32'
+registers:
+ - { id: 0, class: vgpr_32, flags: [ WWM_REG ], split-from: '%2' }
+ - { id: 1, class: vgpr_32, flags: [ WWM_REG ], split-from: '%2' }
+ - { id: 2, class: vgpr_32, flags: [ WWM_REG ] }
+body: |
+ bb.0:
+ successors: %bb.1, %bb.2
+ liveins: $sgpr0_sgpr1, $scc
+ %2:vgpr_32 = KILL
+ %0:vgpr_32 = COPY %2
+ S_CBRANCH_SCC1 %bb.2, implicit $scc
+
+ bb.1:
+ successors: %bb.2
+ liveins: $sgpr0_sgpr1
+ %0:vgpr_32 = COPY %2
+
+ bb.2:
+ liveins: $sgpr0_sgpr1
+ %1:vgpr_32 = lr-split WWM_COPY %0:vgpr_32
+ $exec = S_AND_B64 $exec, killed $sgpr0_sgpr1, implicit-def $scc
+ KILL implicit-def $vgpr0, implicit-def $vgpr1, implicit-def $vgpr2, implicit-def $vgpr3, implicit-def $vgpr4, implicit-def $vgpr5, implicit-def $vgpr6, implicit-def $vgpr7, implicit-def $vgpr8, implicit-def $vgpr9, implicit-def $vgpr10, implicit-def $vgpr11, implicit-def $vgpr12, implicit-def $vgpr13, implicit-def $vgpr14, implicit-def $vgpr15, implicit-def $vgpr16, implicit-def $vgpr17, implicit-def $vgpr18, implicit-def $vgpr19, implicit-def $vgpr20, implicit-def $vgpr21, implicit-def $vgpr22, implicit-def $vgpr23, implicit-def $vgpr24, implicit-def $vgpr25, implicit-def $vgpr26, implicit-def $vgpr27, implicit-def $vgpr28, implicit-def $vgpr29, implicit-def $vgpr30, implicit-def $vgpr31, implicit-def $vgpr32, implicit-def $vgpr33, implicit-def $vgpr34, implicit-def $vgpr35, implicit-def $vgpr36, implicit-def $vgpr37, implicit-def $vgpr38, implicit-def $vgpr39, implicit-def $vgpr40, implicit-def $vgpr41, implicit-def $vgpr42, implicit-def $vgpr43, implicit-def $vgpr44, implicit-def $vgpr45, implicit-def $vgpr46, implicit-def $vgpr47, implicit-def $vgpr48, implicit-def $vgpr49, implicit-def $vgpr50, implicit-def $vgpr51, implicit-def $vgpr52, implicit-def $vgpr53, implicit-def $vgpr54, implicit-def $vgpr55, implicit-def $vgpr56, implicit-def $vgpr57, implicit-def $vgpr58, implicit-def $vgpr59, implicit-def $vgpr60, implicit-def $vgpr61, implicit-def $vgpr62, implicit-def $vgpr63, implicit-def $vgpr64, implicit-def $vgpr65, implicit-def $vgpr66, implicit-def $vgpr67, implicit-def $vgpr68, implicit-def $vgpr69, implicit-def $vgpr70, implicit-def $vgpr71, implicit-def $vgpr72, implicit-def $vgpr73, implicit-def $vgpr74, implicit-def $vgpr75, implicit-def $vgpr76, implicit-def $vgpr77, implicit-def $vgpr78, implicit-def $vgpr79, implicit-def $vgpr80, implicit-def $vgpr81, implicit-def $vgpr82, implicit-def $vgpr83, implicit-def $vgpr84, implicit-def $vgpr85, implicit-def $vgpr86, implicit-def $vgpr87, implicit-def $vgpr88, implicit-def $vgpr89, implicit-def $vgpr90, implicit-def $vgpr91, implicit-def $vgpr92, implicit-def $vgpr93, implicit-def $vgpr94, implicit-def $vgpr95, implicit-def $vgpr96, implicit-def $vgpr97, implicit-def $vgpr98, implicit-def $vgpr99, implicit-def $vgpr100, implicit-def $vgpr101, implicit-def $vgpr102, implicit-def $vgpr103, implicit-def $vgpr104, implicit-def $vgpr105, implicit-def $vgpr106, implicit-def $vgpr107, implicit-def $vgpr108, implicit-def $vgpr109, implicit-def $vgpr110, implicit-def $vgpr111, implicit-def $vgpr112, implicit-def $vgpr113, implicit-def $vgpr114, implicit-def $vgpr115, implicit-def $vgpr116, implicit-def $vgpr117, implicit-def $vgpr118, implicit-def $vgpr119, implicit-def $vgpr120, implicit-def $vgpr121, implicit-def $vgpr122, implicit-def $vgpr123, implicit-def $vgpr124, implicit-def $vgpr125, implicit-def $vgpr126, implicit-def $vgpr127, implicit-def $vgpr128, implicit-def $vgpr129, implicit-def $vgpr130, implicit-def $vgpr131, implicit-def $vgpr132, implicit-def $vgpr133, implicit-def $vgpr134, implicit-def $vgpr135, implicit-def $vgpr136, implicit-def $vgpr137, implicit-def $vgpr138, implicit-def $vgpr139, implicit-def $vgpr140, implicit-def $vgpr141, implicit-def $vgpr142, implicit-def $vgpr143, implicit-def $vgpr144, implicit-def $vgpr145, implicit-def $vgpr146, implicit-def $vgpr147, implicit-def $vgpr148, implicit-def $vgpr149, implicit-def $vgpr150, implicit-def $vgpr151, implicit-def $vgpr152, implicit-def $vgpr153, implicit-def $vgpr154, implicit-def $vgpr155, implicit-def $vgpr156, implicit-def $vgpr157, implicit-def $vgpr158, implicit-def $vgpr159, implicit-def $vgpr160, implicit-def $vgpr161, implicit-def $vgpr162, implicit-def $vgpr163, implicit-def $vgpr164, implicit-def $vgpr165, implicit-def $vgpr166, implicit-def $vgpr167, implicit-def $vgpr168, implicit-def $vgpr169, implicit-def $vgpr170, implicit-def $vgpr171, implicit-def $vgpr172, implicit-def $vgpr173, implicit-def $vgpr174, implicit-def $vgpr175, implicit-def $vgpr176, implicit-def $vgpr177, implicit-def $vgpr178, implicit-def $vgpr179, implicit-def $vgpr180, implicit-def $vgpr181, implicit-def $vgpr182, implicit-def $vgpr183, implicit-def $vgpr184, implicit-def $vgpr185, implicit-def $vgpr186, implicit-def $vgpr187, implicit-def $vgpr188, implicit-def $vgpr189, implicit-def $vgpr190, implicit-def $vgpr191, implicit-def $vgpr192, implicit-def $vgpr193, implicit-def $vgpr194, implicit-def $vgpr195, implicit-def $vgpr196, implicit-def $vgpr197, implicit-def $vgpr198, implicit-def $vgpr199, implicit-def $vgpr200, implicit-def $vgpr201, implicit-def $vgpr202, implicit-def $vgpr203, implicit-def $vgpr204, implicit-def $vgpr205, implicit-def $vgpr206, implicit-def $vgpr207, implicit-def $vgpr208, implicit-def $vgpr209, implicit-def $vgpr210, implicit-def $vgpr211, implicit-def $vgpr212, implicit-def $vgpr213, implicit-def $vgpr214, implicit-def $vgpr215, implicit-def $vgpr216, implicit-def $vgpr217, implicit-def $vgpr218, implicit-def $vgpr219, implicit-def $vgpr220, implicit-def $vgpr221, implicit-def $vgpr222, implicit-def $vgpr223, implicit-def $vgpr224, implicit-def $vgpr225, implicit-def $vgpr226, implicit-def $vgpr227, implicit-def $vgpr228, implicit-def $vgpr229, implicit-def $vgpr230, implicit-def $vgpr231, implicit-def $vgpr232, implicit-def $vgpr233, implicit-def $vgpr234, implicit-def $vgpr235, implicit-def $vgpr236, implicit-def $vgpr237, implicit-def $vgpr238, implicit-def $vgpr239, implicit-def $vgpr240, implicit-def $vgpr241, implicit-def $vgpr242, implicit-def $vgpr243, implicit-def $vgpr244, implicit-def $vgpr245, implicit-def $vgpr246, implicit-def $vgpr247, implicit-def $vgpr248, implicit-def $vgpr249, implicit-def $vgpr250, implicit-def $vgpr251, implicit-def $vgpr252, implicit-def $vgpr253, implicit-def $vgpr254, implicit-def $vgpr255
+ KILL %1:vgpr_32, %2:vgpr_32
+ S_ENDPGM 0
+...
+
>From bbd7624ae664063e5c95265845edb42980d99544 Mon Sep 17 00:00:00 2001
From: Krzysztof Drewniak <Krzysztof.Drewniak at amd.com>
Date: Tue, 23 Jun 2026 21:24:11 +0000
Subject: [PATCH 45/46] [AMDGPU] Precommit for pulling NUW large buffer offsets
into SOFFSET
This commit adds tests with larger constant offsets [being added to
VGPRs] in raw.buffer.* instructions to show the movement of such
offsets to SOFFSET after rounding in a future commit if this won't
cause errors because of overflow.
This also necessitated adding gfx1250 tests (to show the different
offset sizes) and moving tests that ctalled "export" into a new file
so that the gfx1250 (which doesn't support that instruction) doesn't
trip ot those tests.
This also adds a few global-isel specific tests for handling
`or disjoint` like an add we can't just stick -global-isel in the
normal buffer intrinsic tests.
AI usage disclosure: AI wrote these, I skimmed them.
Co-Authored-By: Codex <codex at openai.com>
---
.../llvm.amdgcn.raw.ptr.buffer.load.ll | 807 ++++++++
...llvm.amdgcn.raw.ptr.buffer.offset-split.ll | 92 +
.../legalize-amdgcn.raw.ptr.buffer.load.ll | 1842 +++++++++++++++++
...lvm.amdgcn.raw.ptr.buffer.load-exported.ll | 416 ++++
.../AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.ll | 928 +++++----
5 files changed, 3679 insertions(+), 406 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.offset-split.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load-exported.ll
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll
index 89c3a41eda29c..8ec540029e2a0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=GFX1250 %s
; FIXME: Test with SI when argument lowering not broken for f16
; Natural mapping
@@ -18,6 +19,21 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_s
; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret float %val
}
@@ -39,6 +55,22 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__sgpr_voffset__sgpr_s
; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY6]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__sgpr_voffset__sgpr_soffset
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr7
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY6]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret float %val
}
@@ -91,6 +123,53 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__vgpr_rsrc__vgpr_voffset__sgpr_s
; CHECK-NEXT: bb.5:
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__vgpr_rsrc__vgpr_voffset__sgpr_soffset
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.3(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY8]], [[COPY6]], implicit $exec
+ ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+ ; GFX1250-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+ ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.4:
+ ; GFX1250-NEXT: successors: %bb.5(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.5:
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret float %val
}
@@ -146,6 +225,56 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__vgpr_rsrc__vgpr_voffset__vgpr_s
; CHECK-NEXT: bb.5:
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__vgpr_rsrc__vgpr_voffset__vgpr_soffset
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.3(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY8]], [[COPY6]], implicit $exec
+ ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+ ; GFX1250-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+ ; GFX1250-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY5]], implicit $exec
+ ; GFX1250-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+ ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.4:
+ ; GFX1250-NEXT: successors: %bb.5(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.5:
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret float %val
}
@@ -166,6 +295,21 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_s
; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 1, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 1, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 1)
ret float %val
}
@@ -186,6 +330,21 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_s
; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 2, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 2, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 2)
ret float %val
}
@@ -206,6 +365,21 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_s
; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 4, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 4, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 4)
ret float %val
}
@@ -226,6 +400,21 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_s
; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 6, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_dlc
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 6, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 6)
ret float %val
}
@@ -246,6 +435,21 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_s
; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 5, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc_dlc
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 5, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 5)
ret float %val
}
@@ -266,6 +470,21 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_s
; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 7, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc_slc_dlc
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 7, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 7)
ret float %val
}
@@ -285,6 +504,21 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_s
; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (volatile dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_volatile
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (volatile dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 -2147483648)
ret float %val
}
@@ -308,6 +542,24 @@ define amdgpu_ps <2 x float> @raw_ptr_buffer_load_v2f32__sgpr_rsrc__vgpr_voffset
; CHECK-NEXT: $vgpr0 = COPY [[COPY6]]
; CHECK-NEXT: $vgpr1 = COPY [[COPY7]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_v2f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN]].sub0
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN]].sub1
+ ; GFX1250-NEXT: $vgpr0 = COPY [[COPY6]]
+ ; GFX1250-NEXT: $vgpr1 = COPY [[COPY7]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
%val = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret <2 x float> %val
}
@@ -332,6 +584,26 @@ define amdgpu_ps <3 x float> @raw_ptr_buffer_load_v3f32__sgpr_rsrc__vgpr_voffset
; CHECK-NEXT: $vgpr1 = COPY [[COPY7]]
; CHECK-NEXT: $vgpr2 = COPY [[COPY8]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_v3f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN:%[0-9]+]]:vreg_96_align2 = BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN]].sub0
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN]].sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN]].sub2
+ ; GFX1250-NEXT: $vgpr0 = COPY [[COPY6]]
+ ; GFX1250-NEXT: $vgpr1 = COPY [[COPY7]]
+ ; GFX1250-NEXT: $vgpr2 = COPY [[COPY8]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
%val = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret <3 x float> %val
}
@@ -358,6 +630,28 @@ define amdgpu_ps <4 x float> @raw_ptr_buffer_load_v4f32__sgpr_rsrc__vgpr_voffset
; CHECK-NEXT: $vgpr2 = COPY [[COPY8]]
; CHECK-NEXT: $vgpr3 = COPY [[COPY9]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_v4f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN]].sub0
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN]].sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN]].sub2
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN]].sub3
+ ; GFX1250-NEXT: $vgpr0 = COPY [[COPY6]]
+ ; GFX1250-NEXT: $vgpr1 = COPY [[COPY7]]
+ ; GFX1250-NEXT: $vgpr2 = COPY [[COPY8]]
+ ; GFX1250-NEXT: $vgpr3 = COPY [[COPY9]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
%val = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret <4 x float> %val
}
@@ -377,6 +671,21 @@ define amdgpu_ps half @raw_ptr_buffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_so
; CHECK-NEXT: [[BUFFER_LOAD_USHORT_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16) from %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret half %val
}
@@ -396,6 +705,21 @@ define amdgpu_ps <2 x half> @raw_ptr_buffer_load_v2f16__sgpr_rsrc__vgpr_voffset_
; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s16>) from %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s16>) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call <2 x half> @llvm.amdgcn.raw.ptr.buffer.load.v2f16(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret <2 x half> %val
}
@@ -424,6 +748,24 @@ define amdgpu_ps <4 x half> @raw_ptr_buffer_load_v4f16__sgpr_rsrc__vgpr_voffset_
; CHECK-NEXT: $vgpr0 = COPY [[COPY6]]
; CHECK-NEXT: $vgpr1 = COPY [[COPY7]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN]].sub0
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN]].sub1
+ ; GFX1250-NEXT: $vgpr0 = COPY [[COPY6]]
+ ; GFX1250-NEXT: $vgpr1 = COPY [[COPY7]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
%val = call <4 x half> @llvm.amdgcn.raw.ptr.buffer.load.v4f16(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret <4 x half> %val
}
@@ -443,6 +785,21 @@ define amdgpu_ps float @raw_ptr_buffer_load_i8__sgpr_rsrc__vgpr_voffset__sgpr_so
; CHECK-NEXT: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8) from %ir.rsrc, addrspace 8)
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffset_zext
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_UBYTE_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8) from %ir.rsrc, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
%zext = zext i8 %val to i32
%cast = bitcast i32 %zext to float
@@ -464,6 +821,21 @@ define amdgpu_ps float @raw_ptr_buffer_load_i8__sgpr_rsrc__vgpr_voffset__sgpr_so
; CHECK-NEXT: [[BUFFER_LOAD_SBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_SBYTE_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8) from %ir.rsrc, addrspace 8)
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_SBYTE_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sext
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_SBYTE_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_SBYTE_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8) from %ir.rsrc, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_SBYTE_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
%zext = sext i8 %val to i32
%cast = bitcast i32 %zext to float
@@ -485,6 +857,21 @@ define amdgpu_ps float @raw_ptr_buffer_load_i16__sgpr_rsrc__vgpr_voffset__sgpr_s
; CHECK-NEXT: [[BUFFER_LOAD_USHORT_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16) from %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_i16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_zext
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
%zext = zext i16 %val to i32
%cast = bitcast i32 %zext to float
@@ -506,6 +893,21 @@ define amdgpu_ps float @raw_ptr_buffer_load_i16__sgpr_rsrc__vgpr_voffset__sgpr_s
; CHECK-NEXT: [[BUFFER_LOAD_SSHORT_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_SSHORT_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16) from %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_SSHORT_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_i16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sext
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_SSHORT_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_SSHORT_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_SSHORT_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
%sext = sext i16 %val to i32
%cast = bitcast i32 %sext to float
@@ -560,6 +962,53 @@ define amdgpu_ps half @raw_ptr_buffer_load_f16__vgpr_rsrc__vgpr_voffset__sgpr_so
; CHECK-NEXT: bb.5:
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.3(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY8]], [[COPY6]], implicit $exec
+ ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+ ; GFX1250-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+ ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.4:
+ ; GFX1250-NEXT: successors: %bb.5(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.5:
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret half %val
}
@@ -612,6 +1061,53 @@ define amdgpu_ps float @raw_ptr_buffer_load_i8__vgpr_rsrc__vgpr_voffset__sgpr_so
; CHECK-NEXT: bb.5:
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffset
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.3(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY8]], [[COPY6]], implicit $exec
+ ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+ ; GFX1250-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+ ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_UBYTE_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8) from %ir.rsrc, addrspace 8)
+ ; GFX1250-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.4:
+ ; GFX1250-NEXT: successors: %bb.5(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.5:
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
%zext = zext i8 %val to i32
%cast = bitcast i32 %zext to float
@@ -632,6 +1128,20 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vdpr_voffset__sgpr_s
; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET [[REG_SEQUENCE]], [[COPY4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFSET]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vdpr_voffset__sgpr_soffset__voffset0
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET [[REG_SEQUENCE]], [[COPY4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 0, i32 %soffset, i32 0)
ret float %val
}
@@ -650,6 +1160,20 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_s
; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET [[REG_SEQUENCE]], [[COPY4]], 4095, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFSET]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset4095
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET [[REG_SEQUENCE]], [[COPY4]], 4095, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 4095, i32 %soffset, i32 0)
ret float %val
}
@@ -670,10 +1194,59 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_s
; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY5]], [[REG_SEQUENCE]], [[COPY4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset4096
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET [[REG_SEQUENCE]], [[COPY4]], 4096, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 4096, i32 %soffset, i32 0)
ret float %val
}
+define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset0(ptr addrspace(8) inreg %rsrc) {
+ ; CHECK-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset0
+ ; CHECK: bb.1 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; CHECK-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 67104768
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 4092, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+ ; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset0
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 58720256
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 8388604, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ %val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 67108860, i32 0, i32 0)
+ ret float %val
+}
+
define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add16(ptr addrspace(8) inreg %rsrc, i32 %voffset.base, i32 inreg %soffset) {
; CHECK-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add16
; CHECK: bb.1 (%ir-block.0):
@@ -689,6 +1262,24 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_s
; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 16, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add16
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX1250-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY4]], [[COPY6]], 0, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[V_ADD_U32_e64_]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%voffset = add i32 %voffset.base, 16
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret float %val
@@ -709,6 +1300,24 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_s
; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4095
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX1250-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY4]], [[COPY6]], 0, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[V_ADD_U32_e64_]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%voffset = add i32 %voffset.base, 4095
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret float %val
@@ -732,6 +1341,24 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_s
; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4096
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX1250-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY4]], [[COPY6]], 0, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[V_ADD_U32_e64_]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%voffset = add i32 %voffset.base, 4096
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret float %val
@@ -752,6 +1379,21 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_s
; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset4095
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 4095, i32 0)
ret float %val
}
@@ -771,6 +1413,21 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_s
; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset4096
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 4096, i32 0)
ret float %val
}
@@ -792,6 +1449,23 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_s
; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[S_ADD_I32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add16
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; GFX1250-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY5]], [[S_MOV_B32_]], implicit-def dead $scc
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[S_ADD_I32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%soffset = add i32 %soffset.base, 16
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret float %val
@@ -814,6 +1488,23 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_s
; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[S_ADD_I32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add4095
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+ ; GFX1250-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY5]], [[S_MOV_B32_]], implicit-def dead $scc
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[S_ADD_I32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%soffset = add i32 %soffset.base, 4095
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret float %val
@@ -836,6 +1527,23 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_s
; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[S_ADD_I32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add4096
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+ ; GFX1250-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY5]], [[S_MOV_B32_]], implicit-def dead $scc
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[S_ADD_I32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%soffset = add i32 %soffset.base, 4096
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret float %val
@@ -891,6 +1599,55 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_s
; CHECK-NEXT: bb.5:
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add5000
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 5000
+ ; GFX1250-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY5]], [[S_MOV_B32_]], implicit-def dead $scc
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.3(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY8]], [[COPY6]], implicit $exec
+ ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+ ; GFX1250-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+ ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[S_ADD_I32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.4:
+ ; GFX1250-NEXT: successors: %bb.5(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_MOV_B32_1]]
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.5:
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%soffset = add i32 %soffset.base, 5000
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret float %val
@@ -947,6 +1704,56 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_s
; CHECK-NEXT: bb.5:
; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add5000
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 5000
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX1250-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY4]], [[COPY6]], 0, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.3(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; GFX1250-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+ ; GFX1250-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
+ ; GFX1250-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+ ; GFX1250-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[V_ADD_U32_e64_]], [[REG_SEQUENCE1]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.4:
+ ; GFX1250-NEXT: successors: %bb.5(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_MOV_B32_1]]
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.5:
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%voffset = add i32 %voffset.base, 5000
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret float %val
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.offset-split.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.offset-split.ll
new file mode 100644
index 0000000000000..c61901f42e7a0
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.offset-split.ll
@@ -0,0 +1,92 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -verify-machineinstrs < %s | FileCheck --check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 -amdgpu-enable-delay-alu=0 -verify-machineinstrs < %s | FileCheck --check-prefix=GFX1250 %s
+
+define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset16(ptr addrspace(8) inreg %rsrc) {
+; GFX8-LABEL: raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset16:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_mov_b32 s0, s2
+; GFX8-NEXT: s_mov_b32 s1, s3
+; GFX8-NEXT: s_mov_b32 s2, s4
+; GFX8-NEXT: s_mov_b32 s3, s5
+; GFX8-NEXT: s_mov_b32 s4, 0x3fff010
+; GFX8-NEXT: buffer_load_dword v0, off, s[0:3], s4 offset:4092
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b32 s0, s2
+; GFX1250-NEXT: s_mov_b32 s1, s3
+; GFX1250-NEXT: s_mov_b32 s2, s4
+; GFX1250-NEXT: s_mov_b32 s3, s5
+; GFX1250-NEXT: s_mov_b32 s4, 0x3800010
+; GFX1250-NEXT: buffer_load_b32 v0, off, s[0:3], s4 offset:8388604
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
+ %val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 67108860, i32 16, i32 0)
+ ret float %val
+}
+
+define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset_neg16(ptr addrspace(8) inreg %rsrc) {
+; GFX8-LABEL: raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset_neg16:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_mov_b32 s0, s2
+; GFX8-NEXT: s_mov_b32 s1, s3
+; GFX8-NEXT: s_mov_b32 s2, s4
+; GFX8-NEXT: s_mov_b32 s3, s5
+; GFX8-NEXT: v_mov_b32_e32 v0, 0x3fff000
+; GFX8-NEXT: buffer_load_dword v0, v0, s[0:3], -16 offen offset:4092
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset_neg16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0x3800000
+; GFX1250-NEXT: s_mov_b32 s0, s2
+; GFX1250-NEXT: s_mov_b32 s1, s3
+; GFX1250-NEXT: s_mov_b32 s2, s4
+; GFX1250-NEXT: s_mov_b32 s3, s5
+; GFX1250-NEXT: s_mov_b32 s4, -16
+; GFX1250-NEXT: buffer_load_b32 v0, v0, s[0:3], s4 offen offset:8388604
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
+ %val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 67108860, i32 -16, i32 0)
+ ret float %val
+}
+
+define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__or67108860_soffset16(ptr addrspace(8) inreg %rsrc, i32 %voffset.base) {
+; GFX8-LABEL: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__or67108860_soffset16:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: v_and_b32_e32 v0, 0xfc000000, v0
+; GFX8-NEXT: s_mov_b32 s0, s2
+; GFX8-NEXT: s_mov_b32 s1, s3
+; GFX8-NEXT: s_mov_b32 s2, s4
+; GFX8-NEXT: s_mov_b32 s3, s5
+; GFX8-NEXT: v_or_b32_e32 v0, 0x3fffffc, v0
+; GFX8-NEXT: buffer_load_dword v0, v0, s[0:3], 16 offen
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__or67108860_soffset16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 0x3fffffc
+; GFX1250-NEXT: s_mov_b32 s0, s2
+; GFX1250-NEXT: s_mov_b32 s1, s3
+; GFX1250-NEXT: s_mov_b32 s2, s4
+; GFX1250-NEXT: s_mov_b32 s3, s5
+; GFX1250-NEXT: v_and_or_b32 v0, 0xfc000000, v0, v1
+; GFX1250-NEXT: s_mov_b32 s4, 16
+; GFX1250-NEXT: buffer_load_b32 v0, v0, s[0:3], s4 offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
+ %voffset.masked = and i32 %voffset.base, -67108864
+ %voffset = or disjoint i32 %voffset.masked, 67108860
+ %val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 16, i32 0)
+ ret float %val
+}
+
+declare float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8), i32, i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll
index 2556c67d7235e..952b7f33d3dfa 100644
--- a/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
; RUN: llc -mtriple=amdgcn -mcpu=gfx908 -stop-after=si-fix-sgpr-copies < %s | FileCheck -check-prefix=GFX908 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -stop-after=si-fix-sgpr-copies < %s | FileCheck -check-prefix=GFX1250 %s
; Natural mapping
define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset) {
@@ -52,6 +53,55 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset(ptr
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %32, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret float %val
}
@@ -107,6 +157,55 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__sgpr_voffset__sgpr_soffset(ptr
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__sgpr_voffset__sgpr_soffset
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %32, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret float %val
}
@@ -162,6 +261,55 @@ define float @raw_ptr_buffer_load_f32__vgpr_rsrc__vgpr_voffset__sgpr_soffset(ptr
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__vgpr_rsrc__vgpr_voffset__sgpr_soffset
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %32, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret float %val
}
@@ -217,6 +365,55 @@ define float @raw_ptr_buffer_load_f32__vgpr_rsrc__vgpr_voffset__vgpr_soffset(ptr
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__vgpr_rsrc__vgpr_voffset__vgpr_soffset
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %32, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret float %val
}
@@ -272,6 +469,55 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %32, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 1, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 1)
ret float %val
}
@@ -327,6 +573,55 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %32, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 2, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 2)
ret float %val
}
@@ -382,6 +677,55 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %32, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 4, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 4)
ret float %val
}
@@ -437,6 +781,55 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_dlc
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %32, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 6, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 6)
ret float %val
}
@@ -492,6 +885,55 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc_dlc
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %32, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 5, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 5)
ret float %val
}
@@ -547,6 +989,55 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc_slc_dlc
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %32, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 7, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 7)
ret float %val
}
@@ -605,6 +1096,58 @@ define <2 x float> @raw_ptr_buffer_load_v2f32__sgpr_rsrc__vgpr_voffset__sgpr_sof
; GFX908-NEXT: $vgpr0 = COPY [[COPY10]]
; GFX908-NEXT: $vgpr1 = COPY [[COPY11]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_v2f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %34, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s64) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; GFX1250-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN]].sub0
+ ; GFX1250-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN]].sub1
+ ; GFX1250-NEXT: $vgpr0 = COPY [[COPY10]]
+ ; GFX1250-NEXT: $vgpr1 = COPY [[COPY11]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
%val = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret <2 x float> %val
}
@@ -664,6 +1207,60 @@ define <3 x float> @raw_ptr_buffer_load_v3f32__sgpr_rsrc__vgpr_voffset__sgpr_sof
; GFX908-NEXT: $vgpr1 = COPY [[COPY11]]
; GFX908-NEXT: $vgpr2 = COPY [[COPY12]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_v3f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %35, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN:%[0-9]+]]:vreg_96_align2 = BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s96) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; GFX1250-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN]].sub0
+ ; GFX1250-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN]].sub1
+ ; GFX1250-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN]].sub2
+ ; GFX1250-NEXT: $vgpr0 = COPY [[COPY10]]
+ ; GFX1250-NEXT: $vgpr1 = COPY [[COPY11]]
+ ; GFX1250-NEXT: $vgpr2 = COPY [[COPY12]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
%val = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret <3 x float> %val
}
@@ -725,6 +1322,62 @@ define <4 x float> @raw_ptr_buffer_load_v4f32__sgpr_rsrc__vgpr_voffset__sgpr_sof
; GFX908-NEXT: $vgpr2 = COPY [[COPY12]]
; GFX908-NEXT: $vgpr3 = COPY [[COPY13]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_v4f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %36, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; GFX1250-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN]].sub0
+ ; GFX1250-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN]].sub1
+ ; GFX1250-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN]].sub2
+ ; GFX1250-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN]].sub3
+ ; GFX1250-NEXT: $vgpr0 = COPY [[COPY10]]
+ ; GFX1250-NEXT: $vgpr1 = COPY [[COPY11]]
+ ; GFX1250-NEXT: $vgpr2 = COPY [[COPY12]]
+ ; GFX1250-NEXT: $vgpr3 = COPY [[COPY13]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
%val = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret <4 x float> %val
}
@@ -780,6 +1433,56 @@ define half @raw_ptr_buffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset(ptr
; GFX908-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_USHORT_OFFEN]]
; GFX908-NEXT: $vgpr0 = COPY [[COPY10]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %33, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; GFX1250-NEXT: [[COPY10:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]].lo16
+ ; GFX1250-NEXT: $vgpr0 = COPY [[COPY10]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%val = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret half %val
}
@@ -834,6 +1537,55 @@ define <2 x half> @raw_ptr_buffer_load_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %32, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%val = call <2 x half> @llvm.amdgcn.raw.ptr.buffer.load.v2f16(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret <2 x half> %val
}
@@ -891,6 +1643,58 @@ define <4 x half> @raw_ptr_buffer_load_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; GFX908-NEXT: $vgpr0 = COPY [[COPY10]]
; GFX908-NEXT: $vgpr1 = COPY [[COPY11]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %34, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s64) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; GFX1250-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN]].sub0
+ ; GFX1250-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN]].sub1
+ ; GFX1250-NEXT: $vgpr0 = COPY [[COPY10]]
+ ; GFX1250-NEXT: $vgpr1 = COPY [[COPY11]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
%val = call <4 x half> @llvm.amdgcn.raw.ptr.buffer.load.v4f16(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret <4 x half> %val
}
@@ -945,6 +1749,55 @@ define float @raw_ptr_buffer_load_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffset_zext
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffset_zext
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %32, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_UBYTE_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8) from %ir.rsrc, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%val = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
%zext = zext i8 %val to i32
%cast = bitcast i32 %zext to float
@@ -1001,6 +1854,55 @@ define float @raw_ptr_buffer_load_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sext
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_SBYTE_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sext
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %32, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_SBYTE_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_SBYTE_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8) from %ir.rsrc, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_SBYTE_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%val = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
%zext = sext i8 %val to i32
%cast = bitcast i32 %zext to float
@@ -1057,6 +1959,55 @@ define float @raw_ptr_buffer_load_i16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_zex
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_i16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_zext
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %32, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%val = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
%zext = zext i16 %val to i32
%cast = bitcast i32 %zext to float
@@ -1113,6 +2064,55 @@ define float @raw_ptr_buffer_load_i16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sex
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_SSHORT_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_i16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sext
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %32, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_SSHORT_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_SSHORT_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_SSHORT_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%val = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
%sext = sext i16 %val to i32
%cast = bitcast i32 %sext to float
@@ -1171,6 +2171,56 @@ define half @raw_ptr_buffer_load_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset(ptr
; GFX908-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_USHORT_OFFEN]]
; GFX908-NEXT: $vgpr0 = COPY [[COPY10]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %33, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; GFX1250-NEXT: [[COPY10:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]].lo16
+ ; GFX1250-NEXT: $vgpr0 = COPY [[COPY10]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%val = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret half %val
}
@@ -1226,6 +2276,55 @@ define float @raw_ptr_buffer_load_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffset(ptr
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffset
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %32, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_UBYTE_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8) from %ir.rsrc, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%val = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
%zext = zext i8 %val to i32
%cast = bitcast i32 %zext to float
@@ -1281,6 +2380,54 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vdpr_voffset__sgpr_soffset__vo
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFSET]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vdpr_voffset__sgpr_soffset__voffset0
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY8]], %subreg.sub0, killed [[COPY7]], %subreg.sub1, killed [[COPY6]], %subreg.sub2, killed [[COPY5]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %31, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 0, i32 %soffset, i32 0)
ret float %val
}
@@ -1334,6 +2481,54 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__vo
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFSET]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset4095
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY8]], %subreg.sub0, killed [[COPY7]], %subreg.sub1, killed [[COPY6]], %subreg.sub2, killed [[COPY5]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %31, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 4095, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 4095, i32 %soffset, i32 0)
ret float %val
}
@@ -1388,10 +2583,155 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__vo
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset4096
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY8]], %subreg.sub0, killed [[COPY7]], %subreg.sub1, killed [[COPY6]], %subreg.sub2, killed [[COPY5]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %31, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 4096, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 4096, i32 %soffset, i32 0)
ret float %val
}
+define float @raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset0(ptr addrspace(8) %rsrc) {
+ ; GFX908-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset0
+ ; GFX908: bb.0 (%ir-block.0):
+ ; GFX908-NEXT: successors: %bb.1(0x80000000)
+ ; GFX908-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX908-NEXT: {{ $}}
+ ; GFX908-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX908-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX908-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX908-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX908-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY]], %subreg.sub1
+ ; GFX908-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX908-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX908-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX908-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX908-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX908-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128 = REG_SEQUENCE killed [[COPY7]], %subreg.sub0, killed [[COPY6]], %subreg.sub1, killed [[COPY5]], %subreg.sub2, killed [[COPY4]], %subreg.sub3
+ ; GFX908-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX908-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 67104768, implicit $exec
+ ; GFX908-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; GFX908-NEXT: {{ $}}
+ ; GFX908-NEXT: bb.1:
+ ; GFX908-NEXT: successors: %bb.2(0x80000000)
+ ; GFX908-NEXT: {{ $}}
+ ; GFX908-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX908-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX908-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX908-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit $exec
+ ; GFX908-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX908-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX908-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX908-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit $exec
+ ; GFX908-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; GFX908-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX908-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX908-NEXT: {{ $}}
+ ; GFX908-NEXT: bb.2:
+ ; GFX908-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX908-NEXT: {{ $}}
+ ; GFX908-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE5]], [[S_MOV_B32_]], 4092, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX908-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; GFX908-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX908-NEXT: {{ $}}
+ ; GFX908-NEXT: bb.3:
+ ; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
+ ; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+ ; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset0
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY7]], %subreg.sub0, killed [[COPY6]], %subreg.sub1, killed [[COPY5]], %subreg.sub2, killed [[COPY4]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 58720256, implicit $exec
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %30, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE5]], $sgpr_null, 8388604, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
+ %val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 67108860, i32 0, i32 0)
+ ret float %val
+}
+
define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add16(ptr addrspace(8) %rsrc, i32 %voffset.base, i32 %soffset) {
; GFX908-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add16
; GFX908: bb.0 (%ir-block.0):
@@ -1442,6 +2782,57 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_vof
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add16
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; GFX1250-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY1]], killed [[S_MOV_B32_]], 0, implicit $exec
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_2]], %bb.0, %34, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[V_ADD_U32_e64_]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_1]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%voffset = add i32 %voffset.base, 16
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret float %val
@@ -1497,6 +2888,57 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__vo
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4095
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+ ; GFX1250-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY1]], killed [[S_MOV_B32_]], 0, implicit $exec
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_2]], %bb.0, %34, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[V_ADD_U32_e64_]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_1]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%voffset = add i32 %voffset.base, 4095
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret float %val
@@ -1554,6 +2996,57 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__vo
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4096
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+ ; GFX1250-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY1]], killed [[S_MOV_B32_]], 0, implicit $exec
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_2]], %bb.0, %34, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[V_ADD_U32_e64_]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_1]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%voffset = add i32 %voffset.base, 4096
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret float %val
@@ -1606,6 +3099,53 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sof
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset4095
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY8]], %subreg.sub0, killed [[COPY7]], %subreg.sub1, killed [[COPY6]], %subreg.sub2, killed [[COPY5]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_2]], %bb.0, %31, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY]], killed [[REG_SEQUENCE5]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_1]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 4095, i32 0)
ret float %val
}
@@ -1657,6 +3197,53 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sof
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset4096
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY8]], %subreg.sub0, killed [[COPY7]], %subreg.sub1, killed [[COPY6]], %subreg.sub2, killed [[COPY5]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_2]], %bb.0, %31, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY]], killed [[REG_SEQUENCE5]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_1]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 4096, i32 0)
ret float %val
}
@@ -1713,6 +3300,57 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sof
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add16
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; GFX1250-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], killed [[S_MOV_B32_]], 0, implicit $exec
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_2]], %bb.0, %34, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[V_ADD_U32_e64_]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[V_ADD_U32_e64_]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_1]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%soffset = add i32 %soffset.base, 16
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret float %val
@@ -1770,6 +3408,57 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sof
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add4095
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+ ; GFX1250-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], killed [[S_MOV_B32_]], 0, implicit $exec
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_2]], %bb.0, %34, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[V_ADD_U32_e64_]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[V_ADD_U32_e64_]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_1]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%soffset = add i32 %soffset.base, 4095
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret float %val
@@ -1827,6 +3516,57 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sof
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add4096
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+ ; GFX1250-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], killed [[S_MOV_B32_]], 0, implicit $exec
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_2]], %bb.0, %34, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[V_ADD_U32_e64_]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[V_ADD_U32_e64_]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_1]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%soffset = add i32 %soffset.base, 4096
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret float %val
@@ -1885,6 +3625,57 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sof
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add5000
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 5000
+ ; GFX1250-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], killed [[S_MOV_B32_]], 0, implicit $exec
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_2]], %bb.0, %34, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[V_ADD_U32_e64_]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[V_ADD_U32_e64_]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY1]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_1]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%soffset = add i32 %soffset.base, 5000
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret float %val
@@ -1943,6 +3734,57 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_vof
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
+ ;
+ ; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add5000
+ ; GFX1250: bb.0 (%ir-block.0):
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 5000
+ ; GFX1250-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY1]], killed [[S_MOV_B32_]], 0, implicit $exec
+ ; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.2(0x80000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_2]], %bb.0, %34, %bb.2
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub2, implicit $exec
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub3, implicit $exec
+ ; GFX1250-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; GFX1250-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE2]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32_term [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit-def $exec, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[V_ADD_U32_e64_]], killed [[REG_SEQUENCE5]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.3:
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_1]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%voffset = add i32 %voffset.base, 5000
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret float %val
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load-exported.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load-exported.ll
new file mode 100644
index 0000000000000..147e9000eb33f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load-exported.ll
@@ -0,0 +1,416 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+;RUN: llc < %s -mtriple=amdgcn -mcpu=verde | FileCheck %s --check-prefixes=PREGFX10
+;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck %s --check-prefixes=PREGFX10
+;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1010 | FileCheck %s --check-prefixes=GFX10
+;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX11
+
+define amdgpu_ps void @buffer_load_x1_offen_merged_and(ptr addrspace(8) inreg %rsrc, i32 %a) {
+; PREGFX10-LABEL: buffer_load_x1_offen_merged_and:
+; PREGFX10: ; %bb.0: ; %main_body
+; PREGFX10-NEXT: buffer_load_dwordx4 v[1:4], v0, s[0:3], 0 offen offset:4
+; PREGFX10-NEXT: buffer_load_dwordx2 v[5:6], v0, s[0:3], 0 offen offset:28
+; PREGFX10-NEXT: s_waitcnt vmcnt(1)
+; PREGFX10-NEXT: exp mrt0, v1, v2, v3, v4 done vm
+; PREGFX10-NEXT: s_waitcnt vmcnt(0)
+; PREGFX10-NEXT: exp mrt0, v5, v6, v0, v0 done vm
+; PREGFX10-NEXT: s_endpgm
+;
+; GFX10-LABEL: buffer_load_x1_offen_merged_and:
+; GFX10: ; %bb.0: ; %main_body
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: buffer_load_dwordx4 v[1:4], v0, s[0:3], 0 offen offset:4
+; GFX10-NEXT: buffer_load_dwordx2 v[5:6], v0, s[0:3], 0 offen offset:28
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: exp mrt0, v1, v2, v3, v4 done vm
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: exp mrt0, v5, v6, v0, v0 done vm
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: buffer_load_x1_offen_merged_and:
+; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: buffer_load_b128 v[1:4], v0, s[0:3], 0 offen offset:4
+; GFX11-NEXT: buffer_load_b64 v[5:6], v0, s[0:3], 0 offen offset:28
+; GFX11-NEXT: s_waitcnt vmcnt(1)
+; GFX11-NEXT: exp mrt0, v1, v2, v3, v4 done
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: exp mrt0, v5, v6, v0, v0 done
+; GFX11-NEXT: s_endpgm
+main_body:
+ %a1 = add i32 %a, 4
+ %a2 = add i32 %a, 8
+ %a3 = add i32 %a, 12
+ %a4 = add i32 %a, 16
+ %a5 = add i32 %a, 28
+ %a6 = add i32 %a, 32
+ %r1 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a1, i32 0, i32 0)
+ %r2 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a2, i32 0, i32 0)
+ %r3 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a3, i32 0, i32 0)
+ %r4 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a4, i32 0, i32 0)
+ %r5 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a5, i32 0, i32 0)
+ %r6 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a6, i32 0, i32 0)
+ call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
+ call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float poison, float poison, i1 true, i1 true)
+ ret void
+}
+
+define amdgpu_ps void @buffer_load_x1_offen_merged_or(ptr addrspace(8) inreg %rsrc, i32 %inp) {
+; PREGFX10-LABEL: buffer_load_x1_offen_merged_or:
+; PREGFX10: ; %bb.0: ; %main_body
+; PREGFX10-NEXT: v_lshlrev_b32_e32 v4, 6, v0
+; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], v4, s[0:3], 0 offen offset:4
+; PREGFX10-NEXT: buffer_load_dwordx2 v[4:5], v4, s[0:3], 0 offen offset:28
+; PREGFX10-NEXT: s_waitcnt vmcnt(1)
+; PREGFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
+; PREGFX10-NEXT: s_waitcnt vmcnt(0)
+; PREGFX10-NEXT: exp mrt0, v4, v5, v0, v0 done vm
+; PREGFX10-NEXT: s_endpgm
+;
+; GFX10-LABEL: buffer_load_x1_offen_merged_or:
+; GFX10: ; %bb.0: ; %main_body
+; GFX10-NEXT: v_lshlrev_b32_e32 v6, 6, v0
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], v6, s[0:3], 0 offen offset:4
+; GFX10-NEXT: buffer_load_dwordx2 v[4:5], v6, s[0:3], 0 offen offset:28
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: exp mrt0, v4, v5, v0, v0 done vm
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: buffer_load_x1_offen_merged_or:
+; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 6, v0
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: buffer_load_b128 v[0:3], v4, s[0:3], 0 offen offset:4
+; GFX11-NEXT: buffer_load_b64 v[4:5], v4, s[0:3], 0 offen offset:28
+; GFX11-NEXT: s_waitcnt vmcnt(1)
+; GFX11-NEXT: exp mrt0, v0, v1, v2, v3 done
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: exp mrt0, v4, v5, v0, v0 done
+; GFX11-NEXT: s_endpgm
+main_body:
+ %a = shl i32 %inp, 6
+ %a1 = or i32 %a, 4
+ %a2 = or i32 %a, 8
+ %a3 = or i32 %a, 12
+ %a4 = or i32 %a, 16
+ %a5 = or i32 %a, 28
+ %a6 = or i32 %a, 32
+ %r1 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a1, i32 0, i32 0)
+ %r2 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a2, i32 0, i32 0)
+ %r3 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a3, i32 0, i32 0)
+ %r4 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a4, i32 0, i32 0)
+ %r5 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a5, i32 0, i32 0)
+ %r6 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a6, i32 0, i32 0)
+ call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
+ call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float poison, float poison, i1 true, i1 true)
+ ret void
+}
+
+define amdgpu_ps void @buffer_load_x1_offen_merged_glc_slc(ptr addrspace(8) inreg %rsrc, i32 %a) {
+; PREGFX10-LABEL: buffer_load_x1_offen_merged_glc_slc:
+; PREGFX10: ; %bb.0: ; %main_body
+; PREGFX10-NEXT: buffer_load_dwordx2 v[1:2], v0, s[0:3], 0 offen offset:4
+; PREGFX10-NEXT: buffer_load_dwordx2 v[3:4], v0, s[0:3], 0 offen offset:12 glc
+; PREGFX10-NEXT: buffer_load_dwordx2 v[5:6], v0, s[0:3], 0 offen offset:28 glc slc
+; PREGFX10-NEXT: s_waitcnt vmcnt(1)
+; PREGFX10-NEXT: exp mrt0, v1, v2, v3, v4 done vm
+; PREGFX10-NEXT: s_waitcnt vmcnt(0)
+; PREGFX10-NEXT: exp mrt0, v5, v6, v0, v0 done vm
+; PREGFX10-NEXT: s_endpgm
+;
+; GFX10-LABEL: buffer_load_x1_offen_merged_glc_slc:
+; GFX10: ; %bb.0: ; %main_body
+; GFX10-NEXT: s_clause 0x2
+; GFX10-NEXT: buffer_load_dwordx2 v[1:2], v0, s[0:3], 0 offen offset:4
+; GFX10-NEXT: buffer_load_dwordx2 v[3:4], v0, s[0:3], 0 offen offset:12 glc
+; GFX10-NEXT: buffer_load_dwordx2 v[5:6], v0, s[0:3], 0 offen offset:28 glc slc
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: exp mrt0, v1, v2, v3, v4 done vm
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: exp mrt0, v5, v6, v0, v0 done vm
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: buffer_load_x1_offen_merged_glc_slc:
+; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: s_clause 0x2
+; GFX11-NEXT: buffer_load_b64 v[1:2], v0, s[0:3], 0 offen offset:4
+; GFX11-NEXT: buffer_load_b64 v[3:4], v0, s[0:3], 0 offen offset:12 glc
+; GFX11-NEXT: buffer_load_b64 v[5:6], v0, s[0:3], 0 offen offset:28 glc slc
+; GFX11-NEXT: s_waitcnt vmcnt(1)
+; GFX11-NEXT: exp mrt0, v1, v2, v3, v4 done
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: exp mrt0, v5, v6, v0, v0 done
+; GFX11-NEXT: s_endpgm
+main_body:
+ %a1 = add i32 %a, 4
+ %a2 = add i32 %a, 8
+ %a3 = add i32 %a, 12
+ %a4 = add i32 %a, 16
+ %a5 = add i32 %a, 28
+ %a6 = add i32 %a, 32
+ %r1 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a1, i32 0, i32 0)
+ %r2 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a2, i32 0, i32 0)
+ %r3 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a3, i32 0, i32 1)
+ %r4 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a4, i32 0, i32 1)
+ %r5 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a5, i32 0, i32 3)
+ %r6 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a6, i32 0, i32 3)
+ call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
+ call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float poison, float poison, i1 true, i1 true)
+ ret void
+}
+
+define amdgpu_ps void @buffer_load_x2_offen_merged_and(ptr addrspace(8) inreg %rsrc, i32 %a) {
+; PREGFX10-LABEL: buffer_load_x2_offen_merged_and:
+; PREGFX10: ; %bb.0: ; %main_body
+; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4
+; PREGFX10-NEXT: s_waitcnt vmcnt(0)
+; PREGFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
+; PREGFX10-NEXT: s_endpgm
+;
+; GFX10-LABEL: buffer_load_x2_offen_merged_and:
+; GFX10: ; %bb.0: ; %main_body
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: buffer_load_x2_offen_merged_and:
+; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:4
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: exp mrt0, v0, v1, v2, v3 done
+; GFX11-NEXT: s_endpgm
+main_body:
+ %a1 = add i32 %a, 4
+ %a2 = add i32 %a, 12
+ %vr1 = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %a1, i32 0, i32 0)
+ %vr2 = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %a2, i32 0, i32 0)
+ %r1 = extractelement <2 x float> %vr1, i32 0
+ %r2 = extractelement <2 x float> %vr1, i32 1
+ %r3 = extractelement <2 x float> %vr2, i32 0
+ %r4 = extractelement <2 x float> %vr2, i32 1
+ call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
+ ret void
+}
+
+define amdgpu_ps void @buffer_load_x2_offen_merged_or(ptr addrspace(8) inreg %rsrc, i32 %inp) {
+; PREGFX10-LABEL: buffer_load_x2_offen_merged_or:
+; PREGFX10: ; %bb.0: ; %main_body
+; PREGFX10-NEXT: v_lshlrev_b32_e32 v0, 4, v0
+; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4
+; PREGFX10-NEXT: s_waitcnt vmcnt(0)
+; PREGFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
+; PREGFX10-NEXT: s_endpgm
+;
+; GFX10-LABEL: buffer_load_x2_offen_merged_or:
+; GFX10: ; %bb.0: ; %main_body
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 4, v0
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: buffer_load_x2_offen_merged_or:
+; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 4, v0
+; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:4
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: exp mrt0, v0, v1, v2, v3 done
+; GFX11-NEXT: s_endpgm
+main_body:
+ %a = shl i32 %inp, 4
+ %a1 = add i32 %a, 4
+ %a2 = add i32 %a, 12
+ %vr1 = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %a1, i32 0, i32 0)
+ %vr2 = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %a2, i32 0, i32 0)
+ %r1 = extractelement <2 x float> %vr1, i32 0
+ %r2 = extractelement <2 x float> %vr1, i32 1
+ %r3 = extractelement <2 x float> %vr2, i32 0
+ %r4 = extractelement <2 x float> %vr2, i32 1
+ call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
+ ret void
+}
+
+define amdgpu_ps void @buffer_load_x1_offset_merged(ptr addrspace(8) inreg %rsrc) {
+; PREGFX10-LABEL: buffer_load_x1_offset_merged:
+; PREGFX10: ; %bb.0: ; %main_body
+; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], 0 offset:4
+; PREGFX10-NEXT: buffer_load_dwordx2 v[4:5], off, s[0:3], 0 offset:28
+; PREGFX10-NEXT: s_waitcnt vmcnt(1)
+; PREGFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
+; PREGFX10-NEXT: s_waitcnt vmcnt(0)
+; PREGFX10-NEXT: exp mrt0, v4, v5, v0, v0 done vm
+; PREGFX10-NEXT: s_endpgm
+;
+; GFX10-LABEL: buffer_load_x1_offset_merged:
+; GFX10: ; %bb.0: ; %main_body
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], 0 offset:4
+; GFX10-NEXT: buffer_load_dwordx2 v[4:5], off, s[0:3], 0 offset:28
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: exp mrt0, v4, v5, v0, v0 done vm
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: buffer_load_x1_offset_merged:
+; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], 0 offset:4
+; GFX11-NEXT: buffer_load_b64 v[4:5], off, s[0:3], 0 offset:28
+; GFX11-NEXT: s_waitcnt vmcnt(1)
+; GFX11-NEXT: exp mrt0, v0, v1, v2, v3 done
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: exp mrt0, v4, v5, v0, v0 done
+; GFX11-NEXT: s_endpgm
+main_body:
+ %r1 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 4, i32 0, i32 0)
+ %r2 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 8, i32 0, i32 0)
+ %r3 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 12, i32 0, i32 0)
+ %r4 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 16, i32 0, i32 0)
+ %r5 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 28, i32 0, i32 0)
+ %r6 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 32, i32 0, i32 0)
+ call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
+ call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float poison, float poison, i1 true, i1 true)
+ ret void
+}
+
+define amdgpu_ps void @buffer_load_x2_offset_merged(ptr addrspace(8) inreg %rsrc) {
+; PREGFX10-LABEL: buffer_load_x2_offset_merged:
+; PREGFX10: ; %bb.0: ; %main_body
+; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], 0 offset:4
+; PREGFX10-NEXT: s_waitcnt vmcnt(0)
+; PREGFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
+; PREGFX10-NEXT: s_endpgm
+;
+; GFX10-LABEL: buffer_load_x2_offset_merged:
+; GFX10: ; %bb.0: ; %main_body
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], 0 offset:4
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: buffer_load_x2_offset_merged:
+; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], 0 offset:4
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: exp mrt0, v0, v1, v2, v3 done
+; GFX11-NEXT: s_endpgm
+main_body:
+ %vr1 = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 4, i32 0, i32 0)
+ %vr2 = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 12, i32 0, i32 0)
+ %r1 = extractelement <2 x float> %vr1, i32 0
+ %r2 = extractelement <2 x float> %vr1, i32 1
+ %r3 = extractelement <2 x float> %vr2, i32 0
+ %r4 = extractelement <2 x float> %vr2, i32 1
+ call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
+ ret void
+}
+
+define amdgpu_ps void @raw_ptr_buffer_load_x1_offset_merged(ptr addrspace(8) inreg %rsrc) {
+; PREGFX10-LABEL: raw_ptr_buffer_load_x1_offset_merged:
+; PREGFX10: ; %bb.0: ; %main_body
+; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], 0 offset:4
+; PREGFX10-NEXT: buffer_load_dwordx2 v[4:5], off, s[0:3], 0 offset:28
+; PREGFX10-NEXT: s_waitcnt vmcnt(1)
+; PREGFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
+; PREGFX10-NEXT: s_waitcnt vmcnt(0)
+; PREGFX10-NEXT: exp mrt0, v4, v5, v0, v0 done vm
+; PREGFX10-NEXT: s_endpgm
+;
+; GFX10-LABEL: raw_ptr_buffer_load_x1_offset_merged:
+; GFX10: ; %bb.0: ; %main_body
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], 0 offset:4
+; GFX10-NEXT: buffer_load_dwordx2 v[4:5], off, s[0:3], 0 offset:28
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: exp mrt0, v4, v5, v0, v0 done vm
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: raw_ptr_buffer_load_x1_offset_merged:
+; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], 0 offset:4
+; GFX11-NEXT: buffer_load_b64 v[4:5], off, s[0:3], 0 offset:28
+; GFX11-NEXT: s_waitcnt vmcnt(1)
+; GFX11-NEXT: exp mrt0, v0, v1, v2, v3 done
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: exp mrt0, v4, v5, v0, v0 done
+; GFX11-NEXT: s_endpgm
+main_body:
+ %r1 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 4, i32 0, i32 0)
+ %r2 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 8, i32 0, i32 0)
+ %r3 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 12, i32 0, i32 0)
+ %r4 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 16, i32 0, i32 0)
+ %r5 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 28, i32 0, i32 0)
+ %r6 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 32, i32 0, i32 0)
+ call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
+ call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float poison, float poison, i1 true, i1 true)
+ ret void
+}
+
+define amdgpu_ps void @raw_ptr_buffer_load_x1_offset_swizzled_not_merged(ptr addrspace(8) inreg %rsrc) {
+; PREGFX10-LABEL: raw_ptr_buffer_load_x1_offset_swizzled_not_merged:
+; PREGFX10: ; %bb.0: ; %main_body
+; PREGFX10-NEXT: buffer_load_dword v0, off, s[0:3], 0 offset:4
+; PREGFX10-NEXT: buffer_load_dword v1, off, s[0:3], 0 offset:8
+; PREGFX10-NEXT: buffer_load_dword v2, off, s[0:3], 0 offset:12
+; PREGFX10-NEXT: buffer_load_dword v3, off, s[0:3], 0 offset:16
+; PREGFX10-NEXT: buffer_load_dword v4, off, s[0:3], 0 offset:28
+; PREGFX10-NEXT: buffer_load_dword v5, off, s[0:3], 0 offset:32
+; PREGFX10-NEXT: s_waitcnt vmcnt(2)
+; PREGFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
+; PREGFX10-NEXT: s_waitcnt vmcnt(0)
+; PREGFX10-NEXT: exp mrt0, v4, v5, v0, v0 done vm
+; PREGFX10-NEXT: s_endpgm
+;
+; GFX10-LABEL: raw_ptr_buffer_load_x1_offset_swizzled_not_merged:
+; GFX10: ; %bb.0: ; %main_body
+; GFX10-NEXT: s_clause 0x5
+; GFX10-NEXT: buffer_load_dword v0, off, s[0:3], 0 offset:4
+; GFX10-NEXT: buffer_load_dword v1, off, s[0:3], 0 offset:8
+; GFX10-NEXT: buffer_load_dword v2, off, s[0:3], 0 offset:12
+; GFX10-NEXT: buffer_load_dword v3, off, s[0:3], 0 offset:16
+; GFX10-NEXT: buffer_load_dword v4, off, s[0:3], 0 offset:28
+; GFX10-NEXT: buffer_load_dword v5, off, s[0:3], 0 offset:32
+; GFX10-NEXT: s_waitcnt vmcnt(2)
+; GFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: exp mrt0, v4, v5, v0, v0 done vm
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: raw_ptr_buffer_load_x1_offset_swizzled_not_merged:
+; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: s_clause 0x5
+; GFX11-NEXT: buffer_load_b32 v0, off, s[0:3], 0 offset:4
+; GFX11-NEXT: buffer_load_b32 v1, off, s[0:3], 0 offset:8
+; GFX11-NEXT: buffer_load_b32 v2, off, s[0:3], 0 offset:12
+; GFX11-NEXT: buffer_load_b32 v3, off, s[0:3], 0 offset:16
+; GFX11-NEXT: buffer_load_b32 v4, off, s[0:3], 0 offset:28
+; GFX11-NEXT: buffer_load_b32 v5, off, s[0:3], 0 offset:32
+; GFX11-NEXT: s_waitcnt vmcnt(2)
+; GFX11-NEXT: exp mrt0, v0, v1, v2, v3 done
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: exp mrt0, v4, v5, v0, v0 done
+; GFX11-NEXT: s_endpgm
+main_body:
+ %r1 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 4, i32 0, i32 8)
+ %r2 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 8, i32 0, i32 8)
+ %r3 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 12, i32 0, i32 8)
+ %r4 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 16, i32 0, i32 8)
+ %r5 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 28, i32 0, i32 8)
+ %r6 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 32, i32 0, i32 8)
+ call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
+ call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float poison, float poison, i1 true, i1 true)
+ ret void
+}
+
+
+declare float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8), i32, i32, i32) #0
+declare <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8), i32, i32, i32) #0
+declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #0
+attributes #0 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.ll
index 543b619f30746..12997dfa5fabe 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.ll
@@ -3,6 +3,9 @@
;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck %s --check-prefixes=PREGFX10
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1010 | FileCheck %s --check-prefixes=GFX10
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX11
+;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1250 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX1250
+;RUN: llc < %s -mtriple=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck %s --check-prefixes=SI
+;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s --check-prefixes=VI
define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load(ptr addrspace(8) inreg) {
; PREGFX10-LABEL: buffer_load:
@@ -30,6 +33,16 @@ define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load(ptr addrsp
; GFX11-NEXT: buffer_load_b128 v[8:11], off, s[0:3], 0 slc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: buffer_load:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_clause 0x2
+; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], null
+; GFX1250-NEXT: buffer_load_b128 v[4:7], off, s[0:3], null th:TH_LOAD_NT
+; GFX1250-NEXT: buffer_load_b128 v[8:11], off, s[0:3], null th:TH_LOAD_HT
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
main_body:
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 0)
%data_glc = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 1)
@@ -66,6 +79,16 @@ define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load_dlc(ptr ad
; GFX11-NEXT: buffer_load_b128 v[8:11], off, s[0:3], 0 slc dlc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: buffer_load_dlc:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_clause 0x2
+; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], null th:TH_LOAD_NT_RT
+; GFX1250-NEXT: buffer_load_b128 v[4:7], off, s[0:3], null th:TH_LOAD_RT_NT
+; GFX1250-NEXT: buffer_load_b128 v[8:11], off, s[0:3], null th:TH_LOAD_NT_HT
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
main_body:
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 4)
%data_glc = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 5)
@@ -106,6 +129,17 @@ define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load_volatile(p
; GFX11-NEXT: buffer_load_b128 v[8:11], off, s[0:3], 0 glc slc dlc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: buffer_load_volatile:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], null scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: buffer_load_b128 v[4:7], off, s[0:3], null th:TH_LOAD_NT scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: buffer_load_b128 v[8:11], off, s[0:3], null th:TH_LOAD_HT scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
main_body:
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 -2147483648)
%data_glc = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 -2147483647)
@@ -134,6 +168,13 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs(ptr addrspace(8) inreg) {
; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], 0 offset:40
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: buffer_load_immoffs:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], null offset:40
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
main_body:
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 40, i32 0, i32 0)
ret <4 x float> %data
@@ -160,6 +201,14 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs_large(ptr addrspace(8) inreg)
; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:4
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: buffer_load_immoffs_large:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_movk_i32 s4, 0x1ffc
+; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:4
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
main_body:
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 4, i32 8188, i32 0)
ret <4 x float> %data
@@ -183,6 +232,13 @@ define amdgpu_ps <4 x float> @buffer_load_ofs(ptr addrspace(8) inreg, i32) {
; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: buffer_load_ofs:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
main_body:
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 %1, i32 0, i32 0)
ret <4 x float> %data
@@ -206,6 +262,14 @@ define amdgpu_ps <4 x float> @buffer_load_ofs_imm(ptr addrspace(8) inreg, i32) {
; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: buffer_load_ofs_imm:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
main_body:
%ofs = add i32 %1, 60
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 %ofs, i32 0, i32 0)
@@ -230,6 +294,13 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_12bit(ptr addrspace(8) i
; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], 0 offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: buffer_load_voffset_large_12bit:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], null offset:4092
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
main_body:
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 4092, i32 0, i32 0)
ret <4 x float> %data
@@ -256,6 +327,13 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_13bit(ptr addrspace(8) i
; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: buffer_load_voffset_large_13bit:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], null offset:8188
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
main_body:
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 8188, i32 0, i32 0)
ret <4 x float> %data
@@ -282,6 +360,13 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_16bit(ptr addrspace(8) i
; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: buffer_load_voffset_large_16bit:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], null offset:65532
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
main_body:
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 65532, i32 0, i32 0)
ret <4 x float> %data
@@ -308,6 +393,13 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_23bit(ptr addrspace(8) i
; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: buffer_load_voffset_large_23bit:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], null offset:8388604
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
main_body:
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 8388604, i32 0, i32 0)
ret <4 x float> %data
@@ -334,6 +426,14 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_24bit(ptr addrspace(8) i
; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: buffer_load_voffset_large_24bit:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0x800000
+; GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen offset:8388604
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
main_body:
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 16777212, i32 0, i32 0)
ret <4 x float> %data
@@ -358,6 +458,13 @@ define amdgpu_ps float @buffer_load_x1(ptr addrspace(8) inreg %rsrc, i32 %ofs) {
; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: buffer_load_x1:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
main_body:
%data = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 0, i32 0)
ret float %data
@@ -381,6 +488,13 @@ define amdgpu_ps <2 x float> @buffer_load_x2(ptr addrspace(8) inreg %rsrc, i32 %
; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: buffer_load_x2:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
main_body:
%data = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %ofs, i32 0, i32 0)
ret <2 x float> %data
@@ -400,6 +514,14 @@ define amdgpu_ps <4 x float> @buffer_load_negative_offset(ptr addrspace(8) inreg
; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: buffer_load_negative_offset:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, -16, v0
+; GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
main_body:
%ofs.1 = add i32 %ofs, -16
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 %ofs.1, i32 0, i32 0)
@@ -426,6 +548,17 @@ define amdgpu_ps float @buffer_load_mmo(ptr addrspace(8) inreg %rsrc, ptr addrsp
; GFX11-NEXT: v_mov_b32_e32 v0, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: buffer_load_mmo:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: buffer_load_b32 v1, off, s[0:3], null
+; GFX1250-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-NEXT: ds_store_2addr_b32 v0, v2, v2 offset1:4
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v0, v1
+; GFX1250-NEXT: s_wait_dscnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
entry:
store float 0.0, ptr addrspace(3) %lds
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
@@ -434,311 +567,6 @@ entry:
ret float %val
}
-define amdgpu_ps void @buffer_load_x1_offen_merged_and(ptr addrspace(8) inreg %rsrc, i32 %a) {
-; PREGFX10-LABEL: buffer_load_x1_offen_merged_and:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: buffer_load_dwordx4 v[1:4], v0, s[0:3], 0 offen offset:4
-; PREGFX10-NEXT: buffer_load_dwordx2 v[5:6], v0, s[0:3], 0 offen offset:28
-; PREGFX10-NEXT: s_waitcnt vmcnt(1)
-; PREGFX10-NEXT: exp mrt0, v1, v2, v3, v4 done vm
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: exp mrt0, v5, v6, v0, v0 done vm
-; PREGFX10-NEXT: s_endpgm
-;
-; GFX10-LABEL: buffer_load_x1_offen_merged_and:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_dwordx4 v[1:4], v0, s[0:3], 0 offen offset:4
-; GFX10-NEXT: buffer_load_dwordx2 v[5:6], v0, s[0:3], 0 offen offset:28
-; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: exp mrt0, v1, v2, v3, v4 done vm
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: exp mrt0, v5, v6, v0, v0 done vm
-; GFX10-NEXT: s_endpgm
-;
-; GFX11-LABEL: buffer_load_x1_offen_merged_and:
-; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: buffer_load_b128 v[1:4], v0, s[0:3], 0 offen offset:4
-; GFX11-NEXT: buffer_load_b64 v[5:6], v0, s[0:3], 0 offen offset:28
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: exp mrt0, v1, v2, v3, v4 done
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: exp mrt0, v5, v6, v0, v0 done
-; GFX11-NEXT: s_endpgm
-main_body:
- %a1 = add i32 %a, 4
- %a2 = add i32 %a, 8
- %a3 = add i32 %a, 12
- %a4 = add i32 %a, 16
- %a5 = add i32 %a, 28
- %a6 = add i32 %a, 32
- %r1 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a1, i32 0, i32 0)
- %r2 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a2, i32 0, i32 0)
- %r3 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a3, i32 0, i32 0)
- %r4 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a4, i32 0, i32 0)
- %r5 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a5, i32 0, i32 0)
- %r6 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a6, i32 0, i32 0)
- call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
- call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float poison, float poison, i1 true, i1 true)
- ret void
-}
-
-define amdgpu_ps void @buffer_load_x1_offen_merged_or(ptr addrspace(8) inreg %rsrc, i32 %inp) {
-; PREGFX10-LABEL: buffer_load_x1_offen_merged_or:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_lshlrev_b32_e32 v4, 6, v0
-; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], v4, s[0:3], 0 offen offset:4
-; PREGFX10-NEXT: buffer_load_dwordx2 v[4:5], v4, s[0:3], 0 offen offset:28
-; PREGFX10-NEXT: s_waitcnt vmcnt(1)
-; PREGFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: exp mrt0, v4, v5, v0, v0 done vm
-; PREGFX10-NEXT: s_endpgm
-;
-; GFX10-LABEL: buffer_load_x1_offen_merged_or:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 6, v0
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_dwordx4 v[0:3], v6, s[0:3], 0 offen offset:4
-; GFX10-NEXT: buffer_load_dwordx2 v[4:5], v6, s[0:3], 0 offen offset:28
-; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: exp mrt0, v4, v5, v0, v0 done vm
-; GFX10-NEXT: s_endpgm
-;
-; GFX11-LABEL: buffer_load_x1_offen_merged_or:
-; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 6, v0
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: buffer_load_b128 v[0:3], v4, s[0:3], 0 offen offset:4
-; GFX11-NEXT: buffer_load_b64 v[4:5], v4, s[0:3], 0 offen offset:28
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: exp mrt0, v0, v1, v2, v3 done
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: exp mrt0, v4, v5, v0, v0 done
-; GFX11-NEXT: s_endpgm
-main_body:
- %a = shl i32 %inp, 6
- %a1 = or i32 %a, 4
- %a2 = or i32 %a, 8
- %a3 = or i32 %a, 12
- %a4 = or i32 %a, 16
- %a5 = or i32 %a, 28
- %a6 = or i32 %a, 32
- %r1 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a1, i32 0, i32 0)
- %r2 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a2, i32 0, i32 0)
- %r3 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a3, i32 0, i32 0)
- %r4 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a4, i32 0, i32 0)
- %r5 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a5, i32 0, i32 0)
- %r6 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a6, i32 0, i32 0)
- call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
- call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float poison, float poison, i1 true, i1 true)
- ret void
-}
-
-define amdgpu_ps void @buffer_load_x1_offen_merged_glc_slc(ptr addrspace(8) inreg %rsrc, i32 %a) {
-; PREGFX10-LABEL: buffer_load_x1_offen_merged_glc_slc:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: buffer_load_dwordx2 v[1:2], v0, s[0:3], 0 offen offset:4
-; PREGFX10-NEXT: buffer_load_dwordx2 v[3:4], v0, s[0:3], 0 offen offset:12 glc
-; PREGFX10-NEXT: buffer_load_dwordx2 v[5:6], v0, s[0:3], 0 offen offset:28 glc slc
-; PREGFX10-NEXT: s_waitcnt vmcnt(1)
-; PREGFX10-NEXT: exp mrt0, v1, v2, v3, v4 done vm
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: exp mrt0, v5, v6, v0, v0 done vm
-; PREGFX10-NEXT: s_endpgm
-;
-; GFX10-LABEL: buffer_load_x1_offen_merged_glc_slc:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: s_clause 0x2
-; GFX10-NEXT: buffer_load_dwordx2 v[1:2], v0, s[0:3], 0 offen offset:4
-; GFX10-NEXT: buffer_load_dwordx2 v[3:4], v0, s[0:3], 0 offen offset:12 glc
-; GFX10-NEXT: buffer_load_dwordx2 v[5:6], v0, s[0:3], 0 offen offset:28 glc slc
-; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: exp mrt0, v1, v2, v3, v4 done vm
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: exp mrt0, v5, v6, v0, v0 done vm
-; GFX10-NEXT: s_endpgm
-;
-; GFX11-LABEL: buffer_load_x1_offen_merged_glc_slc:
-; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: buffer_load_b64 v[1:2], v0, s[0:3], 0 offen offset:4
-; GFX11-NEXT: buffer_load_b64 v[3:4], v0, s[0:3], 0 offen offset:12 glc
-; GFX11-NEXT: buffer_load_b64 v[5:6], v0, s[0:3], 0 offen offset:28 glc slc
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: exp mrt0, v1, v2, v3, v4 done
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: exp mrt0, v5, v6, v0, v0 done
-; GFX11-NEXT: s_endpgm
-main_body:
- %a1 = add i32 %a, 4
- %a2 = add i32 %a, 8
- %a3 = add i32 %a, 12
- %a4 = add i32 %a, 16
- %a5 = add i32 %a, 28
- %a6 = add i32 %a, 32
- %r1 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a1, i32 0, i32 0)
- %r2 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a2, i32 0, i32 0)
- %r3 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a3, i32 0, i32 1)
- %r4 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a4, i32 0, i32 1)
- %r5 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a5, i32 0, i32 3)
- %r6 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 %a6, i32 0, i32 3)
- call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
- call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float poison, float poison, i1 true, i1 true)
- ret void
-}
-
-define amdgpu_ps void @buffer_load_x2_offen_merged_and(ptr addrspace(8) inreg %rsrc, i32 %a) {
-; PREGFX10-LABEL: buffer_load_x2_offen_merged_and:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
-; PREGFX10-NEXT: s_endpgm
-;
-; GFX10-LABEL: buffer_load_x2_offen_merged_and:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
-; GFX10-NEXT: s_endpgm
-;
-; GFX11-LABEL: buffer_load_x2_offen_merged_and:
-; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:4
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: exp mrt0, v0, v1, v2, v3 done
-; GFX11-NEXT: s_endpgm
-main_body:
- %a1 = add i32 %a, 4
- %a2 = add i32 %a, 12
- %vr1 = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %a1, i32 0, i32 0)
- %vr2 = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %a2, i32 0, i32 0)
- %r1 = extractelement <2 x float> %vr1, i32 0
- %r2 = extractelement <2 x float> %vr1, i32 1
- %r3 = extractelement <2 x float> %vr2, i32 0
- %r4 = extractelement <2 x float> %vr2, i32 1
- call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
- ret void
-}
-
-define amdgpu_ps void @buffer_load_x2_offen_merged_or(ptr addrspace(8) inreg %rsrc, i32 %inp) {
-; PREGFX10-LABEL: buffer_load_x2_offen_merged_or:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_lshlrev_b32_e32 v0, 4, v0
-; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
-; PREGFX10-NEXT: s_endpgm
-;
-; GFX10-LABEL: buffer_load_x2_offen_merged_or:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 4, v0
-; GFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
-; GFX10-NEXT: s_endpgm
-;
-; GFX11-LABEL: buffer_load_x2_offen_merged_or:
-; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 4, v0
-; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:4
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: exp mrt0, v0, v1, v2, v3 done
-; GFX11-NEXT: s_endpgm
-main_body:
- %a = shl i32 %inp, 4
- %a1 = add i32 %a, 4
- %a2 = add i32 %a, 12
- %vr1 = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %a1, i32 0, i32 0)
- %vr2 = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 %a2, i32 0, i32 0)
- %r1 = extractelement <2 x float> %vr1, i32 0
- %r2 = extractelement <2 x float> %vr1, i32 1
- %r3 = extractelement <2 x float> %vr2, i32 0
- %r4 = extractelement <2 x float> %vr2, i32 1
- call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
- ret void
-}
-
-define amdgpu_ps void @buffer_load_x1_offset_merged(ptr addrspace(8) inreg %rsrc) {
-; PREGFX10-LABEL: buffer_load_x1_offset_merged:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], 0 offset:4
-; PREGFX10-NEXT: buffer_load_dwordx2 v[4:5], off, s[0:3], 0 offset:28
-; PREGFX10-NEXT: s_waitcnt vmcnt(1)
-; PREGFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: exp mrt0, v4, v5, v0, v0 done vm
-; PREGFX10-NEXT: s_endpgm
-;
-; GFX10-LABEL: buffer_load_x1_offset_merged:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], 0 offset:4
-; GFX10-NEXT: buffer_load_dwordx2 v[4:5], off, s[0:3], 0 offset:28
-; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: exp mrt0, v4, v5, v0, v0 done vm
-; GFX10-NEXT: s_endpgm
-;
-; GFX11-LABEL: buffer_load_x1_offset_merged:
-; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], 0 offset:4
-; GFX11-NEXT: buffer_load_b64 v[4:5], off, s[0:3], 0 offset:28
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: exp mrt0, v0, v1, v2, v3 done
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: exp mrt0, v4, v5, v0, v0 done
-; GFX11-NEXT: s_endpgm
-main_body:
- %r1 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 4, i32 0, i32 0)
- %r2 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 8, i32 0, i32 0)
- %r3 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 12, i32 0, i32 0)
- %r4 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 16, i32 0, i32 0)
- %r5 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 28, i32 0, i32 0)
- %r6 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 32, i32 0, i32 0)
- call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
- call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float poison, float poison, i1 true, i1 true)
- ret void
-}
-
-define amdgpu_ps void @buffer_load_x2_offset_merged(ptr addrspace(8) inreg %rsrc) {
-; PREGFX10-LABEL: buffer_load_x2_offset_merged:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], 0 offset:4
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
-; PREGFX10-NEXT: s_endpgm
-;
-; GFX10-LABEL: buffer_load_x2_offset_merged:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], 0 offset:4
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
-; GFX10-NEXT: s_endpgm
-;
-; GFX11-LABEL: buffer_load_x2_offset_merged:
-; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], 0 offset:4
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: exp mrt0, v0, v1, v2, v3 done
-; GFX11-NEXT: s_endpgm
-main_body:
- %vr1 = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 4, i32 0, i32 0)
- %vr2 = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) %rsrc, i32 12, i32 0, i32 0)
- %r1 = extractelement <2 x float> %vr1, i32 0
- %r2 = extractelement <2 x float> %vr1, i32 1
- %r3 = extractelement <2 x float> %vr2, i32 0
- %r4 = extractelement <2 x float> %vr2, i32 1
- call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
- ret void
-}
-
define amdgpu_ps {<4 x float>, <2 x float>, float} @buffer_load_int(ptr addrspace(8) inreg) {
; PREGFX10-LABEL: buffer_load_int:
; PREGFX10: ; %bb.0: ; %main_body
@@ -765,6 +593,16 @@ define amdgpu_ps {<4 x float>, <2 x float>, float} @buffer_load_int(ptr addrspac
; GFX11-NEXT: buffer_load_b32 v6, off, s[0:3], 0 slc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: buffer_load_int:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_clause 0x2
+; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], null
+; GFX1250-NEXT: buffer_load_b64 v[4:5], off, s[0:3], null th:TH_LOAD_NT
+; GFX1250-NEXT: buffer_load_b32 v6, off, s[0:3], null th:TH_LOAD_HT
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
main_body:
%data = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) %0, i32 0, i32 0, i32 0)
%data_glc = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) %0, i32 0, i32 0, i32 1)
@@ -799,6 +637,14 @@ define amdgpu_ps float @raw_ptr_buffer_load_ubyte(ptr addrspace(8) inreg %rsrc)
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: raw_ptr_buffer_load_ubyte:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: buffer_load_u8 v0, off, s[0:3], null
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
+; GFX1250-NEXT: ; return to shader part epilog
main_body:
%tmp = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
%tmp2 = zext i8 %tmp to i32
@@ -827,6 +673,14 @@ define amdgpu_ps float @raw_ptr_buffer_load_i16(ptr addrspace(8) inreg %rsrc) {
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cvt_f32_u32_e32 v0, v0
; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: raw_ptr_buffer_load_i16:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: buffer_load_u16 v0, off, s[0:3], null
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_cvt_f32_u32_e32 v0, v0
+; GFX1250-NEXT: ; return to shader part epilog
main_body:
%tmp = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
%tmp2 = zext i16 %tmp to i32
@@ -855,6 +709,14 @@ define amdgpu_ps float @raw_ptr_buffer_load_sbyte(ptr addrspace(8) inreg %rsrc)
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cvt_f32_i32_e32 v0, v0
; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: raw_ptr_buffer_load_sbyte:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: buffer_load_i8 v0, off, s[0:3], null
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX1250-NEXT: ; return to shader part epilog
main_body:
%tmp = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
%tmp2 = sext i8 %tmp to i32
@@ -883,6 +745,14 @@ define amdgpu_ps float @raw_ptr_buffer_load_sshort(ptr addrspace(8) inreg %rsrc)
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cvt_f32_i32_e32 v0, v0
; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: raw_ptr_buffer_load_sshort:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: buffer_load_i16 v0, off, s[0:3], null
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX1250-NEXT: ; return to shader part epilog
main_body:
%tmp = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
%tmp2 = sext i16 %tmp to i32
@@ -912,6 +782,14 @@ define amdgpu_ps void @raw_ptr_buffer_load_f16(ptr addrspace(8) inreg %rsrc, ptr
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ds_store_b16 v0, v1
; GFX11-NEXT: s_endpgm
+;
+; GFX1250-LABEL: raw_ptr_buffer_load_f16:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: buffer_load_u16 v1, off, s[0:3], null
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ds_store_b16 v0, v1
+; GFX1250-NEXT: s_endpgm
main_body:
%val = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
store half %val, ptr addrspace(3) %ptr
@@ -940,6 +818,14 @@ define amdgpu_ps void @raw_ptr_buffer_load_v2f16(ptr addrspace(8) inreg %rsrc, p
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ds_store_b32 v0, v1
; GFX11-NEXT: s_endpgm
+;
+; GFX1250-LABEL: raw_ptr_buffer_load_v2f16:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: buffer_load_b32 v1, off, s[0:3], null
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ds_store_b32 v0, v1
+; GFX1250-NEXT: s_endpgm
main_body:
%val = call <2 x half> @llvm.amdgcn.raw.ptr.buffer.load.v2f16(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
store <2 x half> %val, ptr addrspace(3) %ptr
@@ -968,6 +854,14 @@ define amdgpu_ps void @raw_ptr_buffer_load_v4f16(ptr addrspace(8) inreg %rsrc, p
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ds_store_b64 v0, v[1:2]
; GFX11-NEXT: s_endpgm
+;
+; GFX1250-LABEL: raw_ptr_buffer_load_v4f16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ds_store_b64 v0, v[2:3]
+; GFX1250-NEXT: s_endpgm
%val = call <4 x half> @llvm.amdgcn.raw.ptr.buffer.load.v4f16(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
store <4 x half> %val, ptr addrspace(3) %ptr
ret void
@@ -994,6 +888,14 @@ define amdgpu_ps void @raw_ptr_buffer_load_v8f16(ptr addrspace(8) inreg %rsrc, p
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ds_store_b128 v0, v[1:4]
; GFX11-NEXT: s_endpgm
+;
+; GFX1250-LABEL: raw_ptr_buffer_load_v8f16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: buffer_load_b128 v[2:5], off, s[0:3], null
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ds_store_b128 v0, v[2:5]
+; GFX1250-NEXT: s_endpgm
%val = call <8 x half> @llvm.amdgcn.raw.ptr.buffer.load.v8f16(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
store <8 x half> %val, ptr addrspace(3) %ptr
ret void
@@ -1021,6 +923,14 @@ define amdgpu_ps void @raw_ptr_buffer_load_v2i16(ptr addrspace(8) inreg %rsrc, p
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ds_store_b32 v0, v1
; GFX11-NEXT: s_endpgm
+;
+; GFX1250-LABEL: raw_ptr_buffer_load_v2i16:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: buffer_load_b32 v1, off, s[0:3], null
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ds_store_b32 v0, v1
+; GFX1250-NEXT: s_endpgm
main_body:
%val = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
store <2 x i16> %val, ptr addrspace(3) %ptr
@@ -1049,6 +959,14 @@ define amdgpu_ps void @raw_ptr_buffer_load_v4i16(ptr addrspace(8) inreg %rsrc, p
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ds_store_b64 v0, v[1:2]
; GFX11-NEXT: s_endpgm
+;
+; GFX1250-LABEL: raw_ptr_buffer_load_v4i16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ds_store_b64 v0, v[2:3]
+; GFX1250-NEXT: s_endpgm
%val = call <4 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v4i16(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
store <4 x i16> %val, ptr addrspace(3) %ptr
ret void
@@ -1075,111 +993,19 @@ define amdgpu_ps void @raw_ptr_buffer_load_v8i16(ptr addrspace(8) inreg %rsrc, p
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ds_store_b128 v0, v[1:4]
; GFX11-NEXT: s_endpgm
+;
+; GFX1250-LABEL: raw_ptr_buffer_load_v8i16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: buffer_load_b128 v[2:5], off, s[0:3], null
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ds_store_b128 v0, v[2:5]
+; GFX1250-NEXT: s_endpgm
%val = call <8 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v8i16(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
store <8 x i16> %val, ptr addrspace(3) %ptr
ret void
}
-define amdgpu_ps void @raw_ptr_buffer_load_x1_offset_merged(ptr addrspace(8) inreg %rsrc) {
-; PREGFX10-LABEL: raw_ptr_buffer_load_x1_offset_merged:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], 0 offset:4
-; PREGFX10-NEXT: buffer_load_dwordx2 v[4:5], off, s[0:3], 0 offset:28
-; PREGFX10-NEXT: s_waitcnt vmcnt(1)
-; PREGFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: exp mrt0, v4, v5, v0, v0 done vm
-; PREGFX10-NEXT: s_endpgm
-;
-; GFX10-LABEL: raw_ptr_buffer_load_x1_offset_merged:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], 0 offset:4
-; GFX10-NEXT: buffer_load_dwordx2 v[4:5], off, s[0:3], 0 offset:28
-; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: exp mrt0, v4, v5, v0, v0 done vm
-; GFX10-NEXT: s_endpgm
-;
-; GFX11-LABEL: raw_ptr_buffer_load_x1_offset_merged:
-; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], 0 offset:4
-; GFX11-NEXT: buffer_load_b64 v[4:5], off, s[0:3], 0 offset:28
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: exp mrt0, v0, v1, v2, v3 done
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: exp mrt0, v4, v5, v0, v0 done
-; GFX11-NEXT: s_endpgm
-main_body:
- %r1 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 4, i32 0, i32 0)
- %r2 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 8, i32 0, i32 0)
- %r3 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 12, i32 0, i32 0)
- %r4 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 16, i32 0, i32 0)
- %r5 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 28, i32 0, i32 0)
- %r6 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 32, i32 0, i32 0)
- call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
- call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float poison, float poison, i1 true, i1 true)
- ret void
-}
-
-define amdgpu_ps void @raw_ptr_buffer_load_x1_offset_swizzled_not_merged(ptr addrspace(8) inreg %rsrc) {
-; PREGFX10-LABEL: raw_ptr_buffer_load_x1_offset_swizzled_not_merged:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: buffer_load_dword v0, off, s[0:3], 0 offset:4
-; PREGFX10-NEXT: buffer_load_dword v1, off, s[0:3], 0 offset:8
-; PREGFX10-NEXT: buffer_load_dword v2, off, s[0:3], 0 offset:12
-; PREGFX10-NEXT: buffer_load_dword v3, off, s[0:3], 0 offset:16
-; PREGFX10-NEXT: buffer_load_dword v4, off, s[0:3], 0 offset:28
-; PREGFX10-NEXT: buffer_load_dword v5, off, s[0:3], 0 offset:32
-; PREGFX10-NEXT: s_waitcnt vmcnt(2)
-; PREGFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: exp mrt0, v4, v5, v0, v0 done vm
-; PREGFX10-NEXT: s_endpgm
-;
-; GFX10-LABEL: raw_ptr_buffer_load_x1_offset_swizzled_not_merged:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: s_clause 0x5
-; GFX10-NEXT: buffer_load_dword v0, off, s[0:3], 0 offset:4
-; GFX10-NEXT: buffer_load_dword v1, off, s[0:3], 0 offset:8
-; GFX10-NEXT: buffer_load_dword v2, off, s[0:3], 0 offset:12
-; GFX10-NEXT: buffer_load_dword v3, off, s[0:3], 0 offset:16
-; GFX10-NEXT: buffer_load_dword v4, off, s[0:3], 0 offset:28
-; GFX10-NEXT: buffer_load_dword v5, off, s[0:3], 0 offset:32
-; GFX10-NEXT: s_waitcnt vmcnt(2)
-; GFX10-NEXT: exp mrt0, v0, v1, v2, v3 done vm
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: exp mrt0, v4, v5, v0, v0 done vm
-; GFX10-NEXT: s_endpgm
-;
-; GFX11-LABEL: raw_ptr_buffer_load_x1_offset_swizzled_not_merged:
-; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: s_clause 0x5
-; GFX11-NEXT: buffer_load_b32 v0, off, s[0:3], 0 offset:4
-; GFX11-NEXT: buffer_load_b32 v1, off, s[0:3], 0 offset:8
-; GFX11-NEXT: buffer_load_b32 v2, off, s[0:3], 0 offset:12
-; GFX11-NEXT: buffer_load_b32 v3, off, s[0:3], 0 offset:16
-; GFX11-NEXT: buffer_load_b32 v4, off, s[0:3], 0 offset:28
-; GFX11-NEXT: buffer_load_b32 v5, off, s[0:3], 0 offset:32
-; GFX11-NEXT: s_waitcnt vmcnt(2)
-; GFX11-NEXT: exp mrt0, v0, v1, v2, v3 done
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: exp mrt0, v4, v5, v0, v0 done
-; GFX11-NEXT: s_endpgm
-main_body:
- %r1 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 4, i32 0, i32 8)
- %r2 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 8, i32 0, i32 8)
- %r3 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 12, i32 0, i32 8)
- %r4 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 16, i32 0, i32 8)
- %r5 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 28, i32 0, i32 8)
- %r6 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 32, i32 0, i32 8)
- call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
- call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float poison, float poison, i1 true, i1 true)
- ret void
-}
-
define double @buffer_load_f64__voffset_add(ptr addrspace(8) inreg %rsrc, i32 %voffset) {
; PREGFX10-LABEL: buffer_load_f64__voffset_add:
; PREGFX10: ; %bb.0:
@@ -1201,6 +1027,15 @@ define double @buffer_load_f64__voffset_add(ptr addrspace(8) inreg %rsrc, i32 %v
; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_f64__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call double @llvm.amdgcn.raw.ptr.buffer.load.f64(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret double %data
@@ -1227,6 +1062,15 @@ define <2 x double> @buffer_load_v2f64__voffset_add(ptr addrspace(8) inreg %rsrc
; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_v2f64__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call <2 x double> @llvm.amdgcn.raw.ptr.buffer.load.v2f64(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret <2 x double> %data
@@ -1253,6 +1097,15 @@ define i64 @buffer_load_i64__voffset_add(ptr addrspace(8) inreg %rsrc, i32 %voff
; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_i64__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret i64 %data
@@ -1279,6 +1132,15 @@ define <2 x i64> @buffer_load_v2i64__voffset_add(ptr addrspace(8) inreg %rsrc, i
; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_v2i64__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call <2 x i64> @llvm.amdgcn.raw.ptr.buffer.load.v2i64(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret <2 x i64> %data
@@ -1305,6 +1167,15 @@ define ptr @buffer_load_p0__voffset_add(ptr addrspace(8) inreg %rsrc, i32 %voffs
; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_p0__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call ptr @llvm.amdgcn.raw.ptr.buffer.load.p0(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret ptr %data
@@ -1331,6 +1202,15 @@ define <2 x ptr> @buffer_load_v2p0__voffset_add(ptr addrspace(8) inreg %rsrc, i3
; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_v2p0__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call <2 x ptr> @llvm.amdgcn.raw.ptr.buffer.load.p0(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret <2 x ptr> %data
@@ -1357,6 +1237,15 @@ define ptr addrspace(1) @buffer_load_p1__voffset_add(ptr addrspace(8) inreg %rsr
; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_p1__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call ptr addrspace(1) @llvm.amdgcn.raw.ptr.buffer.load.p1(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret ptr addrspace(1) %data
@@ -1383,6 +1272,15 @@ define <2 x ptr addrspace(1)> @buffer_load_v2p1__voffset_add(ptr addrspace(8) in
; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_v2p1__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call <2 x ptr addrspace(1)> @llvm.amdgcn.raw.ptr.buffer.load.p1(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret <2 x ptr addrspace(1)> %data
@@ -1409,6 +1307,15 @@ define ptr addrspace(4) @buffer_load_p4__voffset_add(ptr addrspace(8) inreg %rsr
; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_p4__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call ptr addrspace(4) @llvm.amdgcn.raw.ptr.buffer.load.p4(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret ptr addrspace(4) %data
@@ -1435,6 +1342,15 @@ define <2 x ptr addrspace(4)> @buffer_load_v2p4__voffset_add(ptr addrspace(8) in
; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_v2p4__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call <2 x ptr addrspace(4)> @llvm.amdgcn.raw.ptr.buffer.load.p4(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret <2 x ptr addrspace(4)> %data
@@ -1461,6 +1377,15 @@ define ptr addrspace(999) @buffer_load_p999__voffset_add(ptr addrspace(8) inreg
; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_p999__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call ptr addrspace(999) @llvm.amdgcn.raw.ptr.buffer.load.p999(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret ptr addrspace(999) %data
@@ -1487,6 +1412,15 @@ define <2 x ptr addrspace(999)> @buffer_load_v2p999__voffset_add(ptr addrspace(8
; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_v2p999__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call <2 x ptr addrspace(999)> @llvm.amdgcn.raw.ptr.buffer.load.p999(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret <2 x ptr addrspace(999)> %data
@@ -1513,6 +1447,15 @@ define ptr addrspace(2) @buffer_load_p2__voffset_add(ptr addrspace(8) inreg %rsr
; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_p2__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call ptr addrspace(2) @llvm.amdgcn.raw.ptr.buffer.load.p2(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret ptr addrspace(2) %data
@@ -1539,6 +1482,15 @@ define <2 x ptr addrspace(2)> @buffer_load_v2p2__voffset_add(ptr addrspace(8) in
; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_v2p2__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call <2 x ptr addrspace(2)> @llvm.amdgcn.raw.ptr.buffer.load.v2p2(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret <2 x ptr addrspace(2)> %data
@@ -1558,6 +1510,15 @@ define <3 x ptr addrspace(2)> @buffer_load_v3p2__voffset_add(ptr addrspace(8) in
; GFX11-NEXT: buffer_load_b96 v[0:2], v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_v3p2__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b96 v[0:2], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call <3 x ptr addrspace(2)> @llvm.amdgcn.raw.ptr.buffer.load.v3p2(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret <3 x ptr addrspace(2)> %data
@@ -1584,6 +1545,15 @@ define <4 x ptr addrspace(2)> @buffer_load_v4p2__voffset_add(ptr addrspace(8) in
; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_v4p2__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call <4 x ptr addrspace(2)> @llvm.amdgcn.raw.ptr.buffer.load.v4p2(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret <4 x ptr addrspace(2)> %data
@@ -1610,6 +1580,15 @@ define ptr addrspace(3) @buffer_load_p3__voffset_add(ptr addrspace(8) inreg %rsr
; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_p3__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call ptr addrspace(3) @llvm.amdgcn.raw.ptr.buffer.load.p3(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret ptr addrspace(3) %data
@@ -1636,6 +1615,15 @@ define <2 x ptr addrspace(3)> @buffer_load_v2p3__voffset_add(ptr addrspace(8) in
; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_v2p3__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call <2 x ptr addrspace(3)> @llvm.amdgcn.raw.ptr.buffer.load.v2p3(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret <2 x ptr addrspace(3)> %data
@@ -1655,6 +1643,15 @@ define <3 x ptr addrspace(3)> @buffer_load_v3p3__voffset_add(ptr addrspace(8) in
; GFX11-NEXT: buffer_load_b96 v[0:2], v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_v3p3__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b96 v[0:2], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call <3 x ptr addrspace(3)> @llvm.amdgcn.raw.ptr.buffer.load.v3p3(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret <3 x ptr addrspace(3)> %data
@@ -1681,6 +1678,15 @@ define <4 x ptr addrspace(3)> @buffer_load_v4p3__voffset_add(ptr addrspace(8) in
; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_v4p3__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call <4 x ptr addrspace(3)> @llvm.amdgcn.raw.ptr.buffer.load.v4p3(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret <4 x ptr addrspace(3)> %data
@@ -1707,6 +1713,15 @@ define ptr addrspace(5) @buffer_load_p5__voffset_add(ptr addrspace(8) inreg %rsr
; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_p5__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call ptr addrspace(5) @llvm.amdgcn.raw.ptr.buffer.load.p5(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret ptr addrspace(5) %data
@@ -1733,6 +1748,15 @@ define <2 x ptr addrspace(5)> @buffer_load_v2p5__voffset_add(ptr addrspace(8) in
; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_v2p5__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call <2 x ptr addrspace(5)> @llvm.amdgcn.raw.ptr.buffer.load.v2p5(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret <2 x ptr addrspace(5)> %data
@@ -1752,6 +1776,15 @@ define <3 x ptr addrspace(5)> @buffer_load_v3p5__voffset_add(ptr addrspace(8) in
; GFX11-NEXT: buffer_load_b96 v[0:2], v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_v3p5__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b96 v[0:2], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call <3 x ptr addrspace(5)> @llvm.amdgcn.raw.ptr.buffer.load.v3p5(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret <3 x ptr addrspace(5)> %data
@@ -1778,6 +1811,15 @@ define <4 x ptr addrspace(5)> @buffer_load_v4p5__voffset_add(ptr addrspace(8) in
; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_v4p5__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call <4 x ptr addrspace(5)> @llvm.amdgcn.raw.ptr.buffer.load.v4p5(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret <4 x ptr addrspace(5)> %data
@@ -1804,6 +1846,15 @@ define ptr addrspace(6) @buffer_load_p6__voffset_add(ptr addrspace(8) inreg %rsr
; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_p6__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call ptr addrspace(6) @llvm.amdgcn.raw.ptr.buffer.load.p6(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret ptr addrspace(6) %data
@@ -1830,6 +1881,15 @@ define <2 x ptr addrspace(6)> @buffer_load_v2p6__voffset_add(ptr addrspace(8) in
; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_v2p6__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call <2 x ptr addrspace(6)> @llvm.amdgcn.raw.ptr.buffer.load.v2p6(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret <2 x ptr addrspace(6)> %data
@@ -1849,6 +1909,15 @@ define <3 x ptr addrspace(6)> @buffer_load_v3p6__voffset_add(ptr addrspace(8) in
; GFX11-NEXT: buffer_load_b96 v[0:2], v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_v3p6__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b96 v[0:2], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call <3 x ptr addrspace(6)> @llvm.amdgcn.raw.ptr.buffer.load.v3p6(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret <3 x ptr addrspace(6)> %data
@@ -1875,18 +1944,65 @@ define <4 x ptr addrspace(6)> @buffer_load_v4p6__voffset_add(ptr addrspace(8) in
; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:60
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: buffer_load_v4p6__voffset_add:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
+; GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 60
%data = call <4 x ptr addrspace(6)> @llvm.amdgcn.raw.ptr.buffer.load.v4p6(ptr addrspace(8) %rsrc, i32 %voffset.add, i32 0, i32 0)
ret <4 x ptr addrspace(6)> %data
}
+define amdgpu_ps <4 x float> @raw_ptr_large_zero_soffset(ptr addrspace(8) inreg %rsrc) {
+; SI-LABEL: raw_ptr_large_zero_soffset:
+; SI: v_mov_b32_e32 [[VOFF:v[0-9]+]], 0x1000
+; SI-NEXT: buffer_load_dwordx4 v[0:3], [[VOFF]], s[0:3], 0 offen offset:4092
+;
+; VI-LABEL: raw_ptr_large_zero_soffset:
+; VI: v_mov_b32_e32 [[VOFF:v[0-9]+]], 0x1000
+; VI-NEXT: buffer_load_dwordx4 v[0:3], [[VOFF]], s[0:3], 0 offen offset:4092
+main_body:
+ %data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 8188, i32 0, i32 0)
+ ret <4 x float> %data
+}
+
+define amdgpu_ps <4 x float> @raw_ptr_large_safe_soffset(ptr addrspace(8) inreg %rsrc) {
+; SI-LABEL: raw_ptr_large_safe_soffset:
+; SI: v_mov_b32_e32 [[VOFF:v[0-9]+]], 0x1000
+; SI-NEXT: buffer_load_dwordx4 v[0:3], [[VOFF]], s[0:3], 16 offen offset:4092
+;
+; VI-LABEL: raw_ptr_large_safe_soffset:
+; VI: v_mov_b32_e32 [[VOFF:v[0-9]+]], 0x1000
+; VI-NEXT: buffer_load_dwordx4 v[0:3], [[VOFF]], s[0:3], 16 offen offset:4092
+main_body:
+ %data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 8188, i32 16, i32 0)
+ ret <4 x float> %data
+}
+
+define amdgpu_ps <4 x float> @raw_ptr_large_overflowing_soffset(ptr addrspace(8) inreg %rsrc) {
+; SI-LABEL: raw_ptr_large_overflowing_soffset:
+; SI: v_mov_b32_e32 [[VOFF:v[0-9]+]], 0x1000
+; SI-NEXT: buffer_load_dwordx4 v[0:3], [[VOFF]], s[0:3], -16 offen offset:4092
+;
+; VI-LABEL: raw_ptr_large_overflowing_soffset:
+; VI: v_mov_b32_e32 [[VOFF:v[0-9]+]], 0x1000
+; VI-NEXT: buffer_load_dwordx4 v[0:3], [[VOFF]], s[0:3], -16 offen offset:4092
+main_body:
+ %data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 8188, i32 -16, i32 0)
+ ret <4 x float> %data
+}
+
declare float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8), i32, i32, i32) #0
declare <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8), i32, i32, i32) #0
declare <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8), i32, i32, i32) #0
declare i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8), i32, i32, i32) #0
declare <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8), i32, i32, i32) #0
declare <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8), i32, i32, i32) #0
-declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #0
declare i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8), i32, i32, i32) #0
declare i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8), i32, i32, i32) #0
declare <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8), i32, i32, i32) #0
>From d642f108fc75216809caa1af24ff88c884f2afac Mon Sep 17 00:00:00 2001
From: Krzysztof Drewniak <Krzysztof.Drewniak at amd.com>
Date: Wed, 24 Jun 2026 21:51:42 +0000
Subject: [PATCH 46/46] [AMDGPU] Split large raw buffer offsets to SOFFSET
where allowed
If we're not doing vindex or swizzling and aren't very old (that is,
we're not in a case where the buffer OOB formula is
`voffset +[checked] imm + payload > num_records -[saturating]
soffset`) or a politer version on gfx1250 *and* the large constant
offset is added to voffset (if any) in a NUW like way *and* the add to
SOFFSET wouldn't overflow (so, in practice, if SOFFSET is a constant
for now), then we can safely move the part that doesn't fit into the
instruction immediate into the SOFFSET field instead of the VOFFSET
field, saving on VGPRs.
This commit implements this change - and, while we're here, makes
GlobalIsel match SelectionDAG in terms of recognizing an `or disjoint`
as add-like.
Note that the exclusion of SEA_ISLANDS and older is taken from
selectMUBEFOffsets (the s_buffer_load version) just to be safe.
AI disclosure: AI wrote this and I poked it into shape a bunch.
Co-Authored-By: Codex <codex at openai.com>
---
.../Target/AMDGPU/AMDGPUGlobalISelUtils.cpp | 8 +-
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 116 ++++--
llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h | 12 +-
.../Target/AMDGPU/AMDGPURegisterBankInfo.cpp | 60 ---
.../Target/AMDGPU/AMDGPURegisterBankInfo.h | 3 -
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 166 +++++---
llvm/lib/Target/AMDGPU/SIISelLowering.h | 17 +-
.../llvm.amdgcn.raw.ptr.buffer.load.ll | 13 +-
...llvm.amdgcn.raw.ptr.buffer.offset-split.ll | 13 +-
.../legalize-amdgcn.raw.ptr.buffer.load.ll | 19 +-
.../AMDGPU/llvm.amdgcn.raw.buffer.load.ll | 198 ++++++----
.../AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.ll | 353 +++++++++++++++---
.../llvm.amdgcn.raw.ptr.tbuffer.load.ll | 112 +++---
.../llvm.amdgcn.raw.ptr.tbuffer.store.ll | 100 +++--
.../AMDGPU/llvm.amdgcn.raw.tbuffer.load.ll | 116 +++---
.../AMDGPU/llvm.amdgcn.raw.tbuffer.store.ll | 104 ++++--
16 files changed, 928 insertions(+), 482 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp
index f36935d8c0e8f..2bfeb0546e1af 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp
@@ -46,7 +46,6 @@ AMDGPU::getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg,
assert(MRI.getType(Reg).getScalarSizeInBits() == 32);
return std::pair(Reg, 0);
}
- // TODO: Handle G_OR used for add case
if (mi_match(Def->getOperand(2).getReg(), MRI, m_ICst(Offset)))
return std::pair(Def->getOperand(1).getReg(), Offset);
@@ -56,9 +55,10 @@ AMDGPU::getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg,
}
Register Base;
- if (ValueTracking && mi_match(Reg, MRI, m_GOr(m_Reg(Base), m_ICst(Offset))) &&
- ValueTracking->maskedValueIsZero(Base,
- APInt(32, Offset, /*isSigned=*/true)))
+ if (mi_match(Reg, MRI, m_GOr(m_Reg(Base), m_ICst(Offset))) &&
+ (Def->getFlag(MachineInstr::Disjoint) ||
+ (ValueTracking && ValueTracking->maskedValueIsZero(
+ Base, APInt(32, Offset, /*isSigned=*/true)))))
return std::pair(Base, Offset);
// Handle G_PTRTOINT (G_PTR_ADD base, const) case
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index c57ef9392a4ca..7281bec93850b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -36,6 +36,7 @@
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
+#include "llvm/Support/CheckedArithmetic.h"
#define DEBUG_TYPE "amdgpu-legalinfo"
@@ -6493,60 +6494,115 @@ bool AMDGPULegalizerInfo::legalizeIsAddrSpace(MachineInstr &MI,
return true;
}
+static bool canMoveBufferOffsetToSOffset(const GCNSubtarget &ST,
+ bool HasLinearOOB, bool IsNUW,
+ uint32_t Offset) {
+ // There is a hardware bug in SI and CI which prevents address clamping in
+ // MUBUF instructions from working correctly with SOffsets. The immediate
+ // offset is unaffected.
+ return HasLinearOOB && IsNUW && static_cast<int32_t>(Offset) > 0 &&
+ ST.getGeneration() > AMDGPUSubtarget::SEA_ISLANDS;
+}
+
+static std::optional<uint32_t>
+getCombinedBufferSOffset(MachineRegisterInfo &MRI, Register SOffset,
+ uint32_t Offset) {
+ APInt C;
+ if (!mi_match(SOffset, MRI, m_ICst(C)))
+ return std::nullopt;
+ return checkedAddUnsigned<uint32_t>(static_cast<uint32_t>(C.getZExtValue()),
+ Offset);
+}
+
// The raw.(t)buffer and struct.(t)buffer intrinsics have two offset args:
// offset (the offset that is included in bounds checking and swizzling, to be
// split between the instruction's voffset and immoffset fields) and soffset
// (the offset that is excluded from bounds checking and swizzling, to go in
-// the instruction's soffset field). This function takes the first kind of
-// offset and figures out how to split it between voffset and immoffset.
-std::pair<Register, unsigned>
+// the instruction's soffset field, except that it does affect num_records and
+// so can (if there's no unsigned overflow) be used for bounds checking in raw.*
+// intrinsics). This function takes both offsets and figures out how to split
+// them between voffset, soffset, and immoffset.
+std::tuple<Register, Register, unsigned>
AMDGPULegalizerInfo::splitBufferOffsets(MachineIRBuilder &B,
- Register OrigOffset) const {
+ Register OrigOffset,
+ Register OrigSOffset, bool HasLinearOOB,
+ GISelValueTracking *VT) const {
const unsigned MaxImm = SIInstrInfo::getMaxMUBUFImmOffset(ST);
Register BaseReg;
unsigned ImmOffset;
const LLT S32 = LLT::scalar(32);
MachineRegisterInfo &MRI = *B.getMRI();
-
+ Register SOffset = OrigSOffset;
// On GFX1250+, voffset and immoffset are zero-extended from 32 bits before
// being added, so we can only safely match a 32-bit addition with no unsigned
// overflow.
bool CheckNUW = ST.hasGFX1250Insts();
- std::tie(BaseReg, ImmOffset) = AMDGPU::getBaseWithConstantOffset(
- MRI, OrigOffset, /*KnownBits=*/nullptr, CheckNUW);
+ std::tie(BaseReg, ImmOffset) =
+ AMDGPU::getBaseWithConstantOffset(MRI, OrigOffset, VT, CheckNUW);
+ bool IsNUW = false;
+ if (ImmOffset) {
+ if (!BaseReg) {
+ IsNUW = true;
+ } else if (MachineInstr *OffsetDef =
+ getDefIgnoringCopies(OrigOffset, MRI)) {
+ // Match SelectionDAG: a G_OR with a constant reaches here only when it is
+ // add-like, and such an OR satisfies the no-wrap condition.
+ IsNUW = OffsetDef->getOpcode() == TargetOpcode::G_OR ||
+ (OffsetDef->getOpcode() == TargetOpcode::G_ADD &&
+ OffsetDef->getFlag(MachineInstr::NoUWrap));
+ }
+ }
// If BaseReg is a pointer, convert it to int.
- if (MRI.getType(BaseReg).isPointer())
+ if (BaseReg && MRI.getType(BaseReg).isPointer())
BaseReg = B.buildPtrToInt(MRI.getType(OrigOffset), BaseReg).getReg(0);
- // If the immediate value is too big for the immoffset field, put only bits
- // that would normally fit in the immoffset field. The remaining value that
- // is copied/added for the voffset field is a large power of 2, and it
- // stands more chance of being CSEd with the copy/add for another similar
- // load/store.
- // However, do not do that rounding down if that is a negative
- // number, as it appears to be illegal to have a negative offset in the
- // vgpr, even if adding the immediate offset makes it positive.
unsigned Overflow = ImmOffset & ~MaxImm;
ImmOffset -= Overflow;
+
+ // If the immediate value is too big for the immoffset field, put only bits
+ // that would normally fit in the immoffset field. The remaining value
+ // copied/added for the voffset field is a large power of 2, and it stands
+ // more chance of being CSEd with the copy/add for another similar
+ // load/store. For eligible raw.(t)buffer.* operations, the branch below may
+ // instead move this overflow to soffset.
+ //
+ // Do not do the rounding-down split for negative numbers, as the addition of
+ // immoffset and voffset is checked for unsigned overflow (which causes the
+ // load/store to be marked OOB) and the soffset is (on gfx8 through gfx12; see
+ // the ISA MUBUF/MTBUF addressing description) subtracted from num_records and
+ // not added to the offset.
if ((int32_t)Overflow < 0) {
Overflow += ImmOffset;
ImmOffset = 0;
}
- if (Overflow != 0) {
- if (!BaseReg) {
- BaseReg = B.buildConstant(S32, Overflow).getReg(0);
- } else {
- auto OverflowVal = B.buildConstant(S32, Overflow);
- BaseReg = B.buildAdd(S32, BaseReg, OverflowVal).getReg(0);
+ std::optional<uint32_t> NewSOffset;
+ if (Overflow &&
+ canMoveBufferOffsetToSOffset(ST, HasLinearOOB, IsNUW, Overflow))
+ NewSOffset = getCombinedBufferSOffset(MRI, SOffset, Overflow);
+
+ if (NewSOffset) {
+ // For raw.(t)buffer.* operations with a bare constant or no-wrap add, the
+ // overflow can be added to an existing soffset if the addition would not
+ // cause unsigned overflow. We conservatively restrict ourselves to
+ // constant soffsets here.
+ SOffset = B.buildConstant(S32, *NewSOffset).getReg(0);
+ } else {
+ if (Overflow != 0) {
+ if (!BaseReg) {
+ BaseReg = B.buildConstant(S32, Overflow).getReg(0);
+ } else {
+ auto OverflowVal = B.buildConstant(S32, Overflow);
+ BaseReg = B.buildAdd(S32, BaseReg, OverflowVal).getReg(0);
+ }
}
}
if (!BaseReg)
BaseReg = B.buildConstant(S32, 0).getReg(0);
- return std::pair(BaseReg, ImmOffset);
+ return {BaseReg, SOffset, ImmOffset};
}
/// Handle register layout difference for f16 images for some subtargets.
@@ -6694,7 +6750,8 @@ bool AMDGPULegalizerInfo::legalizeBufferStore(MachineInstr &MI,
unsigned AuxiliaryData = MI.getOperand(5 + OpOffset).getImm();
- std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);
+ std::tie(VOffset, SOffset, ImmOffset) = splitBufferOffsets(
+ B, VOffset, SOffset, !HasVIndex, Helper.getValueTracking());
unsigned Opc;
if (IsTyped) {
@@ -6832,7 +6889,8 @@ bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
const bool IsD16 = IsFormat && (EltTy.getSizeInBits() == 16);
const bool Unpacked = ST.hasUnpackedD16VMem();
- std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);
+ std::tie(VOffset, SOffset, ImmOffset) = splitBufferOffsets(
+ B, VOffset, SOffset, !HasVIndex, Helper.getValueTracking());
unsigned Opc;
@@ -7016,8 +7074,9 @@ static unsigned getBufferAtomicPseudo(Intrinsic::ID IntrID) {
}
bool AMDGPULegalizerInfo::legalizeBufferAtomic(MachineInstr &MI,
- MachineIRBuilder &B,
+ LegalizerHelper &Helper,
Intrinsic::ID IID) const {
+ MachineIRBuilder &B = Helper.MIRBuilder;
const bool IsCmpSwap =
IID == Intrinsic::amdgcn_raw_buffer_atomic_cmpswap ||
IID == Intrinsic::amdgcn_struct_buffer_atomic_cmpswap ||
@@ -7058,7 +7117,8 @@ bool AMDGPULegalizerInfo::legalizeBufferAtomic(MachineInstr &MI,
MachineMemOperand *MMO = *MI.memoperands_begin();
unsigned ImmOffset;
- std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);
+ std::tie(VOffset, SOffset, ImmOffset) = splitBufferOffsets(
+ B, VOffset, SOffset, !HasVIndex, Helper.getValueTracking());
auto MIB = B.buildInstr(getBufferAtomicPseudo(IID))
.addDef(Dst)
@@ -8511,7 +8571,7 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
case Intrinsic::amdgcn_struct_buffer_atomic_fadd:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
- return legalizeBufferAtomic(MI, B, IntrID);
+ return legalizeBufferAtomic(MI, Helper, IntrID);
case Intrinsic::amdgcn_rsq_clamp:
return legalizeRsqClampIntrinsic(MI, MRI, B);
case Intrinsic::amdgcn_image_bvh_intersect_ray:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
index 89819fe990f5a..27facfe7b82ec 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
@@ -14,14 +14,16 @@
#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUMACHINELEGALIZER_H
#define LLVM_LIB_TARGET_AMDGPU_AMDGPUMACHINELEGALIZER_H
-#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
#include "AMDGPUArgumentUsageInfo.h"
#include "SIInstrInfo.h"
+#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
+#include <tuple>
namespace llvm {
class GCNTargetMachine;
class GCNSubtarget;
+class GISelValueTracking;
class MachineIRBuilder;
namespace AMDGPU {
@@ -207,8 +209,10 @@ class AMDGPULegalizerInfo final : public LegalizerInfo {
bool legalizeIsAddrSpace(MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B, unsigned AddrSpace) const;
- std::pair<Register, unsigned> splitBufferOffsets(MachineIRBuilder &B,
- Register OrigOffset) const;
+ std::tuple<Register, Register, unsigned>
+ splitBufferOffsets(MachineIRBuilder &B, Register OrigOffset,
+ Register OrigSOffset, bool HasLinearOOB,
+ GISelValueTracking *VT) const;
Register handleD16VData(MachineIRBuilder &B, MachineRegisterInfo &MRI,
Register Reg, bool ImageStore = false) const;
@@ -219,7 +223,7 @@ class AMDGPULegalizerInfo final : public LegalizerInfo {
bool IsTyped, bool IsFormat) const;
bool legalizeBufferLoad(MachineInstr &MI, LegalizerHelper &Helper,
bool IsFormat, bool IsTyped) const;
- bool legalizeBufferAtomic(MachineInstr &MI, MachineIRBuilder &B,
+ bool legalizeBufferAtomic(MachineInstr &MI, LegalizerHelper &Helper,
Intrinsic::ID IID) const;
bool legalizeBVHIntersectRayIntrinsic(MachineInstr &MI,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 200234c23c886..4a3e402f53c83 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -1807,66 +1807,6 @@ Register AMDGPURegisterBankInfo::handleD16VData(MachineIRBuilder &B,
.getReg(0);
}
-static std::pair<Register, unsigned>
-getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg) {
- int64_t Const;
- if (mi_match(Reg, MRI, m_ICst(Const)))
- return std::pair(Register(), Const);
-
- Register Base;
- if (mi_match(Reg, MRI, m_GAdd(m_Reg(Base), m_ICst(Const))))
- return std::pair(Base, Const);
-
- // TODO: Handle G_OR used for add case
- return std::pair(Reg, 0);
-}
-
-std::pair<Register, unsigned>
-AMDGPURegisterBankInfo::splitBufferOffsets(MachineIRBuilder &B,
- Register OrigOffset) const {
- const unsigned MaxImm = SIInstrInfo::getMaxMUBUFImmOffset(Subtarget);
- Register BaseReg;
- unsigned ImmOffset;
- const LLT S32 = LLT::scalar(32);
-
- // TODO: Use AMDGPU::getBaseWithConstantOffset() instead.
- std::tie(BaseReg, ImmOffset) = getBaseWithConstantOffset(*B.getMRI(),
- OrigOffset);
-
- unsigned C1 = 0;
- if (ImmOffset != 0) {
- // If the immediate value is too big for the immoffset field, put only bits
- // that would normally fit in the immoffset field. The remaining value that
- // is copied/added for the voffset field is a large power of 2, and it
- // stands more chance of being CSEd with the copy/add for another similar
- // load/store.
- // However, do not do that rounding down if that is a negative
- // number, as it appears to be illegal to have a negative offset in the
- // vgpr, even if adding the immediate offset makes it positive.
- unsigned Overflow = ImmOffset & ~MaxImm;
- ImmOffset -= Overflow;
- if (static_cast<int32_t>(Overflow) < 0) {
- Overflow += ImmOffset;
- ImmOffset = 0;
- }
-
- C1 = ImmOffset;
- if (Overflow != 0) {
- if (!BaseReg)
- BaseReg = B.buildConstant(S32, Overflow).getReg(0);
- else {
- auto OverflowVal = B.buildConstant(S32, Overflow);
- BaseReg = B.buildAdd(S32, BaseReg, OverflowVal).getReg(0);
- }
- }
- }
-
- if (!BaseReg)
- BaseReg = B.buildConstant(S32, 0).getReg(0);
-
- return {BaseReg, C1};
-}
-
bool AMDGPURegisterBankInfo::buildVCopy(MachineIRBuilder &B, Register DstReg,
Register SrcReg) const {
MachineRegisterInfo &MRI = *B.getMRI();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h
index 5f550b426ec09..6ac00008d4d4d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h
@@ -90,9 +90,6 @@ class AMDGPURegisterBankInfo final : public AMDGPUGenRegisterBankInfo {
Register handleD16VData(MachineIRBuilder &B, MachineRegisterInfo &MRI,
Register Reg) const;
- std::pair<Register, unsigned>
- splitBufferOffsets(MachineIRBuilder &B, Register Offset) const;
-
/// See RegisterBankInfo::applyMapping.
void applyMappingImpl(MachineIRBuilder &Builder,
const OperandsMapper &OpdMapper) const override;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index a29070953ee38..2a01718b1f653 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -44,6 +44,7 @@
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
#include "llvm/IR/MDBuilder.h"
+#include "llvm/Support/CheckedArithmetic.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/ModRef.h"
@@ -11384,8 +11385,10 @@ SDValue SITargetLowering::lowerRawBufferAtomicIntrin(SDValue Op,
SDValue VData = Op.getOperand(2);
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(3), DAG);
- auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(4), DAG);
- auto SOffset = selectSOffset(Op.getOperand(5), DAG, Subtarget);
+ auto [VOffset, SOffset, Offset] =
+ splitBufferOffsets(Op.getOperand(4), Op.getOperand(5), DAG,
+ /*HasLinearOOB=*/true);
+ SOffset = selectSOffset(SOffset, DAG, Subtarget);
SDValue Ops[] = {
Op.getOperand(0), // Chain
VData, // vdata
@@ -11412,8 +11415,10 @@ SITargetLowering::lowerStructBufferAtomicIntrin(SDValue Op, SelectionDAG &DAG,
SDValue VData = Op.getOperand(2);
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(3), DAG);
- auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(5), DAG);
- auto SOffset = selectSOffset(Op.getOperand(6), DAG, Subtarget);
+ auto [VOffset, SOffset, Offset] =
+ splitBufferOffsets(Op.getOperand(5), Op.getOperand(6), DAG,
+ /*HasLinearOOB=*/false);
+ SOffset = selectSOffset(SOffset, DAG, Subtarget);
SDValue Ops[] = {
Op.getOperand(0), // Chain
VData, // vdata
@@ -11513,8 +11518,10 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
IntrID == Intrinsic::amdgcn_raw_ptr_buffer_load_format;
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(2), DAG);
- auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(3), DAG);
- auto SOffset = selectSOffset(Op.getOperand(4), DAG, Subtarget);
+ auto [VOffset, SOffset, Offset] =
+ splitBufferOffsets(Op.getOperand(3), Op.getOperand(4), DAG,
+ /*HasLinearOOB=*/true);
+ SOffset = selectSOffset(SOffset, DAG, Subtarget);
SDValue Ops[] = {
Op.getOperand(0), // Chain
Rsrc, // rsrc
@@ -11540,8 +11547,10 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
IntrID == Intrinsic::amdgcn_struct_ptr_buffer_load_format;
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(2), DAG);
- auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(4), DAG);
- auto SOffset = selectSOffset(Op.getOperand(5), DAG, Subtarget);
+ auto [VOffset, SOffset, Offset] =
+ splitBufferOffsets(Op.getOperand(4), Op.getOperand(5), DAG,
+ /*HasLinearOOB=*/false);
+ SOffset = selectSOffset(SOffset, DAG, Subtarget);
SDValue Ops[] = {
Op.getOperand(0), // Chain
Rsrc, // rsrc
@@ -11560,8 +11569,10 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
MemSDNode *M = cast<MemSDNode>(Op);
EVT LoadVT = Op.getValueType();
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(2), DAG);
- auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(3), DAG);
- auto SOffset = selectSOffset(Op.getOperand(4), DAG, Subtarget);
+ auto [VOffset, SOffset, Offset] =
+ splitBufferOffsets(Op.getOperand(3), Op.getOperand(4), DAG,
+ /*HasLinearOOB=*/true);
+ SOffset = selectSOffset(SOffset, DAG, Subtarget);
SDValue Ops[] = {
Op.getOperand(0), // Chain
@@ -11587,8 +11598,10 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
MemSDNode *M = cast<MemSDNode>(Op);
EVT LoadVT = Op.getValueType();
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(2), DAG);
- auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(4), DAG);
- auto SOffset = selectSOffset(Op.getOperand(5), DAG, Subtarget);
+ auto [VOffset, SOffset, Offset] =
+ splitBufferOffsets(Op.getOperand(4), Op.getOperand(5), DAG,
+ /*HasLinearOOB=*/false);
+ SOffset = selectSOffset(SOffset, DAG, Subtarget);
SDValue Ops[] = {
Op.getOperand(0), // Chain
@@ -11725,8 +11738,10 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap: {
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(4), DAG);
- auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(5), DAG);
- auto SOffset = selectSOffset(Op.getOperand(6), DAG, Subtarget);
+ auto [VOffset, SOffset, Offset] =
+ splitBufferOffsets(Op.getOperand(5), Op.getOperand(6), DAG,
+ /*HasLinearOOB=*/true);
+ SOffset = selectSOffset(SOffset, DAG, Subtarget);
SDValue Ops[] = {
Op.getOperand(0), // Chain
Op.getOperand(2), // src
@@ -11749,8 +11764,10 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap: {
SDValue Rsrc = bufferRsrcPtrToVector(Op->getOperand(4), DAG);
- auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(6), DAG);
- auto SOffset = selectSOffset(Op.getOperand(7), DAG, Subtarget);
+ auto [VOffset, SOffset, Offset] =
+ splitBufferOffsets(Op.getOperand(6), Op.getOperand(7), DAG,
+ /*HasLinearOOB=*/false);
+ SOffset = selectSOffset(SOffset, DAG, Subtarget);
SDValue Ops[] = {
Op.getOperand(0), // Chain
Op.getOperand(2), // src
@@ -12258,8 +12275,10 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
if (IsD16)
VData = handleD16VData(VData, DAG);
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(3), DAG);
- auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(5), DAG);
- auto SOffset = selectSOffset(Op.getOperand(6), DAG, Subtarget);
+ auto [VOffset, SOffset, Offset] =
+ splitBufferOffsets(Op.getOperand(5), Op.getOperand(6), DAG,
+ /*HasLinearOOB=*/false);
+ SOffset = selectSOffset(SOffset, DAG, Subtarget);
SDValue Ops[] = {
Chain,
VData, // vdata
@@ -12286,8 +12305,10 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
if (IsD16)
VData = handleD16VData(VData, DAG);
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(3), DAG);
- auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(4), DAG);
- auto SOffset = selectSOffset(Op.getOperand(5), DAG, Subtarget);
+ auto [VOffset, SOffset, Offset] =
+ splitBufferOffsets(Op.getOperand(4), Op.getOperand(5), DAG,
+ /*HasLinearOOB=*/true);
+ SOffset = selectSOffset(SOffset, DAG, Subtarget);
SDValue Ops[] = {
Chain,
VData, // vdata
@@ -12331,8 +12352,10 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
}
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(3), DAG);
- auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(4), DAG);
- auto SOffset = selectSOffset(Op.getOperand(5), DAG, Subtarget);
+ auto [VOffset, SOffset, Offset] =
+ splitBufferOffsets(Op.getOperand(4), Op.getOperand(5), DAG,
+ /*HasLinearOOB=*/true);
+ SOffset = selectSOffset(SOffset, DAG, Subtarget);
SDValue Ops[] = {
Chain,
VData,
@@ -12382,8 +12405,10 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
}
auto Rsrc = bufferRsrcPtrToVector(Op.getOperand(3), DAG);
- auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(5), DAG);
- auto SOffset = selectSOffset(Op.getOperand(6), DAG, Subtarget);
+ auto [VOffset, SOffset, Offset] =
+ splitBufferOffsets(Op.getOperand(5), Op.getOperand(6), DAG,
+ /*HasLinearOOB=*/false);
+ SOffset = selectSOffset(SOffset, DAG, Subtarget);
SDValue Ops[] = {
Chain,
VData,
@@ -12771,6 +12796,25 @@ static bool isNoUnsignedWrap(SDValue Addr) {
Addr->getOpcode() == ISD::OR;
}
+static bool canMoveBufferOffsetToSOffset(const GCNSubtarget *Subtarget,
+ bool HasLinearOOB, bool IsNUW,
+ uint32_t Offset) {
+ // There is a hardware bug in SI and CI which prevents address clamping in
+ // MUBUF instructions from working correctly with SOffsets. The immediate
+ // offset is unaffected.
+ return HasLinearOOB && IsNUW && static_cast<int32_t>(Offset) > 0 &&
+ Subtarget->getGeneration() > AMDGPUSubtarget::SEA_ISLANDS;
+}
+
+static std::optional<uint32_t> getCombinedBufferSOffset(SDValue SOffset,
+ uint32_t Offset) {
+ auto *C = dyn_cast<ConstantSDNode>(SOffset);
+ if (!C)
+ return std::nullopt;
+ return checkedAddUnsigned<uint32_t>(static_cast<uint32_t>(C->getZExtValue()),
+ Offset);
+}
+
bool SITargetLowering::shouldPreservePtrArith(const Function &F,
EVT PtrVT) const {
return PtrVT == MVT::i64;
@@ -12785,23 +12829,30 @@ bool SITargetLowering::canTransformPtrArithOutOfBounds(const Function &F,
// offset (the offset that is included in bounds checking and swizzling, to be
// split between the instruction's voffset and immoffset fields) and soffset
// (the offset that is excluded from bounds checking and swizzling, to go in
-// the instruction's soffset field). This function takes the first kind of
-// offset and figures out how to split it between voffset and immoffset.
-std::pair<SDValue, SDValue>
-SITargetLowering::splitBufferOffsets(SDValue Offset, SelectionDAG &DAG) const {
+// the instruction's soffset field, except that it does affect num_records and
+// so can (if there's no unsigned overflow) be used for bounds checking in raw.*
+// intrinsics). This function takes both offsets and figures out how to split
+// them between voffset, soffset, and immoffset.
+std::tuple<SDValue, SDValue, SDValue>
+SITargetLowering::splitBufferOffsets(SDValue Offset, SDValue SOffset,
+ SelectionDAG &DAG,
+ bool HasLinearOOB) const {
SDLoc DL(Offset);
const unsigned MaxImm = SIInstrInfo::getMaxMUBUFImmOffset(*Subtarget);
SDValue N0 = Offset;
ConstantSDNode *C1 = nullptr;
+ bool IsNUW = false;
- if ((C1 = dyn_cast<ConstantSDNode>(N0)))
+ if ((C1 = dyn_cast<ConstantSDNode>(N0))) {
N0 = SDValue();
- else if (DAG.isBaseWithConstantOffset(N0)) {
+ IsNUW = true;
+ } else if (DAG.isBaseWithConstantOffset(N0)) {
// On GFX1250+, voffset and immoffset are zero-extended from 32 bits before
// being added, so we can only safely match a 32-bit addition with no
// unsigned overflow.
bool CheckNUW = Subtarget->hasGFX1250Insts();
- if (!CheckNUW || isNoUnsignedWrap(N0)) {
+ IsNUW = isNoUnsignedWrap(N0);
+ if (!CheckNUW || IsNUW) {
C1 = cast<ConstantSDNode>(N0.getOperand(1));
N0 = N0.getOperand(0);
}
@@ -12809,36 +12860,55 @@ SITargetLowering::splitBufferOffsets(SDValue Offset, SelectionDAG &DAG) const {
if (C1) {
unsigned ImmOffset = C1->getZExtValue();
- // If the immediate value is too big for the immoffset field, put only bits
- // that would normally fit in the immoffset field. The remaining value that
- // is copied/added for the voffset field is a large power of 2, and it
- // stands more chance of being CSEd with the copy/add for another similar
- // load/store.
- // However, do not do that rounding down if that is a negative
- // number, as it appears to be illegal to have a negative offset in the
- // vgpr, even if adding the immediate offset makes it positive.
unsigned Overflow = ImmOffset & ~MaxImm;
ImmOffset -= Overflow;
+
+ // If the immediate value is too big for the immoffset field, put only
+ // bits that would normally fit in the immoffset field. The remaining
+ // value copied/added for the voffset field is a large power of 2, and it
+ // stands more chance of being CSEd with the copy/add for another similar
+ // load/store. For eligible raw.(t)buffer.* operations, the branch below
+ // may instead move this overflow to soffset.
+ //
+ // Do not do the rounding-down split for negative numbers, as the addition
+ // of immoffset and voffset is checked for unsigned overflow (which causes
+ // the load/store to be marked OOB) and the soffset is (on gfx8 through
+ // gfx12; see the ISA MUBUF/MTBUF addressing description) subtracted from
+ // num_records and not added to the offset.
if ((int32_t)Overflow < 0) {
Overflow += ImmOffset;
ImmOffset = 0;
}
- C1 = cast<ConstantSDNode>(DAG.getTargetConstant(ImmOffset, DL, MVT::i32));
- if (Overflow) {
- auto OverflowVal = DAG.getConstant(Overflow, DL, MVT::i32);
- if (!N0)
- N0 = OverflowVal;
- else {
- SDValue Ops[] = {N0, OverflowVal};
- N0 = DAG.getNode(ISD::ADD, DL, MVT::i32, Ops);
+
+ std::optional<uint32_t> NewSOffset;
+ if (Overflow &&
+ canMoveBufferOffsetToSOffset(Subtarget, HasLinearOOB, IsNUW, Overflow))
+ NewSOffset = getCombinedBufferSOffset(SOffset, Overflow);
+
+ if (NewSOffset) {
+ // For raw.(t)buffer.* operations with a bare constant or no-wrap add,
+ // the overflow can be added to an existing soffset if the addition would
+ // not cause unsigned overflow. We conservatively restrict ourselves to
+ // constant soffsets here.
+ SOffset = DAG.getConstant(*NewSOffset, DL, MVT::i32);
+ } else {
+ if (Overflow) {
+ auto OverflowVal = DAG.getConstant(Overflow, DL, MVT::i32);
+ if (!N0)
+ N0 = OverflowVal;
+ else {
+ SDValue Ops[] = {N0, OverflowVal};
+ N0 = DAG.getNode(ISD::ADD, DL, MVT::i32, Ops);
+ }
}
}
+ C1 = cast<ConstantSDNode>(DAG.getTargetConstant(ImmOffset, DL, MVT::i32));
}
if (!N0)
N0 = DAG.getConstant(0, DL, MVT::i32);
if (!C1)
C1 = cast<ConstantSDNode>(DAG.getTargetConstant(0, DL, MVT::i32));
- return {N0, SDValue(C1, 0)};
+ return {N0, SOffset, SDValue(C1, 0)};
}
// Analyze a combined offset from an amdgcn_s_buffer_load intrinsic and store
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index c98426cdac0b1..073d577eaad63 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -18,6 +18,7 @@
#include "AMDGPUISelLowering.h"
#include "SIDefines.h"
#include "llvm/CodeGen/MachineFunction.h"
+#include <tuple>
namespace llvm {
@@ -104,14 +105,14 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
- // The raw.tbuffer and struct.tbuffer intrinsics have two offset args: offset
- // (the offset that is included in bounds checking and swizzling, to be split
- // between the instruction's voffset and immoffset fields) and soffset (the
- // offset that is excluded from bounds checking and swizzling, to go in the
- // instruction's soffset field). This function takes the first kind of
- // offset and figures out how to split it between voffset and immoffset.
- std::pair<SDValue, SDValue> splitBufferOffsets(SDValue Offset,
- SelectionDAG &DAG) const;
+ // The raw.(t)buffer and struct.(t)buffer intrinsics have two offset args:
+ // offset (included in bounds checking and swizzling) and soffset (excluded
+ // from bounds checking and swizzling, but affecting raw.* num_records
+ // checking). This splits them into the instruction's voffset, soffset, and
+ // immoffset fields.
+ std::tuple<SDValue, SDValue, SDValue>
+ splitBufferOffsets(SDValue Offset, SDValue SOffset, SelectionDAG &DAG,
+ bool HasLinearOOB) const;
SDValue widenLoad(LoadSDNode *Ld, DAGCombinerInfo &DCI) const;
SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll
index 8ec540029e2a0..e4caec9016d36 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll
@@ -1221,12 +1221,10 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soff
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
; CHECK-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; CHECK-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 67104768
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 4092, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+ ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 67104768
+ ; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET [[REG_SEQUENCE]], [[S_MOV_B32_]], 4092, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFSET]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset0
@@ -1239,9 +1237,8 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soff
; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 58720256
- ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 8388604, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET [[REG_SEQUENCE]], [[S_MOV_B32_]], 8388604, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET]]
; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 67108860, i32 0, i32 0)
ret float %val
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.offset-split.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.offset-split.ll
index c61901f42e7a0..a84f56dd91e94 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.offset-split.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.offset-split.ll
@@ -60,27 +60,26 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soff
define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__or67108860_soffset16(ptr addrspace(8) inreg %rsrc, i32 %voffset.base) {
; GFX8-LABEL: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__or67108860_soffset16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_and_b32_e32 v0, 0xfc000000, v0
; GFX8-NEXT: s_mov_b32 s0, s2
; GFX8-NEXT: s_mov_b32 s1, s3
; GFX8-NEXT: s_mov_b32 s2, s4
; GFX8-NEXT: s_mov_b32 s3, s5
-; GFX8-NEXT: v_or_b32_e32 v0, 0x3fffffc, v0
-; GFX8-NEXT: buffer_load_dword v0, v0, s[0:3], 16 offen
+; GFX8-NEXT: v_and_b32_e32 v0, 0xfc000000, v0
+; GFX8-NEXT: s_mov_b32 s4, 0x3fff010
+; GFX8-NEXT: buffer_load_dword v0, v0, s[0:3], s4 offen offset:4092
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
; GFX1250-LABEL: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__or67108860_soffset16:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0x3fffffc
+; GFX1250-NEXT: v_and_b32_e32 v0, 0xfc000000, v0
; GFX1250-NEXT: s_mov_b32 s0, s2
; GFX1250-NEXT: s_mov_b32 s1, s3
; GFX1250-NEXT: s_mov_b32 s2, s4
; GFX1250-NEXT: s_mov_b32 s3, s5
-; GFX1250-NEXT: v_and_or_b32 v0, 0xfc000000, v0, v1
-; GFX1250-NEXT: s_mov_b32 s4, 16
-; GFX1250-NEXT: buffer_load_b32 v0, v0, s[0:3], s4 offen
+; GFX1250-NEXT: s_mov_b32 s4, 0x3800010
+; GFX1250-NEXT: buffer_load_b32 v0, v0, s[0:3], s4 offen offset:8388604
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
%voffset.masked = and i32 %voffset.base, -67108864
diff --git a/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll
index 952b7f33d3dfa..97ae0a66aedc6 100644
--- a/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll
@@ -2652,8 +2652,7 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset0(ptr a
; GFX908-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
; GFX908-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
; GFX908-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128 = REG_SEQUENCE killed [[COPY7]], %subreg.sub0, killed [[COPY6]], %subreg.sub1, killed [[COPY5]], %subreg.sub2, killed [[COPY4]], %subreg.sub3
- ; GFX908-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; GFX908-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 67104768, implicit $exec
+ ; GFX908-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 67104768
; GFX908-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; GFX908-NEXT: {{ $}}
; GFX908-NEXT: bb.1:
@@ -2674,13 +2673,13 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset0(ptr a
; GFX908-NEXT: bb.2:
; GFX908-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX908-NEXT: {{ $}}
- ; GFX908-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE5]], [[S_MOV_B32_]], 4092, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX908-NEXT: [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET killed [[REG_SEQUENCE5]], [[S_MOV_B32_]], 4092, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
; GFX908-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; GFX908-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX908-NEXT: {{ $}}
; GFX908-NEXT: bb.3:
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
- ; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+ ; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFSET]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
;
; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset0
@@ -2699,14 +2698,14 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset0(ptr a
; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY7]], %subreg.sub0, killed [[COPY6]], %subreg.sub1, killed [[COPY5]], %subreg.sub2, killed [[COPY4]], %subreg.sub3
- ; GFX1250-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 58720256, implicit $exec
- ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 58720256
; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.1:
; GFX1250-NEXT: successors: %bb.2(0x80000000)
; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %30, %bb.2
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_2]], %bb.0, %30, %bb.2
; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
@@ -2720,13 +2719,13 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset0(ptr a
; GFX1250-NEXT: bb.2:
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE5]], $sgpr_null, 8388604, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET killed [[REG_SEQUENCE5]], [[S_MOV_B32_]], 8388604, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
- ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
- ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_1]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET]]
; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 67108860, i32 0, i32 0)
ret float %val
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll
index 68613502b43be..97c4e74eaf734 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-;RUN: llc < %s -mtriple=amdgcn -mcpu=verde | FileCheck %s --check-prefixes=PREGFX10
-;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck %s --check-prefixes=PREGFX10
+;RUN: llc < %s -mtriple=amdgcn -mcpu=verde | FileCheck %s --check-prefixes=PREGFX10,SI
+;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck %s --check-prefixes=PREGFX10,VI
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1010 | FileCheck %s --check-prefixes=GFX10
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX11
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1200 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
@@ -247,24 +247,31 @@ main_body:
}
define amdgpu_ps <4 x float> @buffer_load_voffset_large_13bit(<4 x i32> inreg) {
-; PREGFX10-LABEL: buffer_load_voffset_large_13bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0x1000
-; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: buffer_load_voffset_large_13bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0x1000
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: buffer_load_voffset_large_13bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_movk_i32 s4, 0x1000
+; VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: buffer_load_voffset_large_13bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0x1000
-; GFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX10-NEXT: s_movk_i32 s4, 0x1000
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: buffer_load_voffset_large_13bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0x1000
-; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX11-NEXT: s_movk_i32 s4, 0x1000
+; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
@@ -279,24 +286,31 @@ main_body:
}
define amdgpu_ps <4 x float> @buffer_load_voffset_large_16bit(<4 x i32> inreg) {
-; PREGFX10-LABEL: buffer_load_voffset_large_16bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0xf000
-; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: buffer_load_voffset_large_16bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0xf000
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: buffer_load_voffset_large_16bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0xf000
+; VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: buffer_load_voffset_large_16bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0xf000
-; GFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0xf000
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: buffer_load_voffset_large_16bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xf000
-; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0xf000
+; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
@@ -311,24 +325,31 @@ main_body:
}
define amdgpu_ps <4 x float> @buffer_load_voffset_large_23bit(<4 x i32> inreg) {
-; PREGFX10-LABEL: buffer_load_voffset_large_23bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0x7ff000
-; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: buffer_load_voffset_large_23bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0x7ff000
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: buffer_load_voffset_large_23bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0x7ff000
+; VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: buffer_load_voffset_large_23bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0x7ff000
-; GFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0x7ff000
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: buffer_load_voffset_large_23bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0x7ff000
-; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0x7ff000
+; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
@@ -343,31 +364,38 @@ main_body:
}
define amdgpu_ps <4 x float> @buffer_load_voffset_large_24bit(<4 x i32> inreg) {
-; PREGFX10-LABEL: buffer_load_voffset_large_24bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0xfff000
-; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: buffer_load_voffset_large_24bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0xfff000
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: buffer_load_voffset_large_24bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0xfff000
+; VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: buffer_load_voffset_large_24bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0xfff000
-; GFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0xfff000
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: buffer_load_voffset_large_24bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xfff000
-; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0xfff000
+; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: buffer_load_voffset_large_24bit:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_mov_b32_e32 v0, 0x800000
-; GFX12-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen offset:8388604
+; GFX12-NEXT: s_mov_b32 s4, 0x800000
+; GFX12-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:8388604
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
@@ -435,6 +463,20 @@ main_body:
}
define amdgpu_ps <4 x float> @buffer_load_negative_offset(<4 x i32> inreg, i32 %ofs) {
+; SI-LABEL: buffer_load_negative_offset:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_add_i32_e32 v0, vcc, -16, v0
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: buffer_load_negative_offset:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: v_add_u32_e32 v0, vcc, -16, v0
+; VI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: buffer_load_negative_offset:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_add_nc_u32_e32 v0, -16, v0
@@ -462,6 +504,30 @@ main_body:
}
define amdgpu_ps float @buffer_load_mmo(<4 x i32> inreg %rsrc, ptr addrspace(3) %lds) {
+; SI-LABEL: buffer_load_mmo:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0
+; SI-NEXT: v_mov_b32_e32 v2, 0
+; SI-NEXT: s_mov_b32 m0, -1
+; SI-NEXT: ds_write_b32 v0, v2
+; SI-NEXT: v_add_i32_e32 v0, vcc, 16, v0
+; SI-NEXT: ds_write_b32 v0, v2
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_mov_b32_e32 v0, v1
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: buffer_load_mmo:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0
+; VI-NEXT: v_mov_b32_e32 v2, 0
+; VI-NEXT: s_mov_b32 m0, -1
+; VI-NEXT: ds_write2_b32 v0, v2, v2 offset1:4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v0, v1
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: buffer_load_mmo:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: buffer_load_dword v1, off, s[0:3], 0
@@ -596,39 +662,17 @@ define amdgpu_ps void @buffer_load_x1_offen_merged_or(<4 x i32> inreg %rsrc, i32
; GFX11-NEXT: exp mrt0, v4, v5, v0, v0 done
; GFX11-NEXT: s_endpgm
;
-; GFX12-SDAG-LABEL: buffer_load_x1_offen_merged_or:
-; GFX12-SDAG: ; %bb.0: ; %main_body
-; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v4, 6, v0
-; GFX12-SDAG-NEXT: s_clause 0x1
-; GFX12-SDAG-NEXT: buffer_load_b128 v[0:3], v4, s[0:3], null offen offset:4
-; GFX12-SDAG-NEXT: buffer_load_b64 v[4:5], v4, s[0:3], null offen offset:28
-; GFX12-SDAG-NEXT: s_wait_loadcnt 0x1
-; GFX12-SDAG-NEXT: export mrt0, v0, v1, v2, v3 done
-; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-NEXT: export mrt0, v4, v5, v0, v0 done
-; GFX12-SDAG-NEXT: s_endpgm
-;
-; GFX12-GISEL-LABEL: buffer_load_x1_offen_merged_or:
-; GFX12-GISEL: ; %bb.0: ; %main_body
-; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, 6, v0
-; GFX12-GISEL-NEXT: v_or_b32_e32 v1, 4, v0
-; GFX12-GISEL-NEXT: v_or_b32_e32 v2, 8, v0
-; GFX12-GISEL-NEXT: v_or_b32_e32 v3, 12, v0
-; GFX12-GISEL-NEXT: v_or_b32_e32 v4, 16, v0
-; GFX12-GISEL-NEXT: v_or_b32_e32 v5, 28, v0
-; GFX12-GISEL-NEXT: v_or_b32_e32 v0, 32, v0
-; GFX12-GISEL-NEXT: s_clause 0x5
-; GFX12-GISEL-NEXT: buffer_load_b32 v1, v1, s[0:3], null offen
-; GFX12-GISEL-NEXT: buffer_load_b32 v2, v2, s[0:3], null offen
-; GFX12-GISEL-NEXT: buffer_load_b32 v3, v3, s[0:3], null offen
-; GFX12-GISEL-NEXT: buffer_load_b32 v4, v4, s[0:3], null offen
-; GFX12-GISEL-NEXT: buffer_load_b32 v5, v5, s[0:3], null offen
-; GFX12-GISEL-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen
-; GFX12-GISEL-NEXT: s_wait_loadcnt 0x2
-; GFX12-GISEL-NEXT: export mrt0, v1, v2, v3, v4 done
-; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-NEXT: export mrt0, v5, v0, v0, v0 done
-; GFX12-GISEL-NEXT: s_endpgm
+; GFX12-LABEL: buffer_load_x1_offen_merged_or:
+; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: v_lshlrev_b32_e32 v4, 6, v0
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: buffer_load_b128 v[0:3], v4, s[0:3], null offen offset:4
+; GFX12-NEXT: buffer_load_b64 v[4:5], v4, s[0:3], null offen offset:28
+; GFX12-NEXT: s_wait_loadcnt 0x1
+; GFX12-NEXT: export mrt0, v0, v1, v2, v3 done
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: export mrt0, v4, v5, v0, v0 done
+; GFX12-NEXT: s_endpgm
main_body:
%a = shl i32 %inp, 6
%a1 = or i32 %a, 4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.ll
index 12997dfa5fabe..94e042104dbbf 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.ll
@@ -1,11 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-;RUN: llc < %s -mtriple=amdgcn -mcpu=verde | FileCheck %s --check-prefixes=PREGFX10
-;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck %s --check-prefixes=PREGFX10
+;RUN: llc < %s -mtriple=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck %s --check-prefixes=PREGFX10,SI
+;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s --check-prefixes=PREGFX10,VI
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1010 | FileCheck %s --check-prefixes=GFX10
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX11
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1250 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX1250
-;RUN: llc < %s -mtriple=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck %s --check-prefixes=SI
-;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s --check-prefixes=VI
define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load(ptr addrspace(8) inreg) {
; PREGFX10-LABEL: buffer_load:
@@ -307,24 +305,31 @@ main_body:
}
define amdgpu_ps <4 x float> @buffer_load_voffset_large_13bit(ptr addrspace(8) inreg) {
-; PREGFX10-LABEL: buffer_load_voffset_large_13bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0x1000
-; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: buffer_load_voffset_large_13bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0x1000
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: buffer_load_voffset_large_13bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_movk_i32 s4, 0x1000
+; VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: buffer_load_voffset_large_13bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0x1000
-; GFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX10-NEXT: s_movk_i32 s4, 0x1000
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: buffer_load_voffset_large_13bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0x1000
-; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX11-NEXT: s_movk_i32 s4, 0x1000
+; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
@@ -340,24 +345,31 @@ main_body:
}
define amdgpu_ps <4 x float> @buffer_load_voffset_large_16bit(ptr addrspace(8) inreg) {
-; PREGFX10-LABEL: buffer_load_voffset_large_16bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0xf000
-; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: buffer_load_voffset_large_16bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0xf000
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: buffer_load_voffset_large_16bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0xf000
+; VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: buffer_load_voffset_large_16bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0xf000
-; GFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0xf000
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: buffer_load_voffset_large_16bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xf000
-; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0xf000
+; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
@@ -373,24 +385,31 @@ main_body:
}
define amdgpu_ps <4 x float> @buffer_load_voffset_large_23bit(ptr addrspace(8) inreg) {
-; PREGFX10-LABEL: buffer_load_voffset_large_23bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0x7ff000
-; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: buffer_load_voffset_large_23bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0x7ff000
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: buffer_load_voffset_large_23bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0x7ff000
+; VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: buffer_load_voffset_large_23bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0x7ff000
-; GFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0x7ff000
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: buffer_load_voffset_large_23bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0x7ff000
-; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0x7ff000
+; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
@@ -406,32 +425,39 @@ main_body:
}
define amdgpu_ps <4 x float> @buffer_load_voffset_large_24bit(ptr addrspace(8) inreg) {
-; PREGFX10-LABEL: buffer_load_voffset_large_24bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0xfff000
-; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: buffer_load_voffset_large_24bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0xfff000
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: buffer_load_voffset_large_24bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0xfff000
+; VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: buffer_load_voffset_large_24bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0xfff000
-; GFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0xfff000
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: buffer_load_voffset_large_24bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xfff000
-; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0xfff000
+; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX1250-LABEL: buffer_load_voffset_large_24bit:
; GFX1250: ; %bb.0: ; %main_body
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_mov_b32_e32 v0, 0x800000
-; GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen offset:8388604
+; GFX1250-NEXT: s_mov_b32 s4, 0x800000
+; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:8388604
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
main_body:
@@ -439,7 +465,6 @@ main_body:
ret <4 x float> %data
}
-
define amdgpu_ps float @buffer_load_x1(ptr addrspace(8) inreg %rsrc, i32 %ofs) {
; PREGFX10-LABEL: buffer_load_x1:
; PREGFX10: ; %bb.0: ; %main_body
@@ -501,6 +526,20 @@ main_body:
}
define amdgpu_ps <4 x float> @buffer_load_negative_offset(ptr addrspace(8) inreg, i32 %ofs) {
+; SI-LABEL: buffer_load_negative_offset:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_add_i32_e32 v0, vcc, -16, v0
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: buffer_load_negative_offset:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: v_add_u32_e32 v0, vcc, -16, v0
+; VI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: buffer_load_negative_offset:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_add_nc_u32_e32 v0, -16, v0
@@ -529,6 +568,30 @@ main_body:
}
define amdgpu_ps float @buffer_load_mmo(ptr addrspace(8) inreg %rsrc, ptr addrspace(3) %lds) {
+; SI-LABEL: buffer_load_mmo:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0
+; SI-NEXT: v_mov_b32_e32 v2, 0
+; SI-NEXT: s_mov_b32 m0, -1
+; SI-NEXT: ds_write_b32 v0, v2
+; SI-NEXT: v_add_i32_e32 v0, vcc, 16, v0
+; SI-NEXT: ds_write_b32 v0, v2
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_mov_b32_e32 v0, v1
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: buffer_load_mmo:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0
+; VI-NEXT: v_mov_b32_e32 v2, 0
+; VI-NEXT: s_mov_b32 m0, -1
+; VI-NEXT: ds_write2_b32 v0, v2, v2 offset1:4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v0, v1
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: buffer_load_mmo:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: buffer_load_dword v1, off, s[0:3], 0
@@ -875,6 +938,24 @@ define amdgpu_ps void @raw_ptr_buffer_load_v4f16(ptr addrspace(8) inreg %rsrc, p
; }
define amdgpu_ps void @raw_ptr_buffer_load_v8f16(ptr addrspace(8) inreg %rsrc, ptr addrspace(3) %ptr) {
+; SI-LABEL: raw_ptr_buffer_load_v8f16:
+; SI: ; %bb.0:
+; SI-NEXT: buffer_load_dwordx4 v[1:4], off, s[0:3], 0
+; SI-NEXT: v_add_i32_e32 v5, vcc, 8, v0
+; SI-NEXT: s_mov_b32 m0, -1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ds_write_b64 v5, v[3:4]
+; SI-NEXT: ds_write_b64 v0, v[1:2]
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: raw_ptr_buffer_load_v8f16:
+; VI: ; %bb.0:
+; VI-NEXT: buffer_load_dwordx4 v[1:4], off, s[0:3], 0
+; VI-NEXT: s_mov_b32 m0, -1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ds_write_b128 v0, v[1:4]
+; VI-NEXT: s_endpgm
+;
; GFX10-LABEL: raw_ptr_buffer_load_v8f16:
; GFX10: ; %bb.0:
; GFX10-NEXT: buffer_load_dwordx4 v[1:4], off, s[0:3], 0
@@ -980,6 +1061,24 @@ define amdgpu_ps void @raw_ptr_buffer_load_v4i16(ptr addrspace(8) inreg %rsrc, p
; }
define amdgpu_ps void @raw_ptr_buffer_load_v8i16(ptr addrspace(8) inreg %rsrc, ptr addrspace(3) %ptr) {
+; SI-LABEL: raw_ptr_buffer_load_v8i16:
+; SI: ; %bb.0:
+; SI-NEXT: buffer_load_dwordx4 v[1:4], off, s[0:3], 0
+; SI-NEXT: v_add_i32_e32 v5, vcc, 8, v0
+; SI-NEXT: s_mov_b32 m0, -1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ds_write_b64 v5, v[3:4]
+; SI-NEXT: ds_write_b64 v0, v[1:2]
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: raw_ptr_buffer_load_v8i16:
+; VI: ; %bb.0:
+; VI-NEXT: buffer_load_dwordx4 v[1:4], off, s[0:3], 0
+; VI-NEXT: s_mov_b32 m0, -1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ds_write_b128 v0, v[1:4]
+; VI-NEXT: s_endpgm
+;
; GFX10-LABEL: raw_ptr_buffer_load_v8i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: buffer_load_dwordx4 v[1:4], off, s[0:3], 0
@@ -1497,6 +1596,20 @@ define <2 x ptr addrspace(2)> @buffer_load_v2p2__voffset_add(ptr addrspace(8) in
}
define <3 x ptr addrspace(2)> @buffer_load_v3p2__voffset_add(ptr addrspace(8) inreg %rsrc, i32 %voffset) {
+; SI-LABEL: buffer_load_v3p2__voffset_add:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[16:19], 0 offen offset:60
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-LABEL: buffer_load_v3p2__voffset_add:
+; VI: ; %bb.0:
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: buffer_load_dwordx3 v[0:2], v0, s[16:19], 0 offen offset:60
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: buffer_load_v3p2__voffset_add:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1630,6 +1743,20 @@ define <2 x ptr addrspace(3)> @buffer_load_v2p3__voffset_add(ptr addrspace(8) in
}
define <3 x ptr addrspace(3)> @buffer_load_v3p3__voffset_add(ptr addrspace(8) inreg %rsrc, i32 %voffset) {
+; SI-LABEL: buffer_load_v3p3__voffset_add:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[16:19], 0 offen offset:60
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-LABEL: buffer_load_v3p3__voffset_add:
+; VI: ; %bb.0:
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: buffer_load_dwordx3 v[0:2], v0, s[16:19], 0 offen offset:60
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: buffer_load_v3p3__voffset_add:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1763,6 +1890,20 @@ define <2 x ptr addrspace(5)> @buffer_load_v2p5__voffset_add(ptr addrspace(8) in
}
define <3 x ptr addrspace(5)> @buffer_load_v3p5__voffset_add(ptr addrspace(8) inreg %rsrc, i32 %voffset) {
+; SI-LABEL: buffer_load_v3p5__voffset_add:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[16:19], 0 offen offset:60
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-LABEL: buffer_load_v3p5__voffset_add:
+; VI: ; %bb.0:
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: buffer_load_dwordx3 v[0:2], v0, s[16:19], 0 offen offset:60
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: buffer_load_v3p5__voffset_add:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1896,6 +2037,20 @@ define <2 x ptr addrspace(6)> @buffer_load_v2p6__voffset_add(ptr addrspace(8) in
}
define <3 x ptr addrspace(6)> @buffer_load_v3p6__voffset_add(ptr addrspace(8) inreg %rsrc, i32 %voffset) {
+; SI-LABEL: buffer_load_v3p6__voffset_add:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[16:19], 0 offen offset:60
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-LABEL: buffer_load_v3p6__voffset_add:
+; VI: ; %bb.0:
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: buffer_load_dwordx3 v[0:2], v0, s[16:19], 0 offen offset:60
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: buffer_load_v3p6__voffset_add:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1960,12 +2115,39 @@ define <4 x ptr addrspace(6)> @buffer_load_v4p6__voffset_add(ptr addrspace(8) in
define amdgpu_ps <4 x float> @raw_ptr_large_zero_soffset(ptr addrspace(8) inreg %rsrc) {
; SI-LABEL: raw_ptr_large_zero_soffset:
-; SI: v_mov_b32_e32 [[VOFF:v[0-9]+]], 0x1000
-; SI-NEXT: buffer_load_dwordx4 v[0:3], [[VOFF]], s[0:3], 0 offen offset:4092
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0x1000
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
;
; VI-LABEL: raw_ptr_large_zero_soffset:
-; VI: v_mov_b32_e32 [[VOFF:v[0-9]+]], 0x1000
-; VI-NEXT: buffer_load_dwordx4 v[0:3], [[VOFF]], s[0:3], 0 offen offset:4092
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_movk_i32 s4, 0x1000
+; VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
+;
+; GFX10-LABEL: raw_ptr_large_zero_soffset:
+; GFX10: ; %bb.0: ; %main_body
+; GFX10-NEXT: s_movk_i32 s4, 0x1000
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: raw_ptr_large_zero_soffset:
+; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: s_movk_i32 s4, 0x1000
+; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:4092
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: raw_ptr_large_zero_soffset:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], null offset:8188
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
main_body:
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 8188, i32 0, i32 0)
ret <4 x float> %data
@@ -1973,25 +2155,74 @@ main_body:
define amdgpu_ps <4 x float> @raw_ptr_large_safe_soffset(ptr addrspace(8) inreg %rsrc) {
; SI-LABEL: raw_ptr_large_safe_soffset:
-; SI: v_mov_b32_e32 [[VOFF:v[0-9]+]], 0x1000
-; SI-NEXT: buffer_load_dwordx4 v[0:3], [[VOFF]], s[0:3], 16 offen offset:4092
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0x1000
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 16 offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
;
; VI-LABEL: raw_ptr_large_safe_soffset:
-; VI: v_mov_b32_e32 [[VOFF:v[0-9]+]], 0x1000
-; VI-NEXT: buffer_load_dwordx4 v[0:3], [[VOFF]], s[0:3], 16 offen offset:4092
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_movk_i32 s4, 0x1010
+; VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
+;
+; GFX10-LABEL: raw_ptr_large_safe_soffset:
+; GFX10: ; %bb.0: ; %main_body
+; GFX10-NEXT: s_movk_i32 s4, 0x1010
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: raw_ptr_large_safe_soffset:
+; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: s_movk_i32 s4, 0x1010
+; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:4092
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: raw_ptr_large_safe_soffset:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b32 s4, 16
+; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:8188
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
main_body:
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 8188, i32 16, i32 0)
ret <4 x float> %data
}
define amdgpu_ps <4 x float> @raw_ptr_large_overflowing_soffset(ptr addrspace(8) inreg %rsrc) {
-; SI-LABEL: raw_ptr_large_overflowing_soffset:
-; SI: v_mov_b32_e32 [[VOFF:v[0-9]+]], 0x1000
-; SI-NEXT: buffer_load_dwordx4 v[0:3], [[VOFF]], s[0:3], -16 offen offset:4092
+; PREGFX10-LABEL: raw_ptr_large_overflowing_soffset:
+; PREGFX10: ; %bb.0: ; %main_body
+; PREGFX10-NEXT: v_mov_b32_e32 v0, 0x1000
+; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], -16 offen offset:4092
+; PREGFX10-NEXT: s_waitcnt vmcnt(0)
+; PREGFX10-NEXT: ; return to shader part epilog
;
-; VI-LABEL: raw_ptr_large_overflowing_soffset:
-; VI: v_mov_b32_e32 [[VOFF:v[0-9]+]], 0x1000
-; VI-NEXT: buffer_load_dwordx4 v[0:3], [[VOFF]], s[0:3], -16 offen offset:4092
+; GFX10-LABEL: raw_ptr_large_overflowing_soffset:
+; GFX10: ; %bb.0: ; %main_body
+; GFX10-NEXT: v_mov_b32_e32 v0, 0x1000
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], -16 offen offset:4092
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: raw_ptr_large_overflowing_soffset:
+; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: v_mov_b32_e32 v0, 0x1000
+; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], -16 offen offset:4092
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: raw_ptr_large_overflowing_soffset:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b32 s4, -16
+; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:8188
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
main_body:
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 8188, i32 -16, i32 0)
ret <4 x float> %data
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.ll
index b3115253bf29f..95b3276a8e5aa 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-;RUN: llc < %s -mtriple=amdgcn -mcpu=verde | FileCheck -check-prefix=PREGFX10 %s
-;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck -check-prefix=PREGFX10 %s
+;RUN: llc < %s -mtriple=amdgcn -mcpu=verde | FileCheck -check-prefixes=PREGFX10,SI %s
+;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck -check-prefixes=PREGFX10,VI %s
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1010 | FileCheck -check-prefix=GFX10 %s
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 | FileCheck -check-prefix=GFX11 %s
@@ -96,24 +96,31 @@ main_body:
}
define amdgpu_ps <4 x float> @tbuffer_load_voffset_large_13bit(ptr addrspace(8) inreg) {
-; PREGFX10-LABEL: tbuffer_load_voffset_large_13bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0x1000
-; PREGFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: tbuffer_load_voffset_large_13bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0x1000
+; SI-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: tbuffer_load_voffset_large_13bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_movk_i32 s4, 0x1000
+; VI-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: tbuffer_load_voffset_large_13bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0x1000
-; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_movk_i32 s4, 0x1000
+; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: tbuffer_load_voffset_large_13bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0x1000
-; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_movk_i32 s4, 0x1000
+; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
main_body:
@@ -122,24 +129,31 @@ main_body:
}
define amdgpu_ps <4 x float> @tbuffer_load_voffset_large_16bit(ptr addrspace(8) inreg) {
-; PREGFX10-LABEL: tbuffer_load_voffset_large_16bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0xf000
-; PREGFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: tbuffer_load_voffset_large_16bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0xf000
+; SI-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: tbuffer_load_voffset_large_16bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0xf000
+; VI-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: tbuffer_load_voffset_large_16bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0xf000
-; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0xf000
+; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: tbuffer_load_voffset_large_16bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xf000
-; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0xf000
+; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
main_body:
@@ -148,24 +162,31 @@ main_body:
}
define amdgpu_ps <4 x float> @tbuffer_load_voffset_large_23bit(ptr addrspace(8) inreg) {
-; PREGFX10-LABEL: tbuffer_load_voffset_large_23bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0x7ff000
-; PREGFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: tbuffer_load_voffset_large_23bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0x7ff000
+; SI-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: tbuffer_load_voffset_large_23bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0x7ff000
+; VI-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: tbuffer_load_voffset_large_23bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0x7ff000
-; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0x7ff000
+; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: tbuffer_load_voffset_large_23bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0x7ff000
-; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0x7ff000
+; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
main_body:
@@ -174,24 +195,31 @@ main_body:
}
define amdgpu_ps <4 x float> @tbuffer_load_voffset_large_24bit(ptr addrspace(8) inreg) {
-; PREGFX10-LABEL: tbuffer_load_voffset_large_24bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0xfff000
-; PREGFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: tbuffer_load_voffset_large_24bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0xfff000
+; SI-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: tbuffer_load_voffset_large_24bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0xfff000
+; VI-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: tbuffer_load_voffset_large_24bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0xfff000
-; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0xfff000
+; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: tbuffer_load_voffset_large_24bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xfff000
-; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0xfff000
+; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.store.ll
index f778304c414a9..eda2c0247bcc1 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.store.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-;RUN: llc < %s -mtriple=amdgcn -mcpu=verde | FileCheck -check-prefixes=PREGFX10 %s
-;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck -check-prefixes=PREGFX10 %s
+;RUN: llc < %s -mtriple=amdgcn -mcpu=verde | FileCheck -check-prefixes=PREGFX10,SI %s
+;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck -check-prefixes=PREGFX10,VI %s
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1010 | FileCheck -check-prefixes=GFX10 %s
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 | FileCheck -check-prefixes=GFX11 %s
@@ -166,22 +166,28 @@ main_body:
}
define amdgpu_ps void @buffer_store_voffset_large_13bit(ptr addrspace(8) inreg %rsrc, <4 x float> %data) {
-; PREGFX10-LABEL: buffer_store_voffset_large_13bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v4, 0x1000
-; PREGFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_endpgm
+; SI-LABEL: buffer_store_voffset_large_13bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v4, 0x1000
+; SI-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: buffer_store_voffset_large_13bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_movk_i32 s4, 0x1000
+; VI-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_endpgm
;
; GFX10-LABEL: buffer_store_voffset_large_13bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v4, 0x1000
-; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_movk_i32 s4, 0x1000
+; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: buffer_store_voffset_large_13bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v4, 0x1000
-; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_movk_i32 s4, 0x1000
+; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_endpgm
main_body:
call void @llvm.amdgcn.raw.ptr.tbuffer.store.v4f32(<4 x float> %data, ptr addrspace(8) %rsrc, i32 8188, i32 0, i32 63, i32 0)
@@ -189,22 +195,28 @@ main_body:
}
define amdgpu_ps void @buffer_store_voffset_large_16bit(ptr addrspace(8) inreg %rsrc, <4 x float> %data) {
-; PREGFX10-LABEL: buffer_store_voffset_large_16bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v4, 0xf000
-; PREGFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_endpgm
+; SI-LABEL: buffer_store_voffset_large_16bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v4, 0xf000
+; SI-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: buffer_store_voffset_large_16bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0xf000
+; VI-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_endpgm
;
; GFX10-LABEL: buffer_store_voffset_large_16bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v4, 0xf000
-; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0xf000
+; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: buffer_store_voffset_large_16bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v4, 0xf000
-; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0xf000
+; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_endpgm
main_body:
call void @llvm.amdgcn.raw.ptr.tbuffer.store.v4f32(<4 x float> %data, ptr addrspace(8) %rsrc, i32 65532, i32 0, i32 63, i32 0)
@@ -212,22 +224,28 @@ main_body:
}
define amdgpu_ps void @buffer_store_voffset_large_23bit(ptr addrspace(8) inreg %rsrc, <4 x float> %data) {
-; PREGFX10-LABEL: buffer_store_voffset_large_23bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v4, 0x7ff000
-; PREGFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_endpgm
+; SI-LABEL: buffer_store_voffset_large_23bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v4, 0x7ff000
+; SI-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: buffer_store_voffset_large_23bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0x7ff000
+; VI-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_endpgm
;
; GFX10-LABEL: buffer_store_voffset_large_23bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v4, 0x7ff000
-; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0x7ff000
+; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: buffer_store_voffset_large_23bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v4, 0x7ff000
-; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0x7ff000
+; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_endpgm
main_body:
call void @llvm.amdgcn.raw.ptr.tbuffer.store.v4f32(<4 x float> %data, ptr addrspace(8) %rsrc, i32 8388604, i32 0, i32 63, i32 0)
@@ -235,22 +253,28 @@ main_body:
}
define amdgpu_ps void @buffer_store_voffset_large_24bit(ptr addrspace(8) inreg %rsrc, <4 x float> %data) {
-; PREGFX10-LABEL: buffer_store_voffset_large_24bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v4, 0xfff000
-; PREGFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_endpgm
+; SI-LABEL: buffer_store_voffset_large_24bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v4, 0xfff000
+; SI-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: buffer_store_voffset_large_24bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0xfff000
+; VI-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_endpgm
;
; GFX10-LABEL: buffer_store_voffset_large_24bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v4, 0xfff000
-; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0xfff000
+; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: buffer_store_voffset_large_24bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v4, 0xfff000
-; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0xfff000
+; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_endpgm
main_body:
call void @llvm.amdgcn.raw.ptr.tbuffer.store.v4f32(<4 x float> %data, ptr addrspace(8) %rsrc, i32 16777212, i32 0, i32 63, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.ll
index f01e85a2e4a02..7626e58e9b506 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-;RUN: llc < %s -mtriple=amdgcn -mcpu=verde | FileCheck -check-prefix=PREGFX10 %s
-;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck -check-prefix=PREGFX10 %s
+;RUN: llc < %s -mtriple=amdgcn -mcpu=verde | FileCheck -check-prefixes=PREGFX10,SI %s
+;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck -check-prefixes=PREGFX10,VI %s
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1010 | FileCheck -check-prefix=GFX10 %s
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 | FileCheck -check-prefix=GFX11 %s
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1200 | FileCheck -check-prefix=GFX12 %s
@@ -120,24 +120,31 @@ main_body:
}
define amdgpu_ps <4 x float> @tbuffer_load_voffset_large_13bit(<4 x i32> inreg) {
-; PREGFX10-LABEL: tbuffer_load_voffset_large_13bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0x1000
-; PREGFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: tbuffer_load_voffset_large_13bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0x1000
+; SI-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: tbuffer_load_voffset_large_13bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_movk_i32 s4, 0x1000
+; VI-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: tbuffer_load_voffset_large_13bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0x1000
-; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_movk_i32 s4, 0x1000
+; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: tbuffer_load_voffset_large_13bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0x1000
-; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_movk_i32 s4, 0x1000
+; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
@@ -152,24 +159,31 @@ main_body:
}
define amdgpu_ps <4 x float> @tbuffer_load_voffset_large_16bit(<4 x i32> inreg) {
-; PREGFX10-LABEL: tbuffer_load_voffset_large_16bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0xf000
-; PREGFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: tbuffer_load_voffset_large_16bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0xf000
+; SI-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: tbuffer_load_voffset_large_16bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0xf000
+; VI-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: tbuffer_load_voffset_large_16bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0xf000
-; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0xf000
+; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: tbuffer_load_voffset_large_16bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xf000
-; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0xf000
+; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
@@ -184,24 +198,31 @@ main_body:
}
define amdgpu_ps <4 x float> @tbuffer_load_voffset_large_23bit(<4 x i32> inreg) {
-; PREGFX10-LABEL: tbuffer_load_voffset_large_23bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0x7ff000
-; PREGFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: tbuffer_load_voffset_large_23bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0x7ff000
+; SI-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: tbuffer_load_voffset_large_23bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0x7ff000
+; VI-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: tbuffer_load_voffset_large_23bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0x7ff000
-; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0x7ff000
+; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: tbuffer_load_voffset_large_23bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0x7ff000
-; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0x7ff000
+; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
@@ -216,31 +237,38 @@ main_body:
}
define amdgpu_ps <4 x float> @tbuffer_load_voffset_large_24bit(<4 x i32> inreg) {
-; PREGFX10-LABEL: tbuffer_load_voffset_large_24bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0xfff000
-; PREGFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: tbuffer_load_voffset_large_24bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0xfff000
+; SI-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: tbuffer_load_voffset_large_24bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0xfff000
+; VI-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: tbuffer_load_voffset_large_24bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0xfff000
-; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0xfff000
+; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: tbuffer_load_voffset_large_24bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xfff000
-; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0xfff000
+; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: tbuffer_load_voffset_large_24bit:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_mov_b32_e32 v0, 0x800000
-; GFX12-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], null format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:8388604
+; GFX12-NEXT: s_mov_b32 s4, 0x800000
+; GFX12-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:8388604
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.ll
index 8afa43a6ebe48..888d91e6a5691 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-;RUN: llc < %s -mtriple=amdgcn -mcpu=verde | FileCheck -check-prefix=PREGFX10 %s
-;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck -check-prefix=PREGFX10 %s
+;RUN: llc < %s -mtriple=amdgcn -mcpu=verde | FileCheck -check-prefixes=PREGFX10,SI %s
+;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck -check-prefixes=PREGFX10,VI %s
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1010 | FileCheck -check-prefix=GFX10 %s
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 | FileCheck -check-prefix=GFX11 %s
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1200 | FileCheck -check-prefix=GFX12 %s
@@ -207,22 +207,28 @@ main_body:
}
define amdgpu_ps void @buffer_store_voffset_large_13bit(<4 x i32> inreg %rsrc, <4 x float> %data) {
-; PREGFX10-LABEL: buffer_store_voffset_large_13bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v4, 0x1000
-; PREGFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_endpgm
+; SI-LABEL: buffer_store_voffset_large_13bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v4, 0x1000
+; SI-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: buffer_store_voffset_large_13bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_movk_i32 s4, 0x1000
+; VI-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_endpgm
;
; GFX10-LABEL: buffer_store_voffset_large_13bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v4, 0x1000
-; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_movk_i32 s4, 0x1000
+; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: buffer_store_voffset_large_13bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v4, 0x1000
-; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_movk_i32 s4, 0x1000
+; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: buffer_store_voffset_large_13bit:
@@ -235,22 +241,28 @@ main_body:
}
define amdgpu_ps void @buffer_store_voffset_large_16bit(<4 x i32> inreg %rsrc, <4 x float> %data) {
-; PREGFX10-LABEL: buffer_store_voffset_large_16bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v4, 0xf000
-; PREGFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_endpgm
+; SI-LABEL: buffer_store_voffset_large_16bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v4, 0xf000
+; SI-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: buffer_store_voffset_large_16bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0xf000
+; VI-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_endpgm
;
; GFX10-LABEL: buffer_store_voffset_large_16bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v4, 0xf000
-; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0xf000
+; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: buffer_store_voffset_large_16bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v4, 0xf000
-; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0xf000
+; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: buffer_store_voffset_large_16bit:
@@ -263,22 +275,28 @@ main_body:
}
define amdgpu_ps void @buffer_store_voffset_large_23bit(<4 x i32> inreg %rsrc, <4 x float> %data) {
-; PREGFX10-LABEL: buffer_store_voffset_large_23bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v4, 0x7ff000
-; PREGFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_endpgm
+; SI-LABEL: buffer_store_voffset_large_23bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v4, 0x7ff000
+; SI-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: buffer_store_voffset_large_23bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0x7ff000
+; VI-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_endpgm
;
; GFX10-LABEL: buffer_store_voffset_large_23bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v4, 0x7ff000
-; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0x7ff000
+; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: buffer_store_voffset_large_23bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v4, 0x7ff000
-; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0x7ff000
+; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: buffer_store_voffset_large_23bit:
@@ -291,28 +309,34 @@ main_body:
}
define amdgpu_ps void @buffer_store_voffset_large_24bit(<4 x i32> inreg %rsrc, <4 x float> %data) {
-; PREGFX10-LABEL: buffer_store_voffset_large_24bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v4, 0xfff000
-; PREGFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_endpgm
+; SI-LABEL: buffer_store_voffset_large_24bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v4, 0xfff000
+; SI-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: buffer_store_voffset_large_24bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0xfff000
+; VI-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_endpgm
;
; GFX10-LABEL: buffer_store_voffset_large_24bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v4, 0xfff000
-; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0xfff000
+; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: buffer_store_voffset_large_24bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v4, 0xfff000
-; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0xfff000
+; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: buffer_store_voffset_large_24bit:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_mov_b32_e32 v4, 0x800000
-; GFX12-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], null format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:8388604
+; GFX12-NEXT: s_mov_b32 s4, 0x800000
+; GFX12-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:8388604
; GFX12-NEXT: s_endpgm
main_body:
call void @llvm.amdgcn.raw.tbuffer.store.v4f32(<4 x float> %data, <4 x i32> %rsrc, i32 16777212, i32 0, i32 63, i32 0)
More information about the llvm-branch-commits
mailing list