[flang-commits] [clang] [flang] [lld] [llvm] [AMDGPU] Add proper object linking support (PR #220120)
Shilei Tian via flang-commits
flang-commits at lists.llvm.org
Mon Aug 31 15:55:31 PDT 2026
https://github.com/shiltian created https://github.com/llvm/llvm-project/pull/220120
None
>From 4c9fc1f7797c9497bf35273764dab790c50c94b0 Mon Sep 17 00:00:00 2001
From: Shilei Tian <i at tianshilei.me>
Date: Mon, 31 Aug 2026 13:20:57 -0400
Subject: [PATCH] [AMDGPU] Add proper object linking support
---
clang/include/clang/Basic/BuiltinsAMDGPU.td | 964 ++++----
clang/include/clang/Options/Options.td | 12 +-
clang/lib/Driver/Driver.cpp | 42 +-
clang/lib/Driver/ToolChains/AMDGPU.cpp | 7 +
clang/lib/Driver/ToolChains/Clang.cpp | 47 +-
clang/lib/Driver/ToolChains/Flang.cpp | 7 +-
clang/lib/Driver/ToolChains/Flang.h | 4 +-
.../Driver/hip-offload-object-linking.hip | 81 +
clang/test/Driver/hip-options.hip | 8 +-
clang/test/Driver/hip-thinlto.hip | 12 +-
clang/test/Driver/openmp-offload-gpu.c | 14 +-
.../Driver/openmp-offload-object-linking.c | 21 +
.../clang-linker-wrapper/linker-wrapper.c | 3 +-
.../Driver/omp-offload-object-linking.f90 | 17 +
lld/ELF/AMDGPUObjectLinking.cpp | 1951 +++++++++++++++++
lld/ELF/AMDGPUObjectLinking.h | 25 +
lld/ELF/Arch/AMDGPU.cpp | 6 +
lld/ELF/CMakeLists.txt | 1 +
lld/ELF/Driver.cpp | 20 +-
lld/ELF/InputFiles.cpp | 6 +-
lld/ELF/Target.h | 3 +
lld/test/ELF/amdgpu-big-endian.test | 19 +
lld/test/ELF/amdgpu-lds-link-time-align.s | 110 +
.../ELF/amdgpu-lds-link-time-dynlds-alias.s | 187 ++
.../ELF/amdgpu-lds-link-time-dynlds-align.s | 182 ++
lld/test/ELF/amdgpu-lds-link-time-dynlds.s | 184 ++
lld/test/ELF/amdgpu-lds-link-time-grouped.s | 425 ++++
.../ELF/amdgpu-lds-link-time-indirect-basic.s | 186 ++
.../amdgpu-lds-link-time-indirect-cross-tu.s | 203 ++
.../ELF/amdgpu-lds-link-time-indirect-mixed.s | 180 ++
.../ELF/amdgpu-lds-link-time-indirect-multi.s | 220 ++
.../amdgpu-lds-link-time-indirect-nested.s | 252 +++
.../amdgpu-lds-link-time-indirect-prototype.s | 245 +++
.../amdgpu-lds-link-time-indirect-resource.s | 176 ++
.../ELF/amdgpu-lds-link-time-instructions.s | 198 ++
.../amdgpu-lds-link-time-ir-instructions.s | 266 +++
lld/test/ELF/amdgpu-lds-link-time-kd.s | 175 ++
lld/test/ELF/amdgpu-lds-link-time-layout.s | 170 ++
.../ELF/amdgpu-lds-link-time-named-barrier.s | 260 +++
.../amdgpu-lds-link-time-ordering-complex.s | 402 ++++
.../amdgpu-lds-link-time-ordering-frequency.s | 311 +++
.../amdgpu-lds-link-time-ordering-frontier.s | 112 +
...amdgpu-lds-link-time-ordering-multigroup.s | 505 +++++
.../amdgpu-lds-link-time-ordering-nested.s | 312 +++
.../ELF/amdgpu-lds-link-time-ordering-sizes.s | 209 ++
.../ELF/amdgpu-lds-link-time-ordering-tiers.s | 313 +++
lld/test/ELF/amdgpu-lds-link-time-padding.s | 116 +
.../amdgpu-lds-link-time-per-kernel-sizes.s | 240 ++
.../ELF/amdgpu-lds-link-time-random-layout.s | 432 ++++
.../ELF/amdgpu-lds-link-time-recursive-scc.s | 161 ++
.../amdgpu-lds-link-time-symbol-resolution.s | 169 ++
lld/test/ELF/amdgpu-lds-link-time.s | 134 ++
lld/test/ELF/amdgpu-named-barrier-cross-tu.s | 193 ++
lld/test/ELF/amdgpu-named-barrier-grouped.s | 109 +
.../ELF/amdgpu-object-linking-asm-roundtrip.s | 145 ++
lld/test/ELF/amdgpu-object-linking-eflags.s | 220 ++
lld/test/ELF/amdgpu-object-linking-lto.ll | 31 +
.../amdgpu-object-linking-malformed-strtab.s | 209 ++
.../ELF/amdgpu-object-linking-wave-size.s | 317 +++
lld/test/ELF/amdgpu-resource-usage-alias.s | 173 ++
.../ELF/amdgpu-resource-usage-recursive-scc.s | 257 +++
.../ELF/amdgpu-resource-usage-section-sym.s | 236 ++
.../ELF/amdgpu-resource-usage-stale-info.s | 230 ++
lld/test/ELF/amdgpu-resource-usage.s | 400 ++++
llvm/docs/AMDGPUUsage.rst | 65 +-
llvm/include/llvm/BinaryFormat/ELF.h | 12 +
llvm/include/llvm/Bitcode/LLVMBitCodes.h | 14 +
llvm/include/llvm/IR/ModuleSummaryIndex.h | 37 +
llvm/include/llvm/LTO/Config.h | 6 +
llvm/include/llvm/Object/ELFTypes.h | 5 +
.../llvm/Support/AMDGPUObjLinkingInfo.h | 9 +-
.../llvm/TargetParser/AMDGPUTargetParser.h | 15 +
llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp | 1 +
llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 65 +
llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 105 +
llvm/lib/LTO/LTO.cpp | 195 ++
llvm/lib/LTO/LTOBackend.cpp | 3 +
llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp | 104 +-
.../AMDGPU/AMDGPUHSAMetadataStreamer.cpp | 3 +-
.../lib/Target/AMDGPU/AMDGPULowerExecSync.cpp | 3 +-
.../AMDGPU/AMDGPULowerModuleLDSPass.cpp | 33 +-
.../Target/AMDGPU/AMDGPUMCResourceInfo.cpp | 35 +-
llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp | 19 +
llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h | 6 +
.../lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp | 15 +-
llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp | 34 +-
llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h | 8 +
.../AMDGPU/AsmParser/AMDGPUAsmParser.cpp | 8 +
llvm/lib/Target/AMDGPU/GCNSubtarget.cpp | 12 +-
llvm/lib/Target/AMDGPU/GCNSubtarget.h | 17 +
.../MCTargetDesc/AMDGPUTargetStreamer.cpp | 12 +
.../MCTargetDesc/AMDGPUTargetStreamer.h | 4 +-
.../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 9 +
llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h | 3 +
llvm/lib/TargetParser/AMDGPUTargetParser.cpp | 162 ++
...lds-link-time-codegen-callgraph-lds-use.ll | 56 +
.../AMDGPU/lds-link-time-codegen-comdat.ll | 18 +
...s-link-time-codegen-cross-tu-addr-taken.ll | 42 +
.../lds-link-time-codegen-dynlds-func.ll | 37 +
.../AMDGPU/lds-link-time-codegen-dynlds.ll | 49 +
.../AMDGPU/lds-link-time-codegen-indirect.ll | 54 +
.../AMDGPU/lds-link-time-codegen-multi.ll | 56 +
.../AMDGPU/lds-link-time-codegen-no-attr.ll | 36 +
.../AMDGPU/lds-link-time-codegen-prototype.ll | 80 +
.../AMDGPU/lds-link-time-codegen-typeid.ll | 3 +-
.../CodeGen/AMDGPU/lds-link-time-codegen.ll | 18 +
.../AMDGPU/lds-link-time-named-barrier.ll | 11 +-
...ct-linking-abi-occupancy-device-no-attr.ll | 27 +
...king-abi-occupancy-flat-workgroup-lower.ll | 24 +
...ccupancy-flat-workgroup-reject-override.ll | 28 +
...ct-linking-abi-occupancy-kernel-no-attr.ll | 31 +
.../object-linking-abi-occupancy-override.ll | 30 +
...ct-linking-abi-occupancy-preserves-attr.ll | 34 +
.../AMDGPU/promote-alloca-object-linking.ll | 22 +
.../amdgpu-abi-waves-per-eu-module-flag.ll | 16 +
llvm/test/MC/AMDGPU/amdgpu-info-roundtrip.s | 16 +-
.../ThinLTO/AMDGPU/amdgpu-occupancy-merge.ll | 59 +
.../AMDGPU/amdgpu-occupancy-propagation.ll | 41 +
.../AMDGPU/amdgpu-occupancy-target-default.ll | 45 +
.../AMDGPU/amdgpu-summary-prevailing.ll | 32 +
.../AMDGPU/amdgpu-summary-roundtrip.ll | 45 +
.../TargetParser/TargetParserTest.cpp | 71 +
122 files changed, 14194 insertions(+), 631 deletions(-)
create mode 100644 clang/test/Driver/hip-offload-object-linking.hip
create mode 100644 clang/test/Driver/openmp-offload-object-linking.c
create mode 100644 flang/test/Driver/omp-offload-object-linking.f90
create mode 100644 lld/ELF/AMDGPUObjectLinking.cpp
create mode 100644 lld/ELF/AMDGPUObjectLinking.h
create mode 100644 lld/test/ELF/amdgpu-big-endian.test
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-align.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-dynlds-alias.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-dynlds-align.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-dynlds.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-grouped.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-indirect-basic.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-indirect-cross-tu.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-indirect-mixed.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-indirect-multi.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-indirect-nested.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-indirect-prototype.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-indirect-resource.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-instructions.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-ir-instructions.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-kd.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-layout.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-named-barrier.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-ordering-complex.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-ordering-frequency.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-ordering-frontier.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-ordering-multigroup.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-ordering-nested.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-ordering-sizes.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-ordering-tiers.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-padding.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-per-kernel-sizes.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-random-layout.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-recursive-scc.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time-symbol-resolution.s
create mode 100644 lld/test/ELF/amdgpu-lds-link-time.s
create mode 100644 lld/test/ELF/amdgpu-named-barrier-cross-tu.s
create mode 100644 lld/test/ELF/amdgpu-named-barrier-grouped.s
create mode 100644 lld/test/ELF/amdgpu-object-linking-asm-roundtrip.s
create mode 100644 lld/test/ELF/amdgpu-object-linking-eflags.s
create mode 100644 lld/test/ELF/amdgpu-object-linking-lto.ll
create mode 100644 lld/test/ELF/amdgpu-object-linking-malformed-strtab.s
create mode 100644 lld/test/ELF/amdgpu-object-linking-wave-size.s
create mode 100644 lld/test/ELF/amdgpu-resource-usage-alias.s
create mode 100644 lld/test/ELF/amdgpu-resource-usage-recursive-scc.s
create mode 100644 lld/test/ELF/amdgpu-resource-usage-section-sym.s
create mode 100644 lld/test/ELF/amdgpu-resource-usage-stale-info.s
create mode 100644 lld/test/ELF/amdgpu-resource-usage.s
create mode 100644 llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-callgraph-lds-use.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-comdat.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-cross-tu-addr-taken.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-dynlds-func.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-dynlds.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-indirect.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-multi.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-no-attr.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-prototype.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/object-linking-abi-occupancy-device-no-attr.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/object-linking-abi-occupancy-flat-workgroup-lower.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/object-linking-abi-occupancy-flat-workgroup-reject-override.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/object-linking-abi-occupancy-kernel-no-attr.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/object-linking-abi-occupancy-override.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/object-linking-abi-occupancy-preserves-attr.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/promote-alloca-object-linking.ll
create mode 100644 llvm/test/Linker/amdgpu-abi-waves-per-eu-module-flag.ll
create mode 100644 llvm/test/ThinLTO/AMDGPU/amdgpu-occupancy-merge.ll
create mode 100644 llvm/test/ThinLTO/AMDGPU/amdgpu-occupancy-propagation.ll
create mode 100644 llvm/test/ThinLTO/AMDGPU/amdgpu-occupancy-target-default.ll
create mode 100644 llvm/test/ThinLTO/AMDGPU/amdgpu-summary-prevailing.ll
create mode 100644 llvm/test/ThinLTO/AMDGPU/amdgpu-summary-roundtrip.ll
diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.td b/clang/include/clang/Basic/BuiltinsAMDGPU.td
index 4e4e0ee275a16..131db6b99bf76 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.td
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.td
@@ -227,11 +227,11 @@ def __builtin_amdgcn_alignbit : AMDGPUBuiltin<"unsigned int(unsigned int, unsign
def __builtin_amdgcn_alignbyte : AMDGPUBuiltin<"unsigned int(unsigned int, unsigned int, unsigned int)", [Const]>;
def __builtin_amdgcn_ubfe : AMDGPUBuiltin<"unsigned int(unsigned int, unsigned int, unsigned int)", [Const]>;
def __builtin_amdgcn_sbfe : AMDGPUBuiltin<"unsigned int(unsigned int, unsigned int, unsigned int)", [Const]>;
-def __builtin_amdgcn_cvt_pkrtz : AMDGPUBuiltin<"_ExtVector<2, __fp16>(float, float)", [Const]>;
-def __builtin_amdgcn_cvt_pknorm_i16 : AMDGPUBuiltin<"_ExtVector<2, short>(float, float)", [Const], "cvt-pknorm-vop2-insts">;
-def __builtin_amdgcn_cvt_pknorm_u16 : AMDGPUBuiltin<"_ExtVector<2, unsigned short>(float, float)", [Const], "cvt-pknorm-vop2-insts">;
-def __builtin_amdgcn_cvt_pk_i16 : AMDGPUBuiltin<"_ExtVector<2, short>(int, int)", [Const]>;
-def __builtin_amdgcn_cvt_pk_u16 : AMDGPUBuiltin<"_ExtVector<2, unsigned short>(unsigned int, unsigned int)", [Const]>;
+def __builtin_amdgcn_cvt_pkrtz : AMDGPUBuiltin<"_Vector<2, __fp16>(float, float)", [Const]>;
+def __builtin_amdgcn_cvt_pknorm_i16 : AMDGPUBuiltin<"_Vector<2, short>(float, float)", [Const], "cvt-pknorm-vop2-insts">;
+def __builtin_amdgcn_cvt_pknorm_u16 : AMDGPUBuiltin<"_Vector<2, unsigned short>(float, float)", [Const], "cvt-pknorm-vop2-insts">;
+def __builtin_amdgcn_cvt_pk_i16 : AMDGPUBuiltin<"_Vector<2, short>(int, int)", [Const]>;
+def __builtin_amdgcn_cvt_pk_u16 : AMDGPUBuiltin<"_Vector<2, unsigned short>(unsigned int, unsigned int)", [Const]>;
def __builtin_amdgcn_cvt_pk_u8_f32 : AMDGPUBuiltin<"unsigned int(float, unsigned int, unsigned int)", [Const]>;
def __builtin_amdgcn_cvt_off_f32_i4 : AMDGPUBuiltin<"float(int)", [Const]>;
def __builtin_amdgcn_msad_u8 : AMDGPUBuiltin<"unsigned int(unsigned int, unsigned int, unsigned int)", [Const], "msad-insts">;
@@ -240,7 +240,7 @@ def __builtin_amdgcn_sad_hi_u8 : AMDGPUBuiltin<"unsigned int(unsigned int, unsig
def __builtin_amdgcn_sad_u16 : AMDGPUBuiltin<"unsigned int(unsigned int, unsigned int, unsigned int)", [Const], "sad-insts">;
def __builtin_amdgcn_qsad_pk_u16_u8 : AMDGPUBuiltin<"uint64_t(uint64_t, unsigned int, uint64_t)", [Const], "qsad-insts">;
def __builtin_amdgcn_mqsad_pk_u16_u8 : AMDGPUBuiltin<"uint64_t(uint64_t, unsigned int, uint64_t)", [Const], "mqsad-pk-insts">;
-def __builtin_amdgcn_mqsad_u32_u8 : AMDGPUBuiltin<"_ExtVector<4, unsigned int>(uint64_t, unsigned int, _ExtVector<4, unsigned int>)", [Const], "mqsad-insts">;
+def __builtin_amdgcn_mqsad_u32_u8 : AMDGPUBuiltin<"_Vector<4, unsigned int>(uint64_t, unsigned int, _Vector<4, unsigned int>)", [Const], "mqsad-insts">;
//===----------------------------------------------------------------------===//
// Buffer builtins.
@@ -250,24 +250,24 @@ def __builtin_amdgcn_make_buffer_rsrc : AMDGPUBuiltin<"__amdgpu_buffer_rsrc_t(vo
def __builtin_amdgcn_raw_buffer_store_b8 : AMDGPUBuiltin<"void(unsigned char, __amdgpu_buffer_rsrc_t, int, int, _Constant int)">;
def __builtin_amdgcn_raw_buffer_store_b16 : AMDGPUBuiltin<"void(unsigned short, __amdgpu_buffer_rsrc_t, int, int, _Constant int)">;
def __builtin_amdgcn_raw_buffer_store_b32 : AMDGPUBuiltin<"void(unsigned int, __amdgpu_buffer_rsrc_t, int, int, _Constant int)">;
-def __builtin_amdgcn_raw_buffer_store_b64 : AMDGPUBuiltin<"void(_ExtVector<2, unsigned int>, __amdgpu_buffer_rsrc_t, int, int, _Constant int)">;
-def __builtin_amdgcn_raw_buffer_store_b96 : AMDGPUBuiltin<"void(_ExtVector<3, unsigned int>, __amdgpu_buffer_rsrc_t, int, int, _Constant int)">;
-def __builtin_amdgcn_raw_buffer_store_b128 : AMDGPUBuiltin<"void(_ExtVector<4, unsigned int>, __amdgpu_buffer_rsrc_t, int, int, _Constant int)">;
+def __builtin_amdgcn_raw_buffer_store_b64 : AMDGPUBuiltin<"void(_Vector<2, unsigned int>, __amdgpu_buffer_rsrc_t, int, int, _Constant int)">;
+def __builtin_amdgcn_raw_buffer_store_b96 : AMDGPUBuiltin<"void(_Vector<3, unsigned int>, __amdgpu_buffer_rsrc_t, int, int, _Constant int)">;
+def __builtin_amdgcn_raw_buffer_store_b128 : AMDGPUBuiltin<"void(_Vector<4, unsigned int>, __amdgpu_buffer_rsrc_t, int, int, _Constant int)">;
def __builtin_amdgcn_raw_buffer_load_b8 : AMDGPUBuiltin<"unsigned char(__amdgpu_buffer_rsrc_t, int, int, _Constant int)">;
def __builtin_amdgcn_raw_buffer_load_b16 : AMDGPUBuiltin<"unsigned short(__amdgpu_buffer_rsrc_t, int, int, _Constant int)">;
def __builtin_amdgcn_raw_buffer_load_b32 : AMDGPUBuiltin<"unsigned int(__amdgpu_buffer_rsrc_t, int, int, _Constant int)">;
-def __builtin_amdgcn_raw_buffer_load_b64 : AMDGPUBuiltin<"_ExtVector<2, unsigned int>(__amdgpu_buffer_rsrc_t, int, int, _Constant int)">;
-def __builtin_amdgcn_raw_buffer_load_b96 : AMDGPUBuiltin<"_ExtVector<3, unsigned int>(__amdgpu_buffer_rsrc_t, int, int, _Constant int)">;
-def __builtin_amdgcn_raw_buffer_load_b128 : AMDGPUBuiltin<"_ExtVector<4, unsigned int>(__amdgpu_buffer_rsrc_t, int, int, _Constant int)">;
-
-def __builtin_amdgcn_raw_buffer_load_format_v4f32 : AMDGPUBuiltin<"_ExtVector<4, float>(__amdgpu_buffer_rsrc_t, int, int, _Constant int)">;
-def __builtin_amdgcn_raw_buffer_load_format_v4f16 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(__amdgpu_buffer_rsrc_t, int, int, _Constant int)", [], "16-bit-insts">;
-def __builtin_amdgcn_raw_buffer_store_format_v4f32 : AMDGPUBuiltin<"void(_ExtVector<4, float>, __amdgpu_buffer_rsrc_t, int, int, _Constant int)">;
-def __builtin_amdgcn_raw_buffer_store_format_v4f16 : AMDGPUBuiltin<"void(_ExtVector<4, _Float16>, __amdgpu_buffer_rsrc_t, int, int, _Constant int)", [], "16-bit-insts">;
-def __builtin_amdgcn_struct_buffer_load_format_v4f32 : AMDGPUBuiltin<"_ExtVector<4, float>(__amdgpu_buffer_rsrc_t, int, int, int, _Constant int)">;
-def __builtin_amdgcn_struct_buffer_load_format_v4f16 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(__amdgpu_buffer_rsrc_t, int, int, int, _Constant int)", [], "16-bit-insts">;
-def __builtin_amdgcn_struct_buffer_store_format_v4f32 : AMDGPUBuiltin<"void(_ExtVector<4, float>, __amdgpu_buffer_rsrc_t, int, int, int, _Constant int)">;
-def __builtin_amdgcn_struct_buffer_store_format_v4f16 : AMDGPUBuiltin<"void(_ExtVector<4, _Float16>, __amdgpu_buffer_rsrc_t, int, int, int, _Constant int)", [], "16-bit-insts">;
+def __builtin_amdgcn_raw_buffer_load_b64 : AMDGPUBuiltin<"_Vector<2, unsigned int>(__amdgpu_buffer_rsrc_t, int, int, _Constant int)">;
+def __builtin_amdgcn_raw_buffer_load_b96 : AMDGPUBuiltin<"_Vector<3, unsigned int>(__amdgpu_buffer_rsrc_t, int, int, _Constant int)">;
+def __builtin_amdgcn_raw_buffer_load_b128 : AMDGPUBuiltin<"_Vector<4, unsigned int>(__amdgpu_buffer_rsrc_t, int, int, _Constant int)">;
+
+def __builtin_amdgcn_raw_buffer_load_format_v4f32 : AMDGPUBuiltin<"_Vector<4, float>(__amdgpu_buffer_rsrc_t, int, int, _Constant int)">;
+def __builtin_amdgcn_raw_buffer_load_format_v4f16 : AMDGPUBuiltin<"_Vector<4, _Float16>(__amdgpu_buffer_rsrc_t, int, int, _Constant int)", [], "16-bit-insts">;
+def __builtin_amdgcn_raw_buffer_store_format_v4f32 : AMDGPUBuiltin<"void(_Vector<4, float>, __amdgpu_buffer_rsrc_t, int, int, _Constant int)">;
+def __builtin_amdgcn_raw_buffer_store_format_v4f16 : AMDGPUBuiltin<"void(_Vector<4, _Float16>, __amdgpu_buffer_rsrc_t, int, int, _Constant int)", [], "16-bit-insts">;
+def __builtin_amdgcn_struct_buffer_load_format_v4f32 : AMDGPUBuiltin<"_Vector<4, float>(__amdgpu_buffer_rsrc_t, int, int, int, _Constant int)">;
+def __builtin_amdgcn_struct_buffer_load_format_v4f16 : AMDGPUBuiltin<"_Vector<4, _Float16>(__amdgpu_buffer_rsrc_t, int, int, int, _Constant int)", [], "16-bit-insts">;
+def __builtin_amdgcn_struct_buffer_store_format_v4f32 : AMDGPUBuiltin<"void(_Vector<4, float>, __amdgpu_buffer_rsrc_t, int, int, int, _Constant int)">;
+def __builtin_amdgcn_struct_buffer_store_format_v4f16 : AMDGPUBuiltin<"void(_Vector<4, _Float16>, __amdgpu_buffer_rsrc_t, int, int, int, _Constant int)", [], "16-bit-insts">;
def __builtin_amdgcn_raw_ptr_buffer_atomic_add_i32 : AMDGPUBuiltin<"int(int, __amdgpu_buffer_rsrc_t, int, int, _Constant int)">;
@@ -276,7 +276,7 @@ def __builtin_amdgcn_raw_ptr_buffer_atomic_fadd_f64 : AMDGPUBuiltin<"double(doub
let Documentation = [DocRawPtrBufferAtomicFAddF64];
let ArgNames = ["vdata", "rsrc", "offset", "soffset", "aux"];
}
-def __builtin_amdgcn_raw_ptr_buffer_atomic_fadd_v2f16 : AMDGPUBuiltin<"_ExtVector<2, _Float16>(_ExtVector<2, _Float16>, __amdgpu_buffer_rsrc_t, int, int, _Constant int)", [], "atomic-buffer-global-pk-add-f16-insts">;
+def __builtin_amdgcn_raw_ptr_buffer_atomic_fadd_v2f16 : AMDGPUBuiltin<"_Vector<2, _Float16>(_Vector<2, _Float16>, __amdgpu_buffer_rsrc_t, int, int, _Constant int)", [], "atomic-buffer-global-pk-add-f16-insts">;
def __builtin_amdgcn_raw_ptr_buffer_atomic_fmin_f32 : AMDGPUBuiltin<"float(float, __amdgpu_buffer_rsrc_t, int, int, _Constant int)", [], "atomic-fmin-fmax-global-f32">;
def __builtin_amdgcn_raw_ptr_buffer_atomic_fmax_f32 : AMDGPUBuiltin<"float(float, __amdgpu_buffer_rsrc_t, int, int, _Constant int)", [], "atomic-fmin-fmax-global-f32">;
@@ -319,12 +319,12 @@ def __builtin_amdgcn_s_buffer_load_v4f16 : AMDGPUBuiltin<"_Vector<4, _Float16>(_
//===----------------------------------------------------------------------===//
def __builtin_amdgcn_av_load_b128
- : AMDGPUBuiltin<"_ExtVector<4, unsigned int>(_ExtVector<4, unsigned int> *, int)", [], "flat-global-insts"> {
+ : AMDGPUBuiltin<"_Vector<4, unsigned int>(_Vector<4, unsigned int> *, int)", [], "flat-global-insts"> {
let Documentation = [DocAVLoadB128];
let ArgNames = ["ptr", "scope"];
}
def __builtin_amdgcn_av_store_b128
- : AMDGPUBuiltin<"void(_ExtVector<4, unsigned int> *, _ExtVector<4, unsigned int>, int)", [], "flat-global-insts"> {
+ : AMDGPUBuiltin<"void(_Vector<4, unsigned int> *, _Vector<4, unsigned int>, int)", [], "flat-global-insts"> {
let Documentation = [DocAVStoreB128];
let ArgNames = ["ptr", "data", "scope"];
}
@@ -419,7 +419,7 @@ def __builtin_amdgcn_s_bitreplicate : AMDGPUBuiltin<"uint64_t(unsigned int)", [C
def __builtin_amdgcn_global_atomic_fadd_f64 : AMDGPUBuiltin<"double(double address_space<1> *, double)", [], "gfx90a-insts">;
def __builtin_amdgcn_global_atomic_fadd_f32 : AMDGPUBuiltin<"float(float address_space<1> *, float)", [], "atomic-fadd-rtn-insts">;
-def __builtin_amdgcn_global_atomic_fadd_v2f16 : AMDGPUBuiltin<"_ExtVector<2, _Float16>(_ExtVector<2, _Float16> address_space<1> *, _ExtVector<2, _Float16>)", [CustomTypeChecking], "atomic-buffer-global-pk-add-f16-insts">;
+def __builtin_amdgcn_global_atomic_fadd_v2f16 : AMDGPUBuiltin<"_Vector<2, _Float16>(_Vector<2, _Float16> address_space<1> *, _Vector<2, _Float16>)", [CustomTypeChecking], "atomic-buffer-global-pk-add-f16-insts">;
def __builtin_amdgcn_global_atomic_fmin_f64 : AMDGPUBuiltin<"double(double address_space<1> *, double)", [], "gfx90a-insts">;
def __builtin_amdgcn_global_atomic_fmax_f64 : AMDGPUBuiltin<"double(double address_space<1> *, double)", [], "gfx90a-insts">;
@@ -431,11 +431,11 @@ def __builtin_amdgcn_ds_atomic_fadd_f64 : AMDGPUBuiltin<"double(double address_s
def __builtin_amdgcn_ds_atomic_fadd_f32 : AMDGPUBuiltin<"float(float address_space<3> *, float)", [], "gfx8-insts">;
def __builtin_amdgcn_flat_atomic_fadd_f32 : AMDGPUBuiltin<"float(float address_space<0> *, float)", [], "gfx940-insts">;
-def __builtin_amdgcn_flat_atomic_fadd_v2f16 : AMDGPUBuiltin<"_ExtVector<2, _Float16>(_ExtVector<2, _Float16> address_space<0> *, _ExtVector<2, _Float16>)", [CustomTypeChecking], "atomic-flat-pk-add-16-insts">;
-def __builtin_amdgcn_flat_atomic_fadd_v2bf16 : AMDGPUBuiltin<"_ExtVector<2, short>(_ExtVector<2, short> address_space<0> *, _ExtVector<2, short>)", [CustomTypeChecking], "atomic-flat-pk-add-16-insts">;
-def __builtin_amdgcn_global_atomic_fadd_v2bf16 : AMDGPUBuiltin<"_ExtVector<2, short>(_ExtVector<2, short> address_space<1> *, _ExtVector<2, short>)", [CustomTypeChecking], "atomic-global-pk-add-bf16-inst">;
-def __builtin_amdgcn_ds_atomic_fadd_v2bf16 : AMDGPUBuiltin<"_ExtVector<2, short>(_ExtVector<2, short> address_space<3> *, _ExtVector<2, short>)", [CustomTypeChecking], "atomic-ds-pk-add-16-insts">;
-def __builtin_amdgcn_ds_atomic_fadd_v2f16 : AMDGPUBuiltin<"_ExtVector<2, _Float16>(_ExtVector<2, _Float16> address_space<3> *, _ExtVector<2, _Float16>)", [CustomTypeChecking], "atomic-ds-pk-add-16-insts">;
+def __builtin_amdgcn_flat_atomic_fadd_v2f16 : AMDGPUBuiltin<"_Vector<2, _Float16>(_Vector<2, _Float16> address_space<0> *, _Vector<2, _Float16>)", [CustomTypeChecking], "atomic-flat-pk-add-16-insts">;
+def __builtin_amdgcn_flat_atomic_fadd_v2bf16 : AMDGPUBuiltin<"_Vector<2, short>(_Vector<2, short> address_space<0> *, _Vector<2, short>)", [CustomTypeChecking], "atomic-flat-pk-add-16-insts">;
+def __builtin_amdgcn_global_atomic_fadd_v2bf16 : AMDGPUBuiltin<"_Vector<2, short>(_Vector<2, short> address_space<1> *, _Vector<2, short>)", [CustomTypeChecking], "atomic-global-pk-add-bf16-inst">;
+def __builtin_amdgcn_ds_atomic_fadd_v2bf16 : AMDGPUBuiltin<"_Vector<2, short>(_Vector<2, short> address_space<3> *, _Vector<2, short>)", [CustomTypeChecking], "atomic-ds-pk-add-16-insts">;
+def __builtin_amdgcn_ds_atomic_fadd_v2f16 : AMDGPUBuiltin<"_Vector<2, _Float16>(_Vector<2, _Float16> address_space<3> *, _Vector<2, _Float16>)", [CustomTypeChecking], "atomic-ds-pk-add-16-insts">;
def __builtin_amdgcn_load_to_lds : AMDGPUBuiltin<"void(void *, void address_space<3> *, _Constant unsigned int, _Constant int, _Constant unsigned int)", [], "vmem-to-lds-load-insts">;
def __builtin_amdgcn_load_async_to_lds : AMDGPUBuiltin<"void(void *, void address_space<3> *, _Constant unsigned int, _Constant int, _Constant unsigned int)", [], "vmem-to-lds-load-insts">;
def __builtin_amdgcn_global_load_lds : AMDGPUBuiltin<"void(void address_space<1> *, void address_space<3> *, _Constant unsigned int, _Constant int, _Constant unsigned int)", [], "vmem-to-lds-load-insts">;
@@ -445,12 +445,12 @@ def __builtin_amdgcn_global_load_async_lds : AMDGPUBuiltin<"void(void address_sp
// Deep learning builtins.
//===----------------------------------------------------------------------===//
-def __builtin_amdgcn_fdot2 : AMDGPUBuiltin<"float(_ExtVector<2, _Float16>, _ExtVector<2, _Float16>, float, _Constant bool)", [Const], "dot10-insts">;
-def __builtin_amdgcn_fdot2_f16_f16 : AMDGPUBuiltin<"_Float16(_ExtVector<2, _Float16>, _ExtVector<2, _Float16>, _Float16)", [Const], "dot9-insts">;
-def __builtin_amdgcn_fdot2_bf16_bf16 : AMDGPUBuiltin<"short(_ExtVector<2, short>, _ExtVector<2, short>, short)", [Const], "dot9-insts">;
-def __builtin_amdgcn_fdot2_f32_bf16 : AMDGPUBuiltin<"float(_ExtVector<2, short>, _ExtVector<2, short>, float, _Constant bool)", [Const], "dot12-insts">;
-def __builtin_amdgcn_sdot2 : AMDGPUBuiltin<"int(_ExtVector<2, short>, _ExtVector<2, short>, int, _Constant bool)", [Const], "dot2-insts">;
-def __builtin_amdgcn_udot2 : AMDGPUBuiltin<"unsigned int(_ExtVector<2, unsigned short>, _ExtVector<2, unsigned short>, unsigned int, _Constant bool)", [Const], "dot2-insts">;
+def __builtin_amdgcn_fdot2 : AMDGPUBuiltin<"float(_Vector<2, _Float16>, _Vector<2, _Float16>, float, _Constant bool)", [Const], "dot10-insts">;
+def __builtin_amdgcn_fdot2_f16_f16 : AMDGPUBuiltin<"_Float16(_Vector<2, _Float16>, _Vector<2, _Float16>, _Float16)", [Const], "dot9-insts">;
+def __builtin_amdgcn_fdot2_bf16_bf16 : AMDGPUBuiltin<"short(_Vector<2, short>, _Vector<2, short>, short)", [Const], "dot9-insts">;
+def __builtin_amdgcn_fdot2_f32_bf16 : AMDGPUBuiltin<"float(_Vector<2, short>, _Vector<2, short>, float, _Constant bool)", [Const], "dot12-insts">;
+def __builtin_amdgcn_sdot2 : AMDGPUBuiltin<"int(_Vector<2, short>, _Vector<2, short>, int, _Constant bool)", [Const], "dot2-insts">;
+def __builtin_amdgcn_udot2 : AMDGPUBuiltin<"unsigned int(_Vector<2, unsigned short>, _Vector<2, unsigned short>, unsigned int, _Constant bool)", [Const], "dot2-insts">;
def __builtin_amdgcn_sdot4 : AMDGPUBuiltin<"int(int, int, int, _Constant bool)", [Const], "dot1-insts">;
def __builtin_amdgcn_udot4 : AMDGPUBuiltin<"unsigned int(unsigned int, unsigned int, unsigned int, _Constant bool)", [Const], "dot7-insts">;
def __builtin_amdgcn_sudot4 : AMDGPUBuiltin<"int(_Constant bool, int, _Constant bool, int, int, _Constant bool)", [Const], "dot8-insts">;
@@ -461,7 +461,7 @@ def __builtin_amdgcn_dot4_f32_fp8_bf8 : AMDGPUBuiltin<"float(unsigned int, unsig
def __builtin_amdgcn_dot4_f32_bf8_fp8 : AMDGPUBuiltin<"float(unsigned int, unsigned int, float)", [Const], "dot11-insts">;
def __builtin_amdgcn_dot4_f32_fp8_fp8 : AMDGPUBuiltin<"float(unsigned int, unsigned int, float)", [Const], "dot11-insts">;
def __builtin_amdgcn_dot4_f32_bf8_bf8 : AMDGPUBuiltin<"float(unsigned int, unsigned int, float)", [Const], "dot11-insts">;
-def __builtin_amdgcn_fdot2c_f32_bf16 : AMDGPUBuiltin<"float(_ExtVector<2, __bf16>, _ExtVector<2, __bf16>, float, _Constant bool)", [Const], "dot13-insts">;
+def __builtin_amdgcn_fdot2c_f32_bf16 : AMDGPUBuiltin<"float(_Vector<2, __bf16>, _Vector<2, __bf16>, float, _Constant bool)", [Const], "dot13-insts">;
//===----------------------------------------------------------------------===//
// GFX10+ only builtins.
@@ -477,10 +477,10 @@ def __builtin_amdgcn_s_ttracedata_imm : AMDGPUBuiltin<"void(_Constant short)", [
// Postfix l indicates the 1st argument is i64.
// Postfix h indicates the 4/5-th arguments are half4.
//===----------------------------------------------------------------------===//
-def __builtin_amdgcn_image_bvh_intersect_ray : AMDGPUBuiltin<"_ExtVector<4, unsigned int>(unsigned int, float, _ExtVector<4, float>, _ExtVector<4, float>, _ExtVector<4, float>, _ExtVector<4, unsigned int>)", [Const], "bvh-ray-tracing-insts">;
-def __builtin_amdgcn_image_bvh_intersect_ray_h : AMDGPUBuiltin<"_ExtVector<4, unsigned int>(unsigned int, float, _ExtVector<4, float>, _ExtVector<4, _Float16>, _ExtVector<4, _Float16>, _ExtVector<4, unsigned int>)", [Const], "bvh-ray-tracing-insts">;
-def __builtin_amdgcn_image_bvh_intersect_ray_l : AMDGPUBuiltin<"_ExtVector<4, unsigned int>(uint64_t, float, _ExtVector<4, float>, _ExtVector<4, float>, _ExtVector<4, float>, _ExtVector<4, unsigned int>)", [Const], "bvh-ray-tracing-insts">;
-def __builtin_amdgcn_image_bvh_intersect_ray_lh : AMDGPUBuiltin<"_ExtVector<4, unsigned int>(uint64_t, float, _ExtVector<4, float>, _ExtVector<4, _Float16>, _ExtVector<4, _Float16>, _ExtVector<4, unsigned int>)", [Const], "bvh-ray-tracing-insts">;
+def __builtin_amdgcn_image_bvh_intersect_ray : AMDGPUBuiltin<"_Vector<4, unsigned int>(unsigned int, float, _Vector<4, float>, _Vector<4, float>, _Vector<4, float>, _Vector<4, unsigned int>)", [Const], "bvh-ray-tracing-insts">;
+def __builtin_amdgcn_image_bvh_intersect_ray_h : AMDGPUBuiltin<"_Vector<4, unsigned int>(unsigned int, float, _Vector<4, float>, _Vector<4, _Float16>, _Vector<4, _Float16>, _Vector<4, unsigned int>)", [Const], "bvh-ray-tracing-insts">;
+def __builtin_amdgcn_image_bvh_intersect_ray_l : AMDGPUBuiltin<"_Vector<4, unsigned int>(uint64_t, float, _Vector<4, float>, _Vector<4, float>, _Vector<4, float>, _Vector<4, unsigned int>)", [Const], "bvh-ray-tracing-insts">;
+def __builtin_amdgcn_image_bvh_intersect_ray_lh : AMDGPUBuiltin<"_Vector<4, unsigned int>(uint64_t, float, _Vector<4, float>, _Vector<4, _Float16>, _Vector<4, _Float16>, _Vector<4, unsigned int>)", [Const], "bvh-ray-tracing-insts">;
//===----------------------------------------------------------------------===//
@@ -497,68 +497,68 @@ def __builtin_amdgcn_s_wait_event : AMDGPUBuiltin<"void(_Constant short)", [], "
// Postfix w32 indicates the builtin requires wavefront size of 32.
// Postfix w64 indicates the builtin requires wavefront size of 64.
//===----------------------------------------------------------------------===//
-def __builtin_amdgcn_wmma_f32_16x16x16_f16_w32 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<16, _Float16>, _ExtVector<16, _Float16>, _ExtVector<8, float>)", [Const], "wmma-256b-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f32_16x16x16_f16_w32 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<16, _Float16>, _Vector<16, _Float16>, _Vector<8, float>)", [Const], "wmma-256b-insts,wavefrontsize32"> {
let Documentation = [DocWMMAF32_16x16x16_GFX11];
let ArgNames = ["a", "b", "c"];
}
-def __builtin_amdgcn_wmma_f32_16x16x16_bf16_w32 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<16, short>, _ExtVector<16, short>, _ExtVector<8, float>)", [Const], "wmma-256b-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f32_16x16x16_bf16_w32 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<16, short>, _Vector<16, short>, _Vector<8, float>)", [Const], "wmma-256b-insts,wavefrontsize32"> {
let Documentation = [DocWMMAF32_16x16x16_GFX11];
let ArgNames = ["a", "b", "c"];
}
-def __builtin_amdgcn_wmma_f16_16x16x16_f16_w32 : AMDGPUBuiltin<"_ExtVector<16, _Float16>(_ExtVector<16, _Float16>, _ExtVector<16, _Float16>, _ExtVector<16, _Float16>, _Constant bool)", [Const], "wmma-256b-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f16_16x16x16_f16_w32 : AMDGPUBuiltin<"_Vector<16, _Float16>(_Vector<16, _Float16>, _Vector<16, _Float16>, _Vector<16, _Float16>, _Constant bool)", [Const], "wmma-256b-insts,wavefrontsize32"> {
let Documentation = [DocWMMAOpsel_16x16x16_GFX11];
let ArgNames = ["a", "b", "c", "opsel"];
}
-def __builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32 : AMDGPUBuiltin<"_ExtVector<16, short>(_ExtVector<16, short>, _ExtVector<16, short>, _ExtVector<16, short>, _Constant bool)", [Const], "wmma-256b-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32 : AMDGPUBuiltin<"_Vector<16, short>(_Vector<16, short>, _Vector<16, short>, _Vector<16, short>, _Constant bool)", [Const], "wmma-256b-insts,wavefrontsize32"> {
let Documentation = [DocWMMAOpsel_16x16x16_GFX11];
let ArgNames = ["a", "b", "c", "opsel"];
}
-def __builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w32 : AMDGPUBuiltin<"_ExtVector<16, _Float16>(_ExtVector<16, _Float16>, _ExtVector<16, _Float16>, _ExtVector<16, _Float16>, _Constant bool)", [Const], "wmma-256b-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w32 : AMDGPUBuiltin<"_Vector<16, _Float16>(_Vector<16, _Float16>, _Vector<16, _Float16>, _Vector<16, _Float16>, _Constant bool)", [Const], "wmma-256b-insts,wavefrontsize32"> {
let Documentation = [DocWMMAOpselTied_16x16x16_GFX11];
let ArgNames = ["a", "b", "c", "opsel"];
}
-def __builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w32 : AMDGPUBuiltin<"_ExtVector<16, short>(_ExtVector<16, short>, _ExtVector<16, short>, _ExtVector<16, short>, _Constant bool)", [Const], "wmma-256b-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w32 : AMDGPUBuiltin<"_Vector<16, short>(_Vector<16, short>, _Vector<16, short>, _Vector<16, short>, _Constant bool)", [Const], "wmma-256b-insts,wavefrontsize32"> {
let Documentation = [DocWMMAOpselTied_16x16x16_GFX11];
let ArgNames = ["a", "b", "c", "opsel"];
}
-def __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32 : AMDGPUBuiltin<"_ExtVector<8, int>(_Constant bool, _ExtVector<4, int>, _Constant bool, _ExtVector<4, int>, _ExtVector<8, int>, _Constant bool)", [Const], "wmma-256b-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32 : AMDGPUBuiltin<"_Vector<8, int>(_Constant bool, _Vector<4, int>, _Constant bool, _Vector<4, int>, _Vector<8, int>, _Constant bool)", [Const], "wmma-256b-insts,wavefrontsize32"> {
let Documentation = [DocWMMAIU_16x16x16_GFX11];
let ArgNames = ["a_sign", "a", "b_sign", "b", "c", "clamp"];
}
-def __builtin_amdgcn_wmma_i32_16x16x16_iu4_w32 : AMDGPUBuiltin<"_ExtVector<8, int>(_Constant bool, _ExtVector<2, int>, _Constant bool, _ExtVector<2, int>, _ExtVector<8, int>, _Constant bool)", [Const], "wmma-256b-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_i32_16x16x16_iu4_w32 : AMDGPUBuiltin<"_Vector<8, int>(_Constant bool, _Vector<2, int>, _Constant bool, _Vector<2, int>, _Vector<8, int>, _Constant bool)", [Const], "wmma-256b-insts,wavefrontsize32"> {
let Documentation = [DocWMMAIU_16x16x16_GFX11];
let ArgNames = ["a_sign", "a", "b_sign", "b", "c", "clamp"];
}
-def __builtin_amdgcn_wmma_f32_16x16x16_f16_w64 : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<16, _Float16>, _ExtVector<16, _Float16>, _ExtVector<4, float>)", [Const], "wmma-256b-insts,wavefrontsize64"> {
+def __builtin_amdgcn_wmma_f32_16x16x16_f16_w64 : AMDGPUBuiltin<"_Vector<4, float>(_Vector<16, _Float16>, _Vector<16, _Float16>, _Vector<4, float>)", [Const], "wmma-256b-insts,wavefrontsize64"> {
let Documentation = [DocWMMAF32_16x16x16_GFX11];
let ArgNames = ["a", "b", "c"];
}
-def __builtin_amdgcn_wmma_f32_16x16x16_bf16_w64 : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<16, short>, _ExtVector<16, short>, _ExtVector<4, float>)", [Const], "wmma-256b-insts,wavefrontsize64"> {
+def __builtin_amdgcn_wmma_f32_16x16x16_bf16_w64 : AMDGPUBuiltin<"_Vector<4, float>(_Vector<16, short>, _Vector<16, short>, _Vector<4, float>)", [Const], "wmma-256b-insts,wavefrontsize64"> {
let Documentation = [DocWMMAF32_16x16x16_GFX11];
let ArgNames = ["a", "b", "c"];
}
-def __builtin_amdgcn_wmma_f16_16x16x16_f16_w64 : AMDGPUBuiltin<"_ExtVector<8, _Float16>(_ExtVector<16, _Float16>, _ExtVector<16, _Float16>, _ExtVector<8, _Float16>, _Constant bool)", [Const], "wmma-256b-insts,wavefrontsize64"> {
+def __builtin_amdgcn_wmma_f16_16x16x16_f16_w64 : AMDGPUBuiltin<"_Vector<8, _Float16>(_Vector<16, _Float16>, _Vector<16, _Float16>, _Vector<8, _Float16>, _Constant bool)", [Const], "wmma-256b-insts,wavefrontsize64"> {
let Documentation = [DocWMMAOpsel_16x16x16_GFX11];
let ArgNames = ["a", "b", "c", "opsel"];
}
-def __builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64 : AMDGPUBuiltin<"_ExtVector<8, short>(_ExtVector<16, short>, _ExtVector<16, short>, _ExtVector<8, short>, _Constant bool)", [Const], "wmma-256b-insts,wavefrontsize64"> {
+def __builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64 : AMDGPUBuiltin<"_Vector<8, short>(_Vector<16, short>, _Vector<16, short>, _Vector<8, short>, _Constant bool)", [Const], "wmma-256b-insts,wavefrontsize64"> {
let Documentation = [DocWMMAOpsel_16x16x16_GFX11];
let ArgNames = ["a", "b", "c", "opsel"];
}
-def __builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w64 : AMDGPUBuiltin<"_ExtVector<8, _Float16>(_ExtVector<16, _Float16>, _ExtVector<16, _Float16>, _ExtVector<8, _Float16>, _Constant bool)", [Const], "wmma-256b-insts,wavefrontsize64"> {
+def __builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w64 : AMDGPUBuiltin<"_Vector<8, _Float16>(_Vector<16, _Float16>, _Vector<16, _Float16>, _Vector<8, _Float16>, _Constant bool)", [Const], "wmma-256b-insts,wavefrontsize64"> {
let Documentation = [DocWMMAOpselTied_16x16x16_GFX11];
let ArgNames = ["a", "b", "c", "opsel"];
}
-def __builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w64 : AMDGPUBuiltin<"_ExtVector<8, short>(_ExtVector<16, short>, _ExtVector<16, short>, _ExtVector<8, short>, _Constant bool)", [Const], "wmma-256b-insts,wavefrontsize64"> {
+def __builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w64 : AMDGPUBuiltin<"_Vector<8, short>(_Vector<16, short>, _Vector<16, short>, _Vector<8, short>, _Constant bool)", [Const], "wmma-256b-insts,wavefrontsize64"> {
let Documentation = [DocWMMAOpselTied_16x16x16_GFX11];
let ArgNames = ["a", "b", "c", "opsel"];
}
-def __builtin_amdgcn_wmma_i32_16x16x16_iu8_w64 : AMDGPUBuiltin<"_ExtVector<4, int>(_Constant bool, _ExtVector<4, int>, _Constant bool, _ExtVector<4, int>, _ExtVector<4, int>, _Constant bool)", [Const], "wmma-256b-insts,wavefrontsize64"> {
+def __builtin_amdgcn_wmma_i32_16x16x16_iu8_w64 : AMDGPUBuiltin<"_Vector<4, int>(_Constant bool, _Vector<4, int>, _Constant bool, _Vector<4, int>, _Vector<4, int>, _Constant bool)", [Const], "wmma-256b-insts,wavefrontsize64"> {
let Documentation = [DocWMMAIU_16x16x16_GFX11];
let ArgNames = ["a_sign", "a", "b_sign", "b", "c", "clamp"];
}
-def __builtin_amdgcn_wmma_i32_16x16x16_iu4_w64 : AMDGPUBuiltin<"_ExtVector<4, int>(_Constant bool, _ExtVector<2, int>, _Constant bool, _ExtVector<2, int>, _ExtVector<4, int>, _Constant bool)", [Const], "wmma-256b-insts,wavefrontsize64"> {
+def __builtin_amdgcn_wmma_i32_16x16x16_iu4_w64 : AMDGPUBuiltin<"_Vector<4, int>(_Constant bool, _Vector<2, int>, _Constant bool, _Vector<2, int>, _Vector<4, int>, _Constant bool)", [Const], "wmma-256b-insts,wavefrontsize64"> {
let Documentation = [DocWMMAIU_16x16x16_GFX11];
let ArgNames = ["a_sign", "a", "b_sign", "b", "c", "clamp"];
}
@@ -566,7 +566,7 @@ def __builtin_amdgcn_wmma_i32_16x16x16_iu4_w64 : AMDGPUBuiltin<"_ExtVector<4, in
def __builtin_amdgcn_s_sendmsg_rtn : AMDGPUBuiltin<"unsigned int(_Constant unsigned int)", [], "gfx11-insts">;
def __builtin_amdgcn_s_sendmsg_rtnl : AMDGPUBuiltin<"uint64_t(_Constant unsigned int)", [], "gfx11-insts">;
-def __builtin_amdgcn_ds_bvh_stack_rtn : AMDGPUBuiltin<"_ExtVector<2, unsigned int>(unsigned int, unsigned int, _ExtVector<4, unsigned int>, _Constant int)", [], "gfx11-insts">;
+def __builtin_amdgcn_ds_bvh_stack_rtn : AMDGPUBuiltin<"_Vector<2, unsigned int>(unsigned int, unsigned int, _Vector<4, unsigned int>, _Constant int)", [], "gfx11-insts">;
//===----------------------------------------------------------------------===//
// Special builtins.
@@ -639,67 +639,67 @@ def __builtin_r600_recipsqrt_ieeef : AMDGPUBuiltin<"float(float)", [Const]>;
// MFMA builtins.
//===----------------------------------------------------------------------===//
-def __builtin_amdgcn_mfma_f32_32x32x1f32 : AMDGPUBuiltin<"_ExtVector<32, float>(float, float, _ExtVector<32, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
-def __builtin_amdgcn_mfma_f32_16x16x1f32 : AMDGPUBuiltin<"_ExtVector<16, float>(float, float, _ExtVector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
-def __builtin_amdgcn_mfma_f32_4x4x1f32 : AMDGPUBuiltin<"_ExtVector<4, float>(float, float, _ExtVector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
-def __builtin_amdgcn_mfma_f32_32x32x2f32 : AMDGPUBuiltin<"_ExtVector<16, float>(float, float, _ExtVector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
-def __builtin_amdgcn_mfma_f32_16x16x4f32 : AMDGPUBuiltin<"_ExtVector<4, float>(float, float, _ExtVector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
-def __builtin_amdgcn_mfma_f32_32x32x4f16 : AMDGPUBuiltin<"_ExtVector<32, float>(_ExtVector<4, _Float16>, _ExtVector<4, _Float16>, _ExtVector<32, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
-def __builtin_amdgcn_mfma_f32_16x16x4f16 : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<4, _Float16>, _ExtVector<4, _Float16>, _ExtVector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
-def __builtin_amdgcn_mfma_f32_4x4x4f16 : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<4, _Float16>, _ExtVector<4, _Float16>, _ExtVector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
-def __builtin_amdgcn_mfma_f32_32x32x8f16 : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<4, _Float16>, _ExtVector<4, _Float16>, _ExtVector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
-def __builtin_amdgcn_mfma_f32_16x16x16f16 : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<4, _Float16>, _ExtVector<4, _Float16>, _ExtVector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
-def __builtin_amdgcn_mfma_i32_32x32x4i8 : AMDGPUBuiltin<"_ExtVector<32, int>(int, int, _ExtVector<32, int>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
-def __builtin_amdgcn_mfma_i32_16x16x4i8 : AMDGPUBuiltin<"_ExtVector<16, int>(int, int, _ExtVector<16, int>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
-def __builtin_amdgcn_mfma_i32_4x4x4i8 : AMDGPUBuiltin<"_ExtVector<4, int>(int, int, _ExtVector<4, int>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
-def __builtin_amdgcn_mfma_i32_32x32x8i8 : AMDGPUBuiltin<"_ExtVector<16, int>(int, int, _ExtVector<16, int>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
-def __builtin_amdgcn_mfma_i32_16x16x16i8 : AMDGPUBuiltin<"_ExtVector<4, int>(int, int, _ExtVector<4, int>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
-def __builtin_amdgcn_mfma_f32_32x32x2bf16 : AMDGPUBuiltin<"_ExtVector<32, float>(_ExtVector<2, short>, _ExtVector<2, short>, _ExtVector<32, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
-def __builtin_amdgcn_mfma_f32_16x16x2bf16 : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<2, short>, _ExtVector<2, short>, _ExtVector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
-def __builtin_amdgcn_mfma_f32_4x4x2bf16 : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<2, short>, _ExtVector<2, short>, _ExtVector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
-def __builtin_amdgcn_mfma_f32_32x32x4bf16 : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<2, short>, _ExtVector<2, short>, _ExtVector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
-def __builtin_amdgcn_mfma_f32_16x16x8bf16 : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<2, short>, _ExtVector<2, short>, _ExtVector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
-
-def __builtin_amdgcn_mfma_f32_32x32x4bf16_1k : AMDGPUBuiltin<"_ExtVector<32, float>(_ExtVector<4, short>, _ExtVector<4, short>, _ExtVector<32, float>, _Constant int, _Constant int, _Constant int)", [Const], "gfx90a-insts">;
-def __builtin_amdgcn_mfma_f32_16x16x4bf16_1k : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<4, short>, _ExtVector<4, short>, _ExtVector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "gfx90a-insts">;
-def __builtin_amdgcn_mfma_f32_4x4x4bf16_1k : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<4, short>, _ExtVector<4, short>, _ExtVector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "gfx90a-insts">;
-def __builtin_amdgcn_mfma_f32_32x32x8bf16_1k : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<4, short>, _ExtVector<4, short>, _ExtVector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "gfx90a-insts">;
-def __builtin_amdgcn_mfma_f32_16x16x16bf16_1k : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<4, short>, _ExtVector<4, short>, _ExtVector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "gfx90a-insts">;
-def __builtin_amdgcn_mfma_f64_16x16x4f64 : AMDGPUBuiltin<"_ExtVector<4, double>(double, double, _ExtVector<4, double>, _Constant int, _Constant int, _Constant int)", [Const], "gfx90a-insts">;
+def __builtin_amdgcn_mfma_f32_32x32x1f32 : AMDGPUBuiltin<"_Vector<32, float>(float, float, _Vector<32, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
+def __builtin_amdgcn_mfma_f32_16x16x1f32 : AMDGPUBuiltin<"_Vector<16, float>(float, float, _Vector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
+def __builtin_amdgcn_mfma_f32_4x4x1f32 : AMDGPUBuiltin<"_Vector<4, float>(float, float, _Vector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
+def __builtin_amdgcn_mfma_f32_32x32x2f32 : AMDGPUBuiltin<"_Vector<16, float>(float, float, _Vector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
+def __builtin_amdgcn_mfma_f32_16x16x4f32 : AMDGPUBuiltin<"_Vector<4, float>(float, float, _Vector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
+def __builtin_amdgcn_mfma_f32_32x32x4f16 : AMDGPUBuiltin<"_Vector<32, float>(_Vector<4, _Float16>, _Vector<4, _Float16>, _Vector<32, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
+def __builtin_amdgcn_mfma_f32_16x16x4f16 : AMDGPUBuiltin<"_Vector<16, float>(_Vector<4, _Float16>, _Vector<4, _Float16>, _Vector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
+def __builtin_amdgcn_mfma_f32_4x4x4f16 : AMDGPUBuiltin<"_Vector<4, float>(_Vector<4, _Float16>, _Vector<4, _Float16>, _Vector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
+def __builtin_amdgcn_mfma_f32_32x32x8f16 : AMDGPUBuiltin<"_Vector<16, float>(_Vector<4, _Float16>, _Vector<4, _Float16>, _Vector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
+def __builtin_amdgcn_mfma_f32_16x16x16f16 : AMDGPUBuiltin<"_Vector<4, float>(_Vector<4, _Float16>, _Vector<4, _Float16>, _Vector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
+def __builtin_amdgcn_mfma_i32_32x32x4i8 : AMDGPUBuiltin<"_Vector<32, int>(int, int, _Vector<32, int>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
+def __builtin_amdgcn_mfma_i32_16x16x4i8 : AMDGPUBuiltin<"_Vector<16, int>(int, int, _Vector<16, int>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
+def __builtin_amdgcn_mfma_i32_4x4x4i8 : AMDGPUBuiltin<"_Vector<4, int>(int, int, _Vector<4, int>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
+def __builtin_amdgcn_mfma_i32_32x32x8i8 : AMDGPUBuiltin<"_Vector<16, int>(int, int, _Vector<16, int>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
+def __builtin_amdgcn_mfma_i32_16x16x16i8 : AMDGPUBuiltin<"_Vector<4, int>(int, int, _Vector<4, int>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
+def __builtin_amdgcn_mfma_f32_32x32x2bf16 : AMDGPUBuiltin<"_Vector<32, float>(_Vector<2, short>, _Vector<2, short>, _Vector<32, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
+def __builtin_amdgcn_mfma_f32_16x16x2bf16 : AMDGPUBuiltin<"_Vector<16, float>(_Vector<2, short>, _Vector<2, short>, _Vector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
+def __builtin_amdgcn_mfma_f32_4x4x2bf16 : AMDGPUBuiltin<"_Vector<4, float>(_Vector<2, short>, _Vector<2, short>, _Vector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
+def __builtin_amdgcn_mfma_f32_32x32x4bf16 : AMDGPUBuiltin<"_Vector<16, float>(_Vector<2, short>, _Vector<2, short>, _Vector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
+def __builtin_amdgcn_mfma_f32_16x16x8bf16 : AMDGPUBuiltin<"_Vector<4, float>(_Vector<2, short>, _Vector<2, short>, _Vector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
+
+def __builtin_amdgcn_mfma_f32_32x32x4bf16_1k : AMDGPUBuiltin<"_Vector<32, float>(_Vector<4, short>, _Vector<4, short>, _Vector<32, float>, _Constant int, _Constant int, _Constant int)", [Const], "gfx90a-insts">;
+def __builtin_amdgcn_mfma_f32_16x16x4bf16_1k : AMDGPUBuiltin<"_Vector<16, float>(_Vector<4, short>, _Vector<4, short>, _Vector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "gfx90a-insts">;
+def __builtin_amdgcn_mfma_f32_4x4x4bf16_1k : AMDGPUBuiltin<"_Vector<4, float>(_Vector<4, short>, _Vector<4, short>, _Vector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "gfx90a-insts">;
+def __builtin_amdgcn_mfma_f32_32x32x8bf16_1k : AMDGPUBuiltin<"_Vector<16, float>(_Vector<4, short>, _Vector<4, short>, _Vector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "gfx90a-insts">;
+def __builtin_amdgcn_mfma_f32_16x16x16bf16_1k : AMDGPUBuiltin<"_Vector<4, float>(_Vector<4, short>, _Vector<4, short>, _Vector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "gfx90a-insts">;
+def __builtin_amdgcn_mfma_f64_16x16x4f64 : AMDGPUBuiltin<"_Vector<4, double>(double, double, _Vector<4, double>, _Constant int, _Constant int, _Constant int)", [Const], "gfx90a-insts">;
def __builtin_amdgcn_mfma_f64_4x4x4f64 : AMDGPUBuiltin<"double(double, double, double, _Constant int, _Constant int, _Constant int)", [Const], "gfx90a-insts">;
-def __builtin_amdgcn_mfma_i32_16x16x32_i8 : AMDGPUBuiltin<"_ExtVector<4, int>(int64_t, int64_t, _ExtVector<4, int>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
-def __builtin_amdgcn_mfma_i32_32x32x16_i8 : AMDGPUBuiltin<"_ExtVector<16, int>(int64_t, int64_t, _ExtVector<16, int>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
-def __builtin_amdgcn_mfma_f32_16x16x8_xf32 : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<2, float>, _ExtVector<2, float>, _ExtVector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
-def __builtin_amdgcn_mfma_f32_32x32x4_xf32 : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<2, float>, _ExtVector<2, float>, _ExtVector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
-def __builtin_amdgcn_mfma_f32_16x16x32_bf8_bf8 : AMDGPUBuiltin<"_ExtVector<4, float>(int64_t, int64_t, _ExtVector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "fp8-insts">;
-def __builtin_amdgcn_mfma_f32_16x16x32_bf8_fp8 : AMDGPUBuiltin<"_ExtVector<4, float>(int64_t, int64_t, _ExtVector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "fp8-insts">;
-def __builtin_amdgcn_mfma_f32_16x16x32_fp8_bf8 : AMDGPUBuiltin<"_ExtVector<4, float>(int64_t, int64_t, _ExtVector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "fp8-insts">;
-def __builtin_amdgcn_mfma_f32_16x16x32_fp8_fp8 : AMDGPUBuiltin<"_ExtVector<4, float>(int64_t, int64_t, _ExtVector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "fp8-insts">;
-def __builtin_amdgcn_mfma_f32_32x32x16_bf8_bf8 : AMDGPUBuiltin<"_ExtVector<16, float>(int64_t, int64_t, _ExtVector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "fp8-insts">;
-def __builtin_amdgcn_mfma_f32_32x32x16_bf8_fp8 : AMDGPUBuiltin<"_ExtVector<16, float>(int64_t, int64_t, _ExtVector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "fp8-insts">;
-def __builtin_amdgcn_mfma_f32_32x32x16_fp8_bf8 : AMDGPUBuiltin<"_ExtVector<16, float>(int64_t, int64_t, _ExtVector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "fp8-insts">;
-def __builtin_amdgcn_mfma_f32_32x32x16_fp8_fp8 : AMDGPUBuiltin<"_ExtVector<16, float>(int64_t, int64_t, _ExtVector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "fp8-insts">;
-def __builtin_amdgcn_smfmac_f32_16x16x32_f16 : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<4, _Float16>, _ExtVector<8, _Float16>, _ExtVector<4, float>, int, _Constant int, _Constant int)", [Const], "gfx940-insts">;
-def __builtin_amdgcn_smfmac_f32_32x32x16_f16 : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<4, _Float16>, _ExtVector<8, _Float16>, _ExtVector<16, float>, int, _Constant int, _Constant int)", [Const], "gfx940-insts">;
-def __builtin_amdgcn_smfmac_f32_16x16x32_bf16 : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<4, short>, _ExtVector<8, short>, _ExtVector<4, float>, int, _Constant int, _Constant int)", [Const], "gfx940-insts">;
-def __builtin_amdgcn_smfmac_f32_32x32x16_bf16 : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<4, short>, _ExtVector<8, short>, _ExtVector<16, float>, int, _Constant int, _Constant int)", [Const], "gfx940-insts">;
-def __builtin_amdgcn_smfmac_i32_16x16x64_i8 : AMDGPUBuiltin<"_ExtVector<4, int>(_ExtVector<2, int>, _ExtVector<4, int>, _ExtVector<4, int>, int, _Constant int, _Constant int)", [Const], "gfx940-insts">;
-def __builtin_amdgcn_smfmac_i32_32x32x32_i8 : AMDGPUBuiltin<"_ExtVector<16, int>(_ExtVector<2, int>, _ExtVector<4, int>, _ExtVector<16, int>, int, _Constant int, _Constant int)", [Const], "gfx940-insts">;
-def __builtin_amdgcn_smfmac_f32_16x16x64_bf8_bf8 : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<2, int>, _ExtVector<4, int>, _ExtVector<4, float>, int, _Constant int, _Constant int)", [Const], "fp8-insts">;
-def __builtin_amdgcn_smfmac_f32_16x16x64_bf8_fp8 : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<2, int>, _ExtVector<4, int>, _ExtVector<4, float>, int, _Constant int, _Constant int)", [Const], "fp8-insts">;
-def __builtin_amdgcn_smfmac_f32_16x16x64_fp8_bf8 : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<2, int>, _ExtVector<4, int>, _ExtVector<4, float>, int, _Constant int, _Constant int)", [Const], "fp8-insts">;
-def __builtin_amdgcn_smfmac_f32_16x16x64_fp8_fp8 : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<2, int>, _ExtVector<4, int>, _ExtVector<4, float>, int, _Constant int, _Constant int)", [Const], "fp8-insts">;
-def __builtin_amdgcn_smfmac_f32_32x32x32_bf8_bf8 : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<2, int>, _ExtVector<4, int>, _ExtVector<16, float>, int, _Constant int, _Constant int)", [Const], "fp8-insts">;
-def __builtin_amdgcn_smfmac_f32_32x32x32_bf8_fp8 : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<2, int>, _ExtVector<4, int>, _ExtVector<16, float>, int, _Constant int, _Constant int)", [Const], "fp8-insts">;
-def __builtin_amdgcn_smfmac_f32_32x32x32_fp8_bf8 : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<2, int>, _ExtVector<4, int>, _ExtVector<16, float>, int, _Constant int, _Constant int)", [Const], "fp8-insts">;
-def __builtin_amdgcn_smfmac_f32_32x32x32_fp8_fp8 : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<2, int>, _ExtVector<4, int>, _ExtVector<16, float>, int, _Constant int, _Constant int)", [Const], "fp8-insts">;
+def __builtin_amdgcn_mfma_i32_16x16x32_i8 : AMDGPUBuiltin<"_Vector<4, int>(int64_t, int64_t, _Vector<4, int>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
+def __builtin_amdgcn_mfma_i32_32x32x16_i8 : AMDGPUBuiltin<"_Vector<16, int>(int64_t, int64_t, _Vector<16, int>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
+def __builtin_amdgcn_mfma_f32_16x16x8_xf32 : AMDGPUBuiltin<"_Vector<4, float>(_Vector<2, float>, _Vector<2, float>, _Vector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
+def __builtin_amdgcn_mfma_f32_32x32x4_xf32 : AMDGPUBuiltin<"_Vector<16, float>(_Vector<2, float>, _Vector<2, float>, _Vector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "mai-insts">;
+def __builtin_amdgcn_mfma_f32_16x16x32_bf8_bf8 : AMDGPUBuiltin<"_Vector<4, float>(int64_t, int64_t, _Vector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "fp8-insts">;
+def __builtin_amdgcn_mfma_f32_16x16x32_bf8_fp8 : AMDGPUBuiltin<"_Vector<4, float>(int64_t, int64_t, _Vector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "fp8-insts">;
+def __builtin_amdgcn_mfma_f32_16x16x32_fp8_bf8 : AMDGPUBuiltin<"_Vector<4, float>(int64_t, int64_t, _Vector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "fp8-insts">;
+def __builtin_amdgcn_mfma_f32_16x16x32_fp8_fp8 : AMDGPUBuiltin<"_Vector<4, float>(int64_t, int64_t, _Vector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "fp8-insts">;
+def __builtin_amdgcn_mfma_f32_32x32x16_bf8_bf8 : AMDGPUBuiltin<"_Vector<16, float>(int64_t, int64_t, _Vector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "fp8-insts">;
+def __builtin_amdgcn_mfma_f32_32x32x16_bf8_fp8 : AMDGPUBuiltin<"_Vector<16, float>(int64_t, int64_t, _Vector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "fp8-insts">;
+def __builtin_amdgcn_mfma_f32_32x32x16_fp8_bf8 : AMDGPUBuiltin<"_Vector<16, float>(int64_t, int64_t, _Vector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "fp8-insts">;
+def __builtin_amdgcn_mfma_f32_32x32x16_fp8_fp8 : AMDGPUBuiltin<"_Vector<16, float>(int64_t, int64_t, _Vector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "fp8-insts">;
+def __builtin_amdgcn_smfmac_f32_16x16x32_f16 : AMDGPUBuiltin<"_Vector<4, float>(_Vector<4, _Float16>, _Vector<8, _Float16>, _Vector<4, float>, int, _Constant int, _Constant int)", [Const], "gfx940-insts">;
+def __builtin_amdgcn_smfmac_f32_32x32x16_f16 : AMDGPUBuiltin<"_Vector<16, float>(_Vector<4, _Float16>, _Vector<8, _Float16>, _Vector<16, float>, int, _Constant int, _Constant int)", [Const], "gfx940-insts">;
+def __builtin_amdgcn_smfmac_f32_16x16x32_bf16 : AMDGPUBuiltin<"_Vector<4, float>(_Vector<4, short>, _Vector<8, short>, _Vector<4, float>, int, _Constant int, _Constant int)", [Const], "gfx940-insts">;
+def __builtin_amdgcn_smfmac_f32_32x32x16_bf16 : AMDGPUBuiltin<"_Vector<16, float>(_Vector<4, short>, _Vector<8, short>, _Vector<16, float>, int, _Constant int, _Constant int)", [Const], "gfx940-insts">;
+def __builtin_amdgcn_smfmac_i32_16x16x64_i8 : AMDGPUBuiltin<"_Vector<4, int>(_Vector<2, int>, _Vector<4, int>, _Vector<4, int>, int, _Constant int, _Constant int)", [Const], "gfx940-insts">;
+def __builtin_amdgcn_smfmac_i32_32x32x32_i8 : AMDGPUBuiltin<"_Vector<16, int>(_Vector<2, int>, _Vector<4, int>, _Vector<16, int>, int, _Constant int, _Constant int)", [Const], "gfx940-insts">;
+def __builtin_amdgcn_smfmac_f32_16x16x64_bf8_bf8 : AMDGPUBuiltin<"_Vector<4, float>(_Vector<2, int>, _Vector<4, int>, _Vector<4, float>, int, _Constant int, _Constant int)", [Const], "fp8-insts">;
+def __builtin_amdgcn_smfmac_f32_16x16x64_bf8_fp8 : AMDGPUBuiltin<"_Vector<4, float>(_Vector<2, int>, _Vector<4, int>, _Vector<4, float>, int, _Constant int, _Constant int)", [Const], "fp8-insts">;
+def __builtin_amdgcn_smfmac_f32_16x16x64_fp8_bf8 : AMDGPUBuiltin<"_Vector<4, float>(_Vector<2, int>, _Vector<4, int>, _Vector<4, float>, int, _Constant int, _Constant int)", [Const], "fp8-insts">;
+def __builtin_amdgcn_smfmac_f32_16x16x64_fp8_fp8 : AMDGPUBuiltin<"_Vector<4, float>(_Vector<2, int>, _Vector<4, int>, _Vector<4, float>, int, _Constant int, _Constant int)", [Const], "fp8-insts">;
+def __builtin_amdgcn_smfmac_f32_32x32x32_bf8_bf8 : AMDGPUBuiltin<"_Vector<16, float>(_Vector<2, int>, _Vector<4, int>, _Vector<16, float>, int, _Constant int, _Constant int)", [Const], "fp8-insts">;
+def __builtin_amdgcn_smfmac_f32_32x32x32_bf8_fp8 : AMDGPUBuiltin<"_Vector<16, float>(_Vector<2, int>, _Vector<4, int>, _Vector<16, float>, int, _Constant int, _Constant int)", [Const], "fp8-insts">;
+def __builtin_amdgcn_smfmac_f32_32x32x32_fp8_bf8 : AMDGPUBuiltin<"_Vector<16, float>(_Vector<2, int>, _Vector<4, int>, _Vector<16, float>, int, _Constant int, _Constant int)", [Const], "fp8-insts">;
+def __builtin_amdgcn_smfmac_f32_32x32x32_fp8_fp8 : AMDGPUBuiltin<"_Vector<16, float>(_Vector<2, int>, _Vector<4, int>, _Vector<16, float>, int, _Constant int, _Constant int)", [Const], "fp8-insts">;
def __builtin_amdgcn_cvt_f32_bf8 : AMDGPUBuiltin<"float(int, _Constant int)", [Const], "fp8-conversion-insts">;
def __builtin_amdgcn_cvt_f32_fp8 : AMDGPUBuiltin<"float(int, _Constant int)", [Const], "fp8-conversion-insts">;
def __builtin_amdgcn_cvt_f32_fp8_e5m3 : AMDGPUBuiltin<"float(int, _Constant int)", [Const], "fp8e5m3-insts">;
-def __builtin_amdgcn_cvt_pk_f32_bf8 : AMDGPUBuiltin<"_ExtVector<2, float>(int, _Constant bool)", [Const], "fp8-conversion-insts">;
-def __builtin_amdgcn_cvt_pk_f32_fp8 : AMDGPUBuiltin<"_ExtVector<2, float>(int, _Constant bool)", [Const], "fp8-conversion-insts">;
+def __builtin_amdgcn_cvt_pk_f32_bf8 : AMDGPUBuiltin<"_Vector<2, float>(int, _Constant bool)", [Const], "fp8-conversion-insts">;
+def __builtin_amdgcn_cvt_pk_f32_fp8 : AMDGPUBuiltin<"_Vector<2, float>(int, _Constant bool)", [Const], "fp8-conversion-insts">;
def __builtin_amdgcn_cvt_pk_bf8_f32 : AMDGPUBuiltin<"int(float, float, int, _Constant bool)", [Const], "fp8-conversion-insts">;
def __builtin_amdgcn_cvt_pk_fp8_f32 : AMDGPUBuiltin<"int(float, float, int, _Constant bool)", [Const], "fp8-conversion-insts">;
def __builtin_amdgcn_cvt_sr_bf8_f32 : AMDGPUBuiltin<"int(float, int, int, _Constant int)", [Const], "fp8-conversion-insts">;
@@ -708,46 +708,46 @@ def __builtin_amdgcn_cvt_sr_fp8_f32 : AMDGPUBuiltin<"int(float, int, int, _Const
//===----------------------------------------------------------------------===//
// GFX950 only builtins.
//===----------------------------------------------------------------------===//
-def __builtin_amdgcn_mfma_scale_f32_16x16x128_f8f6f4 : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<8, int32_t>, _ExtVector<8, int32_t>, _ExtVector<4, float>, _Constant int, _Constant int, _Constant int, int, _Constant int, int)", [Const], "gfx950-insts">;
-def __builtin_amdgcn_mfma_scale_f32_32x32x64_f8f6f4 : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<8, int32_t>, _ExtVector<8, int32_t>, _ExtVector<16, float>, _Constant int, _Constant int, _Constant int, int, _Constant int, int)", [Const], "gfx950-insts">;
-
-def __builtin_amdgcn_mfma_f32_16x16x32_f16 : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<8, _Float16>, _ExtVector<8, _Float16>, _ExtVector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
-def __builtin_amdgcn_mfma_f32_16x16x32_bf16 : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<8, __bf16>, _ExtVector<8, __bf16>, _ExtVector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
-def __builtin_amdgcn_mfma_f32_32x32x16_f16 : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<8, _Float16>, _ExtVector<8, _Float16>, _ExtVector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
-def __builtin_amdgcn_mfma_f32_32x32x16_bf16 : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<8, __bf16>, _ExtVector<8, __bf16>, _ExtVector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
-def __builtin_amdgcn_mfma_i32_16x16x64_i8 : AMDGPUBuiltin<"_ExtVector<4, int>(_ExtVector<4, int>, _ExtVector<4, int>, _ExtVector<4, int>, _Constant int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
-def __builtin_amdgcn_mfma_i32_32x32x32_i8 : AMDGPUBuiltin<"_ExtVector<16, int>(_ExtVector<4, int>, _ExtVector<4, int>, _ExtVector<16, int>, _Constant int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
-
-def __builtin_amdgcn_smfmac_f32_16x16x64_f16 : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<8, _Float16>, _ExtVector<16, _Float16>, _ExtVector<4, float>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
-def __builtin_amdgcn_smfmac_f32_32x32x32_f16 : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<8, _Float16>, _ExtVector<16, _Float16>, _ExtVector<16, float>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
-def __builtin_amdgcn_smfmac_f32_16x16x64_bf16 : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<8, __bf16>, _ExtVector<16, __bf16>, _ExtVector<4, float>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
-def __builtin_amdgcn_smfmac_f32_32x32x32_bf16 : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<8, __bf16>, _ExtVector<16, __bf16>, _ExtVector<16, float>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
-def __builtin_amdgcn_smfmac_i32_16x16x128_i8 : AMDGPUBuiltin<"_ExtVector<4, int>(_ExtVector<4, int>, _ExtVector<8, int>, _ExtVector<4, int>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
-def __builtin_amdgcn_smfmac_i32_32x32x64_i8 : AMDGPUBuiltin<"_ExtVector<16, int>(_ExtVector<4, int>, _ExtVector<8, int>, _ExtVector<16, int>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
-def __builtin_amdgcn_smfmac_f32_16x16x128_bf8_bf8 : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<4, int>, _ExtVector<8, int>, _ExtVector<4, float>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
-def __builtin_amdgcn_smfmac_f32_16x16x128_bf8_fp8 : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<4, int>, _ExtVector<8, int>, _ExtVector<4, float>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
-def __builtin_amdgcn_smfmac_f32_16x16x128_fp8_bf8 : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<4, int>, _ExtVector<8, int>, _ExtVector<4, float>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
-def __builtin_amdgcn_smfmac_f32_16x16x128_fp8_fp8 : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<4, int>, _ExtVector<8, int>, _ExtVector<4, float>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
-def __builtin_amdgcn_smfmac_f32_32x32x64_bf8_bf8 : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<4, int>, _ExtVector<8, int>, _ExtVector<16, float>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
-def __builtin_amdgcn_smfmac_f32_32x32x64_bf8_fp8 : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<4, int>, _ExtVector<8, int>, _ExtVector<16, float>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
-def __builtin_amdgcn_smfmac_f32_32x32x64_fp8_bf8 : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<4, int>, _ExtVector<8, int>, _ExtVector<16, float>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
-def __builtin_amdgcn_smfmac_f32_32x32x64_fp8_fp8 : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<4, int>, _ExtVector<8, int>, _ExtVector<16, float>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
-
-def __builtin_amdgcn_permlane16_swap : AMDGPUBuiltin<"_ExtVector<2, unsigned int>(unsigned int, unsigned int, _Constant bool, _Constant bool)", [Const], "permlane16-swap">;
-def __builtin_amdgcn_permlane32_swap : AMDGPUBuiltin<"_ExtVector<2, unsigned int>(unsigned int, unsigned int, _Constant bool, _Constant bool)", [Const], "permlane32-swap">;
-
-def __builtin_amdgcn_ds_read_tr4_b64_v2i32 : AMDGPUBuiltin<"_ExtVector<2, int>(_ExtVector<2, int> address_space<3> *)", [Const], "gfx950-insts">;
-def __builtin_amdgcn_ds_read_tr6_b96_v3i32 : AMDGPUBuiltin<"_ExtVector<3, int>(_ExtVector<3, int> address_space<3> *)", [Const], "gfx950-insts">;
-def __builtin_amdgcn_ds_read_tr8_b64_v2i32 : AMDGPUBuiltin<"_ExtVector<2, int>(_ExtVector<2, int> address_space<3> *)", [Const], "gfx950-insts">;
-def __builtin_amdgcn_ds_read_tr16_b64_v4i16 : AMDGPUBuiltin<"_ExtVector<4, short>(_ExtVector<4, short> address_space<3> *)", [Const], "gfx950-insts">;
-def __builtin_amdgcn_ds_read_tr16_b64_v4f16 : AMDGPUBuiltin<"_ExtVector<4, __fp16>(_ExtVector<4, __fp16> address_space<3> *)", [Const], "gfx950-insts">;
-def __builtin_amdgcn_ds_read_tr16_b64_v4bf16 : AMDGPUBuiltin<"_ExtVector<4, __bf16>(_ExtVector<4, __bf16> address_space<3> *)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_mfma_scale_f32_16x16x128_f8f6f4 : AMDGPUBuiltin<"_Vector<4, float>(_Vector<8, int32_t>, _Vector<8, int32_t>, _Vector<4, float>, _Constant int, _Constant int, _Constant int, int, _Constant int, int)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_mfma_scale_f32_32x32x64_f8f6f4 : AMDGPUBuiltin<"_Vector<16, float>(_Vector<8, int32_t>, _Vector<8, int32_t>, _Vector<16, float>, _Constant int, _Constant int, _Constant int, int, _Constant int, int)", [Const], "gfx950-insts">;
+
+def __builtin_amdgcn_mfma_f32_16x16x32_f16 : AMDGPUBuiltin<"_Vector<4, float>(_Vector<8, _Float16>, _Vector<8, _Float16>, _Vector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_mfma_f32_16x16x32_bf16 : AMDGPUBuiltin<"_Vector<4, float>(_Vector<8, __bf16>, _Vector<8, __bf16>, _Vector<4, float>, _Constant int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_mfma_f32_32x32x16_f16 : AMDGPUBuiltin<"_Vector<16, float>(_Vector<8, _Float16>, _Vector<8, _Float16>, _Vector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_mfma_f32_32x32x16_bf16 : AMDGPUBuiltin<"_Vector<16, float>(_Vector<8, __bf16>, _Vector<8, __bf16>, _Vector<16, float>, _Constant int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_mfma_i32_16x16x64_i8 : AMDGPUBuiltin<"_Vector<4, int>(_Vector<4, int>, _Vector<4, int>, _Vector<4, int>, _Constant int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_mfma_i32_32x32x32_i8 : AMDGPUBuiltin<"_Vector<16, int>(_Vector<4, int>, _Vector<4, int>, _Vector<16, int>, _Constant int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
+
+def __builtin_amdgcn_smfmac_f32_16x16x64_f16 : AMDGPUBuiltin<"_Vector<4, float>(_Vector<8, _Float16>, _Vector<16, _Float16>, _Vector<4, float>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_smfmac_f32_32x32x32_f16 : AMDGPUBuiltin<"_Vector<16, float>(_Vector<8, _Float16>, _Vector<16, _Float16>, _Vector<16, float>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_smfmac_f32_16x16x64_bf16 : AMDGPUBuiltin<"_Vector<4, float>(_Vector<8, __bf16>, _Vector<16, __bf16>, _Vector<4, float>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_smfmac_f32_32x32x32_bf16 : AMDGPUBuiltin<"_Vector<16, float>(_Vector<8, __bf16>, _Vector<16, __bf16>, _Vector<16, float>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_smfmac_i32_16x16x128_i8 : AMDGPUBuiltin<"_Vector<4, int>(_Vector<4, int>, _Vector<8, int>, _Vector<4, int>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_smfmac_i32_32x32x64_i8 : AMDGPUBuiltin<"_Vector<16, int>(_Vector<4, int>, _Vector<8, int>, _Vector<16, int>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_smfmac_f32_16x16x128_bf8_bf8 : AMDGPUBuiltin<"_Vector<4, float>(_Vector<4, int>, _Vector<8, int>, _Vector<4, float>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_smfmac_f32_16x16x128_bf8_fp8 : AMDGPUBuiltin<"_Vector<4, float>(_Vector<4, int>, _Vector<8, int>, _Vector<4, float>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_smfmac_f32_16x16x128_fp8_bf8 : AMDGPUBuiltin<"_Vector<4, float>(_Vector<4, int>, _Vector<8, int>, _Vector<4, float>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_smfmac_f32_16x16x128_fp8_fp8 : AMDGPUBuiltin<"_Vector<4, float>(_Vector<4, int>, _Vector<8, int>, _Vector<4, float>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_smfmac_f32_32x32x64_bf8_bf8 : AMDGPUBuiltin<"_Vector<16, float>(_Vector<4, int>, _Vector<8, int>, _Vector<16, float>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_smfmac_f32_32x32x64_bf8_fp8 : AMDGPUBuiltin<"_Vector<16, float>(_Vector<4, int>, _Vector<8, int>, _Vector<16, float>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_smfmac_f32_32x32x64_fp8_bf8 : AMDGPUBuiltin<"_Vector<16, float>(_Vector<4, int>, _Vector<8, int>, _Vector<16, float>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_smfmac_f32_32x32x64_fp8_fp8 : AMDGPUBuiltin<"_Vector<16, float>(_Vector<4, int>, _Vector<8, int>, _Vector<16, float>, int, _Constant int, _Constant int)", [Const], "gfx950-insts">;
+
+def __builtin_amdgcn_permlane16_swap : AMDGPUBuiltin<"_Vector<2, unsigned int>(unsigned int, unsigned int, _Constant bool, _Constant bool)", [Const], "permlane16-swap">;
+def __builtin_amdgcn_permlane32_swap : AMDGPUBuiltin<"_Vector<2, unsigned int>(unsigned int, unsigned int, _Constant bool, _Constant bool)", [Const], "permlane32-swap">;
+
+def __builtin_amdgcn_ds_read_tr4_b64_v2i32 : AMDGPUBuiltin<"_Vector<2, int>(_Vector<2, int> address_space<3> *)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_ds_read_tr6_b96_v3i32 : AMDGPUBuiltin<"_Vector<3, int>(_Vector<3, int> address_space<3> *)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_ds_read_tr8_b64_v2i32 : AMDGPUBuiltin<"_Vector<2, int>(_Vector<2, int> address_space<3> *)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_ds_read_tr16_b64_v4i16 : AMDGPUBuiltin<"_Vector<4, short>(_Vector<4, short> address_space<3> *)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_ds_read_tr16_b64_v4f16 : AMDGPUBuiltin<"_Vector<4, __fp16>(_Vector<4, __fp16> address_space<3> *)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_ds_read_tr16_b64_v4bf16 : AMDGPUBuiltin<"_Vector<4, __bf16>(_Vector<4, __bf16> address_space<3> *)", [Const], "gfx950-insts">;
def __builtin_amdgcn_ashr_pk_i8_i32 : AMDGPUBuiltin<"unsigned short(unsigned int, unsigned int, unsigned int)", [Const], "ashr-pk-insts">;
def __builtin_amdgcn_ashr_pk_u8_i32 : AMDGPUBuiltin<"unsigned short(unsigned int, unsigned int, unsigned int)", [Const], "ashr-pk-insts">;
-def __builtin_amdgcn_cvt_scalef32_2xpk16_fp6_f32 : AMDGPUBuiltin<"_ExtVector<6, unsigned int>(_ExtVector<16, float>, _ExtVector<16, float>, float)", [Const], "gfx950-insts">;
-def __builtin_amdgcn_cvt_scalef32_2xpk16_bf6_f32 : AMDGPUBuiltin<"_ExtVector<6, unsigned int>(_ExtVector<16, float>, _ExtVector<16, float>, float)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_cvt_scalef32_2xpk16_fp6_f32 : AMDGPUBuiltin<"_Vector<6, unsigned int>(_Vector<16, float>, _Vector<16, float>, float)", [Const], "gfx950-insts">;
+def __builtin_amdgcn_cvt_scalef32_2xpk16_bf6_f32 : AMDGPUBuiltin<"_Vector<6, unsigned int>(_Vector<16, float>, _Vector<16, float>, float)", [Const], "gfx950-insts">;
//===----------------------------------------------------------------------===//
// GFX12+ only builtins.
@@ -772,28 +772,28 @@ def __builtin_amdgcn_s_prefetch_inst : AMDGPUBuiltin<"void(void const *, unsigne
}
def __builtin_amdgcn_s_buffer_prefetch_data : AMDGPUBuiltin<"void(__amdgpu_buffer_rsrc_t, _Constant int, unsigned int)", [Const], "smem-prefetch-insts">;
-def __builtin_amdgcn_global_load_tr_b64_v2i32 : AMDGPUBuiltin<"_ExtVector<2, int>(_ExtVector<2, int> address_space<1> *)", [Const], "gfx12-insts,wavefrontsize32">;
-def __builtin_amdgcn_global_load_tr_b128_v8i16 : AMDGPUBuiltin<"_ExtVector<8, short>(_ExtVector<8, short> address_space<1> *)", [Const], "gfx12-insts,wavefrontsize32">;
-def __builtin_amdgcn_global_load_tr_b128_v8f16 : AMDGPUBuiltin<"_ExtVector<8, __fp16>(_ExtVector<8, __fp16> address_space<1> *)", [Const], "gfx12-insts,wavefrontsize32">;
-def __builtin_amdgcn_global_load_tr_b128_v8bf16 : AMDGPUBuiltin<"_ExtVector<8, __bf16>(_ExtVector<8, __bf16> address_space<1> *)", [Const], "gfx12-insts,wavefrontsize32">;
+def __builtin_amdgcn_global_load_tr_b64_v2i32 : AMDGPUBuiltin<"_Vector<2, int>(_Vector<2, int> address_space<1> *)", [Const], "gfx12-insts,wavefrontsize32">;
+def __builtin_amdgcn_global_load_tr_b128_v8i16 : AMDGPUBuiltin<"_Vector<8, short>(_Vector<8, short> address_space<1> *)", [Const], "gfx12-insts,wavefrontsize32">;
+def __builtin_amdgcn_global_load_tr_b128_v8f16 : AMDGPUBuiltin<"_Vector<8, __fp16>(_Vector<8, __fp16> address_space<1> *)", [Const], "gfx12-insts,wavefrontsize32">;
+def __builtin_amdgcn_global_load_tr_b128_v8bf16 : AMDGPUBuiltin<"_Vector<8, __bf16>(_Vector<8, __bf16> address_space<1> *)", [Const], "gfx12-insts,wavefrontsize32">;
def __builtin_amdgcn_global_load_tr_b64_i32 : AMDGPUBuiltin<"int(int address_space<1> *)", [Const], "gfx12-insts,wavefrontsize64">;
-def __builtin_amdgcn_global_load_tr_b128_v4i16 : AMDGPUBuiltin<"_ExtVector<4, short>(_ExtVector<4, short> address_space<1> *)", [Const], "gfx12-insts,wavefrontsize64">;
-def __builtin_amdgcn_global_load_tr_b128_v4f16 : AMDGPUBuiltin<"_ExtVector<4, __fp16>(_ExtVector<4, __fp16> address_space<1> *)", [Const], "gfx12-insts,wavefrontsize64">;
-def __builtin_amdgcn_global_load_tr_b128_v4bf16 : AMDGPUBuiltin<"_ExtVector<4, __bf16>(_ExtVector<4, __bf16> address_space<1> *)", [Const], "gfx12-insts,wavefrontsize64">;
+def __builtin_amdgcn_global_load_tr_b128_v4i16 : AMDGPUBuiltin<"_Vector<4, short>(_Vector<4, short> address_space<1> *)", [Const], "gfx12-insts,wavefrontsize64">;
+def __builtin_amdgcn_global_load_tr_b128_v4f16 : AMDGPUBuiltin<"_Vector<4, __fp16>(_Vector<4, __fp16> address_space<1> *)", [Const], "gfx12-insts,wavefrontsize64">;
+def __builtin_amdgcn_global_load_tr_b128_v4bf16 : AMDGPUBuiltin<"_Vector<4, __bf16>(_Vector<4, __bf16> address_space<1> *)", [Const], "gfx12-insts,wavefrontsize64">;
def __builtin_amdgcn_ds_bpermute_fi_b32 : AMDGPUBuiltin<"int(int, int)", [Const], "gfx12-insts">;
// For the following two builtins, the second and third return values of the
// intrinsics are returned through the last two pointer-type function arguments.
-def __builtin_amdgcn_image_bvh8_intersect_ray : AMDGPUBuiltin<"_ExtVector<10, unsigned int>(uint64_t, float, unsigned char, _ExtVector<3, float>, _ExtVector<3, float>, unsigned int, _ExtVector<4, unsigned int>, _ExtVector<3, float> *, _ExtVector<3, float> *)", [Const], "gfx12-insts">;
-def __builtin_amdgcn_image_bvh_dual_intersect_ray : AMDGPUBuiltin<"_ExtVector<10, unsigned int>(uint64_t, float, unsigned char, _ExtVector<3, float>, _ExtVector<3, float>, _ExtVector<2, unsigned int>, _ExtVector<4, unsigned int>, _ExtVector<3, float> *, _ExtVector<3, float> *)", [Const], "gfx12-insts">;
+def __builtin_amdgcn_image_bvh8_intersect_ray : AMDGPUBuiltin<"_Vector<10, unsigned int>(uint64_t, float, unsigned char, _Vector<3, float>, _Vector<3, float>, unsigned int, _Vector<4, unsigned int>, _Vector<3, float> *, _Vector<3, float> *)", [Const], "gfx12-insts">;
+def __builtin_amdgcn_image_bvh_dual_intersect_ray : AMDGPUBuiltin<"_Vector<10, unsigned int>(uint64_t, float, unsigned char, _Vector<3, float>, _Vector<3, float>, _Vector<2, unsigned int>, _Vector<4, unsigned int>, _Vector<3, float> *, _Vector<3, float> *)", [Const], "gfx12-insts">;
-def __builtin_amdgcn_ds_bvh_stack_push4_pop1_rtn : AMDGPUBuiltin<"_ExtVector<2, unsigned int>(unsigned int, unsigned int, _ExtVector<4, unsigned int>, _Constant int)", [], "gfx11-insts">;
-def __builtin_amdgcn_ds_bvh_stack_push8_pop1_rtn : AMDGPUBuiltin<"_ExtVector<2, unsigned int>(unsigned int, unsigned int, _ExtVector<8, unsigned int>, _Constant int)", [], "gfx12-insts">;
+def __builtin_amdgcn_ds_bvh_stack_push4_pop1_rtn : AMDGPUBuiltin<"_Vector<2, unsigned int>(unsigned int, unsigned int, _Vector<4, unsigned int>, _Constant int)", [], "gfx11-insts">;
+def __builtin_amdgcn_ds_bvh_stack_push8_pop1_rtn : AMDGPUBuiltin<"_Vector<2, unsigned int>(unsigned int, unsigned int, _Vector<8, unsigned int>, _Constant int)", [], "gfx12-insts">;
// The intrinsic returns {i64, i32}, the builtin returns <2 x i64>.
// The second return value of the intrinsic is zext'ed.
-def __builtin_amdgcn_ds_bvh_stack_push8_pop2_rtn : AMDGPUBuiltin<"_ExtVector<2, uint64_t>(unsigned int, unsigned int, _ExtVector<8, unsigned int>, _Constant int)", [], "gfx12-insts">;
+def __builtin_amdgcn_ds_bvh_stack_push8_pop2_rtn : AMDGPUBuiltin<"_Vector<2, uint64_t>(unsigned int, unsigned int, _Vector<8, unsigned int>, _Constant int)", [], "gfx12-insts">;
//===----------------------------------------------------------------------===//
// GFX1170, GFX12+ only builtins.
@@ -809,160 +809,160 @@ def __builtin_amdgcn_ds_bvh_stack_push8_pop2_rtn : AMDGPUBuiltin<"_ExtVector<2,
// elements. Therefore, we add an "_gfx12" suffix to distinguish them from the
// existing builtins.
//===----------------------------------------------------------------------===//
-def __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<8, _Float16>, _ExtVector<8, _Float16>, _ExtVector<8, float>)", [Const], "wmma-128b-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<8, _Float16>, _Vector<8, _Float16>, _Vector<8, float>)", [Const], "wmma-128b-insts,wavefrontsize32"> {
let Documentation = [DocWMMAF32_16x16x16_GFX12];
let ArgNames = ["a", "b", "c"];
}
-def __builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<8, short>, _ExtVector<8, short>, _ExtVector<8, float>)", [Const], "wmma-128b-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<8, short>, _Vector<8, short>, _Vector<8, float>)", [Const], "wmma-128b-insts,wavefrontsize32"> {
let Documentation = [DocWMMAF32_16x16x16_GFX12];
let ArgNames = ["a", "b", "c"];
}
-def __builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12 : AMDGPUBuiltin<"_ExtVector<8, _Float16>(_ExtVector<8, _Float16>, _ExtVector<8, _Float16>, _ExtVector<8, _Float16>)", [Const], "wmma-128b-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12 : AMDGPUBuiltin<"_Vector<8, _Float16>(_Vector<8, _Float16>, _Vector<8, _Float16>, _Vector<8, _Float16>)", [Const], "wmma-128b-insts,wavefrontsize32"> {
let Documentation = [DocWMMAHalf_16x16x16_GFX12];
let ArgNames = ["a", "b", "c"];
}
-def __builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32_gfx12 : AMDGPUBuiltin<"_ExtVector<8, short>(_ExtVector<8, short>, _ExtVector<8, short>, _ExtVector<8, short>)", [Const], "wmma-128b-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32_gfx12 : AMDGPUBuiltin<"_Vector<8, short>(_Vector<8, short>, _Vector<8, short>, _Vector<8, short>)", [Const], "wmma-128b-insts,wavefrontsize32"> {
let Documentation = [DocWMMAHalf_16x16x16_GFX12];
let ArgNames = ["a", "b", "c"];
}
-def __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12 : AMDGPUBuiltin<"_ExtVector<8, int>(_Constant bool, _ExtVector<2, int>, _Constant bool, _ExtVector<2, int>, _ExtVector<8, int>, _Constant bool)", [Const], "wmma-128b-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12 : AMDGPUBuiltin<"_Vector<8, int>(_Constant bool, _Vector<2, int>, _Constant bool, _Vector<2, int>, _Vector<8, int>, _Constant bool)", [Const], "wmma-128b-insts,wavefrontsize32"> {
let Documentation = [DocWMMAIU_16x16x16_GFX12];
let ArgNames = ["a_sign", "a", "b_sign", "b", "c", "clamp"];
}
-def __builtin_amdgcn_wmma_i32_16x16x16_iu4_w32_gfx12 : AMDGPUBuiltin<"_ExtVector<8, int>(_Constant bool, int, _Constant bool, int, _ExtVector<8, int>, _Constant bool)", [Const], "wmma-128b-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_i32_16x16x16_iu4_w32_gfx12 : AMDGPUBuiltin<"_Vector<8, int>(_Constant bool, int, _Constant bool, int, _Vector<8, int>, _Constant bool)", [Const], "wmma-128b-insts,wavefrontsize32"> {
let Documentation = [DocWMMAIU_16x16x16_GFX12];
let ArgNames = ["a_sign", "a", "b_sign", "b", "c", "clamp"];
}
// These are gfx1170 and gfx12 only, but for consistency with the other WMMA
// variants we're keeping the "_gfx12" suffix.
-def __builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<2, int>, _ExtVector<2, int>, _ExtVector<8, float>)", [Const], "wmma-128b-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<2, int>, _Vector<2, int>, _Vector<8, float>)", [Const], "wmma-128b-insts,wavefrontsize32"> {
let Documentation = [DocWMMAFP8_16x16x16_GFX12];
let ArgNames = ["a", "b", "c"];
}
-def __builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w32_gfx12 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<2, int>, _ExtVector<2, int>, _ExtVector<8, float>)", [Const], "wmma-128b-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w32_gfx12 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<2, int>, _Vector<2, int>, _Vector<8, float>)", [Const], "wmma-128b-insts,wavefrontsize32"> {
let Documentation = [DocWMMAFP8_16x16x16_GFX12];
let ArgNames = ["a", "b", "c"];
}
-def __builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w32_gfx12 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<2, int>, _ExtVector<2, int>, _ExtVector<8, float>)", [Const], "wmma-128b-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w32_gfx12 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<2, int>, _Vector<2, int>, _Vector<8, float>)", [Const], "wmma-128b-insts,wavefrontsize32"> {
let Documentation = [DocWMMAFP8_16x16x16_GFX12];
let ArgNames = ["a", "b", "c"];
}
-def __builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w32_gfx12 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<2, int>, _ExtVector<2, int>, _ExtVector<8, float>)", [Const], "wmma-128b-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w32_gfx12 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<2, int>, _Vector<2, int>, _Vector<8, float>)", [Const], "wmma-128b-insts,wavefrontsize32"> {
let Documentation = [DocWMMAFP8_16x16x16_GFX12];
let ArgNames = ["a", "b", "c"];
}
-def __builtin_amdgcn_wmma_i32_16x16x32_iu4_w32_gfx12 : AMDGPUBuiltin<"_ExtVector<8, int>(_Constant bool, _ExtVector<2, int>, _Constant bool, _ExtVector<2, int>, _ExtVector<8, int>, _Constant bool)", [Const], "wmma-128b-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_i32_16x16x32_iu4_w32_gfx12 : AMDGPUBuiltin<"_Vector<8, int>(_Constant bool, _Vector<2, int>, _Constant bool, _Vector<2, int>, _Vector<8, int>, _Constant bool)", [Const], "wmma-128b-insts,wavefrontsize32"> {
let Documentation = [DocWMMAIU4_16x16x32_GFX12];
let ArgNames = ["a_sign", "a", "b_sign", "b", "c", "clamp"];
}
-def __builtin_amdgcn_wmma_f32_16x16x16_f16_w64_gfx12 : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<4, _Float16>, _ExtVector<4, _Float16>, _ExtVector<4, float>)", [Const], "wmma-128b-insts,wavefrontsize64"> {
+def __builtin_amdgcn_wmma_f32_16x16x16_f16_w64_gfx12 : AMDGPUBuiltin<"_Vector<4, float>(_Vector<4, _Float16>, _Vector<4, _Float16>, _Vector<4, float>)", [Const], "wmma-128b-insts,wavefrontsize64"> {
let Documentation = [DocWMMAF32_16x16x16_GFX12];
let ArgNames = ["a", "b", "c"];
}
-def __builtin_amdgcn_wmma_f32_16x16x16_bf16_w64_gfx12 : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<4, short>, _ExtVector<4, short>, _ExtVector<4, float>)", [Const], "wmma-128b-insts,wavefrontsize64"> {
+def __builtin_amdgcn_wmma_f32_16x16x16_bf16_w64_gfx12 : AMDGPUBuiltin<"_Vector<4, float>(_Vector<4, short>, _Vector<4, short>, _Vector<4, float>)", [Const], "wmma-128b-insts,wavefrontsize64"> {
let Documentation = [DocWMMAF32_16x16x16_GFX12];
let ArgNames = ["a", "b", "c"];
}
-def __builtin_amdgcn_wmma_f16_16x16x16_f16_w64_gfx12 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(_ExtVector<4, _Float16>, _ExtVector<4, _Float16>, _ExtVector<4, _Float16>)", [Const], "wmma-128b-insts,wavefrontsize64"> {
+def __builtin_amdgcn_wmma_f16_16x16x16_f16_w64_gfx12 : AMDGPUBuiltin<"_Vector<4, _Float16>(_Vector<4, _Float16>, _Vector<4, _Float16>, _Vector<4, _Float16>)", [Const], "wmma-128b-insts,wavefrontsize64"> {
let Documentation = [DocWMMAHalf_16x16x16_GFX12];
let ArgNames = ["a", "b", "c"];
}
-def __builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64_gfx12 : AMDGPUBuiltin<"_ExtVector<4, short>(_ExtVector<4, short>, _ExtVector<4, short>, _ExtVector<4, short>)", [Const], "wmma-128b-insts,wavefrontsize64"> {
+def __builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64_gfx12 : AMDGPUBuiltin<"_Vector<4, short>(_Vector<4, short>, _Vector<4, short>, _Vector<4, short>)", [Const], "wmma-128b-insts,wavefrontsize64"> {
let Documentation = [DocWMMAHalf_16x16x16_GFX12];
let ArgNames = ["a", "b", "c"];
}
-def __builtin_amdgcn_wmma_i32_16x16x16_iu8_w64_gfx12 : AMDGPUBuiltin<"_ExtVector<4, int>(_Constant bool, int, _Constant bool, int, _ExtVector<4, int>, _Constant bool)", [Const], "wmma-128b-insts,wavefrontsize64"> {
+def __builtin_amdgcn_wmma_i32_16x16x16_iu8_w64_gfx12 : AMDGPUBuiltin<"_Vector<4, int>(_Constant bool, int, _Constant bool, int, _Vector<4, int>, _Constant bool)", [Const], "wmma-128b-insts,wavefrontsize64"> {
let Documentation = [DocWMMAIU_16x16x16_GFX12];
let ArgNames = ["a_sign", "a", "b_sign", "b", "c", "clamp"];
}
-def __builtin_amdgcn_wmma_i32_16x16x16_iu4_w64_gfx12 : AMDGPUBuiltin<"_ExtVector<4, int>(_Constant bool, int, _Constant bool, int, _ExtVector<4, int>, _Constant bool)", [Const], "wmma-128b-insts,wavefrontsize64"> {
+def __builtin_amdgcn_wmma_i32_16x16x16_iu4_w64_gfx12 : AMDGPUBuiltin<"_Vector<4, int>(_Constant bool, int, _Constant bool, int, _Vector<4, int>, _Constant bool)", [Const], "wmma-128b-insts,wavefrontsize64"> {
let Documentation = [DocWMMAIU_16x16x16_GFX12];
let ArgNames = ["a_sign", "a", "b_sign", "b", "c", "clamp"];
}
// These are gfx1170 and gfx12 only, but for consistency with the other WMMA
// variants we're keeping the "_gfx12" suffix.
-def __builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w64_gfx12 : AMDGPUBuiltin<"_ExtVector<4, float>(int, int, _ExtVector<4, float>)", [Const], "wmma-128b-insts,wavefrontsize64"> {
+def __builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w64_gfx12 : AMDGPUBuiltin<"_Vector<4, float>(int, int, _Vector<4, float>)", [Const], "wmma-128b-insts,wavefrontsize64"> {
let Documentation = [DocWMMAFP8_16x16x16_GFX12];
let ArgNames = ["a", "b", "c"];
}
-def __builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w64_gfx12 : AMDGPUBuiltin<"_ExtVector<4, float>(int, int, _ExtVector<4, float>)", [Const], "wmma-128b-insts,wavefrontsize64"> {
+def __builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w64_gfx12 : AMDGPUBuiltin<"_Vector<4, float>(int, int, _Vector<4, float>)", [Const], "wmma-128b-insts,wavefrontsize64"> {
let Documentation = [DocWMMAFP8_16x16x16_GFX12];
let ArgNames = ["a", "b", "c"];
}
-def __builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w64_gfx12 : AMDGPUBuiltin<"_ExtVector<4, float>(int, int, _ExtVector<4, float>)", [Const], "wmma-128b-insts,wavefrontsize64"> {
+def __builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w64_gfx12 : AMDGPUBuiltin<"_Vector<4, float>(int, int, _Vector<4, float>)", [Const], "wmma-128b-insts,wavefrontsize64"> {
let Documentation = [DocWMMAFP8_16x16x16_GFX12];
let ArgNames = ["a", "b", "c"];
}
-def __builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w64_gfx12 : AMDGPUBuiltin<"_ExtVector<4, float>(int, int, _ExtVector<4, float>)", [Const], "wmma-128b-insts,wavefrontsize64"> {
+def __builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w64_gfx12 : AMDGPUBuiltin<"_Vector<4, float>(int, int, _Vector<4, float>)", [Const], "wmma-128b-insts,wavefrontsize64"> {
let Documentation = [DocWMMAFP8_16x16x16_GFX12];
let ArgNames = ["a", "b", "c"];
}
-def __builtin_amdgcn_wmma_i32_16x16x32_iu4_w64_gfx12 : AMDGPUBuiltin<"_ExtVector<4, int>(_Constant bool, int, _Constant bool, int, _ExtVector<4, int>, _Constant bool)", [Const], "wmma-128b-insts,wavefrontsize64"> {
+def __builtin_amdgcn_wmma_i32_16x16x32_iu4_w64_gfx12 : AMDGPUBuiltin<"_Vector<4, int>(_Constant bool, int, _Constant bool, int, _Vector<4, int>, _Constant bool)", [Const], "wmma-128b-insts,wavefrontsize64"> {
let Documentation = [DocWMMAIU4_16x16x32_GFX12];
let ArgNames = ["a_sign", "a", "b_sign", "b", "c", "clamp"];
}
-def __builtin_amdgcn_swmmac_f32_16x16x32_f16_w32 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<8, __fp16>, _ExtVector<16, __fp16>, _ExtVector<8, float>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize32">;
-def __builtin_amdgcn_swmmac_f32_16x16x32_bf16_w32 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<8, short>, _ExtVector<16, short>, _ExtVector<8, float>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize32">;
-def __builtin_amdgcn_swmmac_f16_16x16x32_f16_w32 : AMDGPUBuiltin<"_ExtVector<8, __fp16>(_ExtVector<8, __fp16>, _ExtVector<16, __fp16>, _ExtVector<8, __fp16>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize32">;
-def __builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w32 : AMDGPUBuiltin<"_ExtVector<8, short>(_ExtVector<8, short>, _ExtVector<16, short>, _ExtVector<8, short>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize32">;
-def __builtin_amdgcn_swmmac_i32_16x16x32_iu8_w32 : AMDGPUBuiltin<"_ExtVector<8, int>(_Constant bool, _ExtVector<2, int>, _Constant bool, _ExtVector<4, int>, _ExtVector<8, int>, int, _Constant bool)", [Const], "swmmac-gfx1200-insts,wavefrontsize32">;
-def __builtin_amdgcn_swmmac_i32_16x16x32_iu4_w32 : AMDGPUBuiltin<"_ExtVector<8, int>(_Constant bool, int, _Constant bool, _ExtVector<2, int>, _ExtVector<8, int>, int, _Constant bool)", [Const], "swmmac-gfx1200-insts,wavefrontsize32">;
-def __builtin_amdgcn_swmmac_i32_16x16x64_iu4_w32 : AMDGPUBuiltin<"_ExtVector<8, int>(_Constant bool, _ExtVector<2, int>, _Constant bool, _ExtVector<4, int>, _ExtVector<8, int>, int, _Constant bool)", [Const], "swmmac-gfx1200-insts,wavefrontsize32">;
-def __builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w32 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<2, int>, _ExtVector<4, int>, _ExtVector<8, float>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize32">;
-def __builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w32 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<2, int>, _ExtVector<4, int>, _ExtVector<8, float>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize32">;
-def __builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w32 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<2, int>, _ExtVector<4, int>, _ExtVector<8, float>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize32">;
-def __builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w32 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<2, int>, _ExtVector<4, int>, _ExtVector<8, float>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize32">;
-
-def __builtin_amdgcn_swmmac_f32_16x16x32_f16_w64 : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<4, __fp16>, _ExtVector<8, __fp16>, _ExtVector<4, float>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize64">;
-def __builtin_amdgcn_swmmac_f32_16x16x32_bf16_w64 : AMDGPUBuiltin<"_ExtVector<4, float>(_ExtVector<4, short>, _ExtVector<8, short>, _ExtVector<4, float>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize64">;
-def __builtin_amdgcn_swmmac_f16_16x16x32_f16_w64 : AMDGPUBuiltin<"_ExtVector<4, __fp16>(_ExtVector<4, __fp16>, _ExtVector<8, __fp16>, _ExtVector<4, __fp16>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize64">;
-def __builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w64 : AMDGPUBuiltin<"_ExtVector<4, short>(_ExtVector<4, short>, _ExtVector<8, short>, _ExtVector<4, short>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize64">;
-def __builtin_amdgcn_swmmac_i32_16x16x32_iu8_w64 : AMDGPUBuiltin<"_ExtVector<4, int>(_Constant bool, int, _Constant bool, _ExtVector<2, int>, _ExtVector<4, int>, int, _Constant bool)", [Const], "swmmac-gfx1200-insts,wavefrontsize64">;
-def __builtin_amdgcn_swmmac_i32_16x16x32_iu4_w64 : AMDGPUBuiltin<"_ExtVector<4, int>(_Constant bool, int, _Constant bool, int, _ExtVector<4, int>, int, _Constant bool)", [Const], "swmmac-gfx1200-insts,wavefrontsize64">;
-def __builtin_amdgcn_swmmac_i32_16x16x64_iu4_w64 : AMDGPUBuiltin<"_ExtVector<4, int>(_Constant bool, int, _Constant bool, _ExtVector<2, int>, _ExtVector<4, int>, int, _Constant bool)", [Const], "swmmac-gfx1200-insts,wavefrontsize64">;
-def __builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w64 : AMDGPUBuiltin<"_ExtVector<4, float>(int, _ExtVector<2, int>, _ExtVector<4, float>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize64">;
-def __builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w64 : AMDGPUBuiltin<"_ExtVector<4, float>(int, _ExtVector<2, int>, _ExtVector<4, float>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize64">;
-def __builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w64 : AMDGPUBuiltin<"_ExtVector<4, float>(int, _ExtVector<2, int>, _ExtVector<4, float>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize64">;
-def __builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w64 : AMDGPUBuiltin<"_ExtVector<4, float>(int, _ExtVector<2, int>, _ExtVector<4, float>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize64">;
+def __builtin_amdgcn_swmmac_f32_16x16x32_f16_w32 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<8, __fp16>, _Vector<16, __fp16>, _Vector<8, float>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize32">;
+def __builtin_amdgcn_swmmac_f32_16x16x32_bf16_w32 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<8, short>, _Vector<16, short>, _Vector<8, float>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize32">;
+def __builtin_amdgcn_swmmac_f16_16x16x32_f16_w32 : AMDGPUBuiltin<"_Vector<8, __fp16>(_Vector<8, __fp16>, _Vector<16, __fp16>, _Vector<8, __fp16>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize32">;
+def __builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w32 : AMDGPUBuiltin<"_Vector<8, short>(_Vector<8, short>, _Vector<16, short>, _Vector<8, short>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize32">;
+def __builtin_amdgcn_swmmac_i32_16x16x32_iu8_w32 : AMDGPUBuiltin<"_Vector<8, int>(_Constant bool, _Vector<2, int>, _Constant bool, _Vector<4, int>, _Vector<8, int>, int, _Constant bool)", [Const], "swmmac-gfx1200-insts,wavefrontsize32">;
+def __builtin_amdgcn_swmmac_i32_16x16x32_iu4_w32 : AMDGPUBuiltin<"_Vector<8, int>(_Constant bool, int, _Constant bool, _Vector<2, int>, _Vector<8, int>, int, _Constant bool)", [Const], "swmmac-gfx1200-insts,wavefrontsize32">;
+def __builtin_amdgcn_swmmac_i32_16x16x64_iu4_w32 : AMDGPUBuiltin<"_Vector<8, int>(_Constant bool, _Vector<2, int>, _Constant bool, _Vector<4, int>, _Vector<8, int>, int, _Constant bool)", [Const], "swmmac-gfx1200-insts,wavefrontsize32">;
+def __builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w32 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<2, int>, _Vector<4, int>, _Vector<8, float>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize32">;
+def __builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w32 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<2, int>, _Vector<4, int>, _Vector<8, float>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize32">;
+def __builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w32 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<2, int>, _Vector<4, int>, _Vector<8, float>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize32">;
+def __builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w32 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<2, int>, _Vector<4, int>, _Vector<8, float>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize32">;
+
+def __builtin_amdgcn_swmmac_f32_16x16x32_f16_w64 : AMDGPUBuiltin<"_Vector<4, float>(_Vector<4, __fp16>, _Vector<8, __fp16>, _Vector<4, float>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize64">;
+def __builtin_amdgcn_swmmac_f32_16x16x32_bf16_w64 : AMDGPUBuiltin<"_Vector<4, float>(_Vector<4, short>, _Vector<8, short>, _Vector<4, float>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize64">;
+def __builtin_amdgcn_swmmac_f16_16x16x32_f16_w64 : AMDGPUBuiltin<"_Vector<4, __fp16>(_Vector<4, __fp16>, _Vector<8, __fp16>, _Vector<4, __fp16>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize64">;
+def __builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w64 : AMDGPUBuiltin<"_Vector<4, short>(_Vector<4, short>, _Vector<8, short>, _Vector<4, short>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize64">;
+def __builtin_amdgcn_swmmac_i32_16x16x32_iu8_w64 : AMDGPUBuiltin<"_Vector<4, int>(_Constant bool, int, _Constant bool, _Vector<2, int>, _Vector<4, int>, int, _Constant bool)", [Const], "swmmac-gfx1200-insts,wavefrontsize64">;
+def __builtin_amdgcn_swmmac_i32_16x16x32_iu4_w64 : AMDGPUBuiltin<"_Vector<4, int>(_Constant bool, int, _Constant bool, int, _Vector<4, int>, int, _Constant bool)", [Const], "swmmac-gfx1200-insts,wavefrontsize64">;
+def __builtin_amdgcn_swmmac_i32_16x16x64_iu4_w64 : AMDGPUBuiltin<"_Vector<4, int>(_Constant bool, int, _Constant bool, _Vector<2, int>, _Vector<4, int>, int, _Constant bool)", [Const], "swmmac-gfx1200-insts,wavefrontsize64">;
+def __builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w64 : AMDGPUBuiltin<"_Vector<4, float>(int, _Vector<2, int>, _Vector<4, float>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize64">;
+def __builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w64 : AMDGPUBuiltin<"_Vector<4, float>(int, _Vector<2, int>, _Vector<4, float>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize64">;
+def __builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w64 : AMDGPUBuiltin<"_Vector<4, float>(int, _Vector<2, int>, _Vector<4, float>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize64">;
+def __builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w64 : AMDGPUBuiltin<"_Vector<4, float>(int, _Vector<2, int>, _Vector<4, float>, int)", [Const], "swmmac-gfx1200-insts,wavefrontsize64">;
def __builtin_amdgcn_prng_b32 : AMDGPUBuiltin<"unsigned int(unsigned int)", [Const], "prng-inst">;
-def __builtin_amdgcn_cvt_scalef32_pk32_fp6_f16 : AMDGPUBuiltin<"_ExtVector<6, unsigned int>(_ExtVector<32, _Float16>, float)", [Const], "f16bf16-to-fp6bf6-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk32_bf6_f16 : AMDGPUBuiltin<"_ExtVector<6, unsigned int>(_ExtVector<32, _Float16>, float)", [Const], "f16bf16-to-fp6bf6-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk32_fp6_bf16 : AMDGPUBuiltin<"_ExtVector<6, unsigned int>(_ExtVector<32, __bf16>, float)", [Const], "f16bf16-to-fp6bf6-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk32_bf6_bf16 : AMDGPUBuiltin<"_ExtVector<6, unsigned int>(_ExtVector<32, __bf16>, float)", [Const], "f16bf16-to-fp6bf6-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_f16_fp8 : AMDGPUBuiltin<"_ExtVector<2, _Float16>(_ExtVector<2, _Float16>, int, float, _Constant int, _Constant bool)", [Const], "fp8-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_f16_bf8 : AMDGPUBuiltin<"_ExtVector<2, _Float16>(_ExtVector<2, _Float16>, int, float, _Constant int, _Constant bool)", [Const], "bf8-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk32_fp6_f16 : AMDGPUBuiltin<"_Vector<6, unsigned int>(_Vector<32, _Float16>, float)", [Const], "f16bf16-to-fp6bf6-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk32_bf6_f16 : AMDGPUBuiltin<"_Vector<6, unsigned int>(_Vector<32, _Float16>, float)", [Const], "f16bf16-to-fp6bf6-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk32_fp6_bf16 : AMDGPUBuiltin<"_Vector<6, unsigned int>(_Vector<32, __bf16>, float)", [Const], "f16bf16-to-fp6bf6-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk32_bf6_bf16 : AMDGPUBuiltin<"_Vector<6, unsigned int>(_Vector<32, __bf16>, float)", [Const], "f16bf16-to-fp6bf6-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_f16_fp8 : AMDGPUBuiltin<"_Vector<2, _Float16>(_Vector<2, _Float16>, int, float, _Constant int, _Constant bool)", [Const], "fp8-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_f16_bf8 : AMDGPUBuiltin<"_Vector<2, _Float16>(_Vector<2, _Float16>, int, float, _Constant int, _Constant bool)", [Const], "bf8-cvt-scale-insts">;
def __builtin_amdgcn_cvt_scalef32_f32_fp8 : AMDGPUBuiltin<"float(int, float, _Constant int)", [Const], "fp8-cvt-scale-insts">;
def __builtin_amdgcn_cvt_scalef32_f32_bf8 : AMDGPUBuiltin<"float(int, float, _Constant int)", [Const], "bf8-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk_fp8_f32 : AMDGPUBuiltin<"_ExtVector<2, short>(_ExtVector<2, short>, float, float, float, _Constant bool)", [Const], "fp8-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk_bf8_f32 : AMDGPUBuiltin<"_ExtVector<2, short>(_ExtVector<2, short>, float, float, float, _Constant bool)", [Const], "bf8-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk_f32_fp8 : AMDGPUBuiltin<"_ExtVector<2, float>(unsigned int, float, _Constant bool)", [Const], "fp8-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk_f32_bf8 : AMDGPUBuiltin<"_ExtVector<2, float>(unsigned int, float, _Constant bool)", [Const], "bf8-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk_fp8_f16 : AMDGPUBuiltin<"_ExtVector<2, short>(_ExtVector<2, short>, _ExtVector<2, _Float16>, float, _Constant bool)", [Const], "fp8-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk_fp8_bf16 : AMDGPUBuiltin<"_ExtVector<2, short>(_ExtVector<2, short>, _ExtVector<2, __bf16>, float, _Constant bool)", [Const], "fp8-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk_bf8_f16 : AMDGPUBuiltin<"_ExtVector<2, short>(_ExtVector<2, short>, _ExtVector<2, _Float16>, float, _Constant bool)", [Const], "bf8-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk_bf8_bf16 : AMDGPUBuiltin<"_ExtVector<2, short>(_ExtVector<2, short>, _ExtVector<2, __bf16>, float, _Constant bool)", [Const], "bf8-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk_f32_fp4 : AMDGPUBuiltin<"_ExtVector<2, float>(unsigned int, float, _Constant int)", [Const], "fp4-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk_fp8_f32 : AMDGPUBuiltin<"_Vector<2, short>(_Vector<2, short>, float, float, float, _Constant bool)", [Const], "fp8-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk_bf8_f32 : AMDGPUBuiltin<"_Vector<2, short>(_Vector<2, short>, float, float, float, _Constant bool)", [Const], "bf8-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk_f32_fp8 : AMDGPUBuiltin<"_Vector<2, float>(unsigned int, float, _Constant bool)", [Const], "fp8-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk_f32_bf8 : AMDGPUBuiltin<"_Vector<2, float>(unsigned int, float, _Constant bool)", [Const], "bf8-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk_fp8_f16 : AMDGPUBuiltin<"_Vector<2, short>(_Vector<2, short>, _Vector<2, _Float16>, float, _Constant bool)", [Const], "fp8-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk_fp8_bf16 : AMDGPUBuiltin<"_Vector<2, short>(_Vector<2, short>, _Vector<2, __bf16>, float, _Constant bool)", [Const], "fp8-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk_bf8_f16 : AMDGPUBuiltin<"_Vector<2, short>(_Vector<2, short>, _Vector<2, _Float16>, float, _Constant bool)", [Const], "bf8-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk_bf8_bf16 : AMDGPUBuiltin<"_Vector<2, short>(_Vector<2, short>, _Vector<2, __bf16>, float, _Constant bool)", [Const], "bf8-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk_f32_fp4 : AMDGPUBuiltin<"_Vector<2, float>(unsigned int, float, _Constant int)", [Const], "fp4-cvt-scale-insts">;
def __builtin_amdgcn_cvt_scalef32_pk_fp4_f32 : AMDGPUBuiltin<"unsigned int(unsigned int, float, float, float, _Constant int)", [Const], "fp4-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk_f16_fp4 : AMDGPUBuiltin<"_ExtVector<2, _Float16>(unsigned int, float, _Constant int)", [Const], "fp4-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk_bf16_fp4 : AMDGPUBuiltin<"_ExtVector<2, __bf16>(unsigned int, float, _Constant int)", [Const], "fp4-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk32_f32_fp6 : AMDGPUBuiltin<"_ExtVector<32, float>(_ExtVector<6, unsigned int>, float)", [Const], "fp6bf6-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk32_f32_bf6 : AMDGPUBuiltin<"_ExtVector<32, float>(_ExtVector<6, unsigned int>, float)", [Const], "fp6bf6-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk32_f16_fp6 : AMDGPUBuiltin<"_ExtVector<32, _Float16>(_ExtVector<6, unsigned int>, float)", [Const], "fp6bf6-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk32_bf16_fp6 : AMDGPUBuiltin<"_ExtVector<32, __bf16>(_ExtVector<6, unsigned int>, float)", [Const], "fp6bf6-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk32_f16_bf6 : AMDGPUBuiltin<"_ExtVector<32, _Float16>(_ExtVector<6, unsigned int>, float)", [Const], "fp6bf6-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk32_bf16_bf6 : AMDGPUBuiltin<"_ExtVector<32, __bf16>(_ExtVector<6, unsigned int>, float)", [Const], "fp6bf6-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk_f16_fp8 : AMDGPUBuiltin<"_ExtVector<2, _Float16>(unsigned int, float, _Constant bool)", [Const], "fp8-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk_bf16_fp8 : AMDGPUBuiltin<"_ExtVector<2, __bf16>(unsigned int, float, _Constant bool)", [Const], "fp8-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk_f16_bf8 : AMDGPUBuiltin<"_ExtVector<2, _Float16>(unsigned int, float, _Constant bool)", [Const], "bf8-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk_bf16_bf8 : AMDGPUBuiltin<"_ExtVector<2, __bf16>(unsigned int, float, _Constant bool)", [Const], "bf8-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk_fp4_f16 : AMDGPUBuiltin<"unsigned int(unsigned int, _ExtVector<2, _Float16>, float, _Constant int)", [Const], "fp4-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk_fp4_bf16 : AMDGPUBuiltin<"unsigned int(unsigned int, _ExtVector<2, __bf16>, float, _Constant int)", [Const], "fp4-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_sr_pk_fp4_f16 : AMDGPUBuiltin<"unsigned int(unsigned int, _ExtVector<2, _Float16>, unsigned int, float, _Constant int)", [Const], "fp4-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_sr_pk_fp4_bf16 : AMDGPUBuiltin<"unsigned int(unsigned int, _ExtVector<2, __bf16>, unsigned int, float, _Constant int)", [Const], "fp4-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_sr_pk_fp4_f32 : AMDGPUBuiltin<"unsigned int(unsigned int, _ExtVector<2, float>, unsigned int, float, _Constant int)", [Const], "fp4-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk_f16_fp4 : AMDGPUBuiltin<"_Vector<2, _Float16>(unsigned int, float, _Constant int)", [Const], "fp4-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk_bf16_fp4 : AMDGPUBuiltin<"_Vector<2, __bf16>(unsigned int, float, _Constant int)", [Const], "fp4-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk32_f32_fp6 : AMDGPUBuiltin<"_Vector<32, float>(_Vector<6, unsigned int>, float)", [Const], "fp6bf6-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk32_f32_bf6 : AMDGPUBuiltin<"_Vector<32, float>(_Vector<6, unsigned int>, float)", [Const], "fp6bf6-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk32_f16_fp6 : AMDGPUBuiltin<"_Vector<32, _Float16>(_Vector<6, unsigned int>, float)", [Const], "fp6bf6-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk32_bf16_fp6 : AMDGPUBuiltin<"_Vector<32, __bf16>(_Vector<6, unsigned int>, float)", [Const], "fp6bf6-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk32_f16_bf6 : AMDGPUBuiltin<"_Vector<32, _Float16>(_Vector<6, unsigned int>, float)", [Const], "fp6bf6-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk32_bf16_bf6 : AMDGPUBuiltin<"_Vector<32, __bf16>(_Vector<6, unsigned int>, float)", [Const], "fp6bf6-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk_f16_fp8 : AMDGPUBuiltin<"_Vector<2, _Float16>(unsigned int, float, _Constant bool)", [Const], "fp8-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk_bf16_fp8 : AMDGPUBuiltin<"_Vector<2, __bf16>(unsigned int, float, _Constant bool)", [Const], "fp8-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk_f16_bf8 : AMDGPUBuiltin<"_Vector<2, _Float16>(unsigned int, float, _Constant bool)", [Const], "bf8-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk_bf16_bf8 : AMDGPUBuiltin<"_Vector<2, __bf16>(unsigned int, float, _Constant bool)", [Const], "bf8-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk_fp4_f16 : AMDGPUBuiltin<"unsigned int(unsigned int, _Vector<2, _Float16>, float, _Constant int)", [Const], "fp4-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk_fp4_bf16 : AMDGPUBuiltin<"unsigned int(unsigned int, _Vector<2, __bf16>, float, _Constant int)", [Const], "fp4-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_sr_pk_fp4_f16 : AMDGPUBuiltin<"unsigned int(unsigned int, _Vector<2, _Float16>, unsigned int, float, _Constant int)", [Const], "fp4-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_sr_pk_fp4_bf16 : AMDGPUBuiltin<"unsigned int(unsigned int, _Vector<2, __bf16>, unsigned int, float, _Constant int)", [Const], "fp4-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_sr_pk_fp4_f32 : AMDGPUBuiltin<"unsigned int(unsigned int, _Vector<2, float>, unsigned int, float, _Constant int)", [Const], "fp4-cvt-scale-insts">;
def __builtin_amdgcn_cvt_scalef32_sr_bf8_bf16 : AMDGPUBuiltin<"int(int, __bf16, unsigned int, float, _Constant int)", [Const], "bf8-cvt-scale-insts">;
def __builtin_amdgcn_cvt_scalef32_sr_bf8_f16 : AMDGPUBuiltin<"int(int, _Float16, unsigned int, float, _Constant int)", [Const], "bf8-cvt-scale-insts">;
def __builtin_amdgcn_cvt_scalef32_sr_bf8_f32 : AMDGPUBuiltin<"int(int, float, unsigned int, float, _Constant int)", [Const], "bf8-cvt-scale-insts">;
@@ -970,17 +970,17 @@ def __builtin_amdgcn_cvt_scalef32_sr_fp8_bf16 : AMDGPUBuiltin<"int(int, __bf16,
def __builtin_amdgcn_cvt_scalef32_sr_fp8_f16 : AMDGPUBuiltin<"int(int, _Float16, unsigned int, float, _Constant int)", [Const], "fp8-cvt-scale-insts">;
def __builtin_amdgcn_cvt_scalef32_sr_fp8_f32 : AMDGPUBuiltin<"int(int, float, unsigned int, float, _Constant int)", [Const], "fp8-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_sr_pk32_bf6_bf16 : AMDGPUBuiltin<"_ExtVector<6, unsigned int>(_ExtVector<32, __bf16>, unsigned int, float)", [Const], "f16bf16-to-fp6bf6-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_sr_pk32_bf6_f16 : AMDGPUBuiltin<"_ExtVector<6, unsigned int>(_ExtVector<32, _Float16>, unsigned int, float)", [Const], "f16bf16-to-fp6bf6-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_sr_pk32_bf6_f32 : AMDGPUBuiltin<"_ExtVector<6, unsigned int>(_ExtVector<32, float>, unsigned int, float)", [Const], "f16bf16-to-fp6bf6-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_sr_pk32_fp6_bf16 : AMDGPUBuiltin<"_ExtVector<6, unsigned int>(_ExtVector<32, __bf16>, unsigned int, float)", [Const], "f16bf16-to-fp6bf6-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_sr_pk32_fp6_f16 : AMDGPUBuiltin<"_ExtVector<6, unsigned int>(_ExtVector<32, _Float16>, unsigned int, float)", [Const], "f16bf16-to-fp6bf6-cvt-scale-insts">;
-def __builtin_amdgcn_cvt_scalef32_sr_pk32_fp6_f32 : AMDGPUBuiltin<"_ExtVector<6, unsigned int>(_ExtVector<32, float>, unsigned int, float)", [Const], "f16bf16-to-fp6bf6-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_sr_pk32_bf6_bf16 : AMDGPUBuiltin<"_Vector<6, unsigned int>(_Vector<32, __bf16>, unsigned int, float)", [Const], "f16bf16-to-fp6bf6-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_sr_pk32_bf6_f16 : AMDGPUBuiltin<"_Vector<6, unsigned int>(_Vector<32, _Float16>, unsigned int, float)", [Const], "f16bf16-to-fp6bf6-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_sr_pk32_bf6_f32 : AMDGPUBuiltin<"_Vector<6, unsigned int>(_Vector<32, float>, unsigned int, float)", [Const], "f16bf16-to-fp6bf6-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_sr_pk32_fp6_bf16 : AMDGPUBuiltin<"_Vector<6, unsigned int>(_Vector<32, __bf16>, unsigned int, float)", [Const], "f16bf16-to-fp6bf6-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_sr_pk32_fp6_f16 : AMDGPUBuiltin<"_Vector<6, unsigned int>(_Vector<32, _Float16>, unsigned int, float)", [Const], "f16bf16-to-fp6bf6-cvt-scale-insts">;
+def __builtin_amdgcn_cvt_scalef32_sr_pk32_fp6_f32 : AMDGPUBuiltin<"_Vector<6, unsigned int>(_Vector<32, float>, unsigned int, float)", [Const], "f16bf16-to-fp6bf6-cvt-scale-insts">;
def __builtin_amdgcn_bitop3_b32 : AMDGPUBuiltin<"int(int, int, int, _Constant unsigned int)", [Const], "bitop3-insts">;
def __builtin_amdgcn_bitop3_b16 : AMDGPUBuiltin<"short(short, short, short, _Constant unsigned int)", [Const], "bitop3-insts">;
-def __builtin_amdgcn_cvt_sr_bf16_f32 : AMDGPUBuiltin<"_ExtVector<2, __bf16>(_ExtVector<2, __bf16>, float, unsigned int, _Constant bool)", [Const], "f32-to-f16bf16-cvt-sr-insts">;
-def __builtin_amdgcn_cvt_sr_f16_f32 : AMDGPUBuiltin<"_ExtVector<2, _Float16>(_ExtVector<2, _Float16>, float, unsigned int, _Constant bool)", [Const], "f32-to-f16bf16-cvt-sr-insts">;
+def __builtin_amdgcn_cvt_sr_bf16_f32 : AMDGPUBuiltin<"_Vector<2, __bf16>(_Vector<2, __bf16>, float, unsigned int, _Constant bool)", [Const], "f32-to-f16bf16-cvt-sr-insts">;
+def __builtin_amdgcn_cvt_sr_f16_f32 : AMDGPUBuiltin<"_Vector<2, _Float16>(_Vector<2, _Float16>, float, unsigned int, _Constant bool)", [Const], "f32-to-f16bf16-cvt-sr-insts">;
//===----------------------------------------------------------------------===//
// GFX1250+ only builtins.
@@ -991,51 +991,51 @@ def __builtin_amdgcn_flat_prefetch : AMDGPUBuiltin<"void(void const address_spac
def __builtin_amdgcn_global_prefetch : AMDGPUBuiltin<"void(void const address_space<1> *, _Constant int)", [Const], "vmem-pref-insts">;
def __builtin_amdgcn_global_load_monitor_b32 : AMDGPUBuiltin<"int(int address_space<1> *, _Constant int, _Constant int)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_global_load_monitor_b64 : AMDGPUBuiltin<"_ExtVector<2, int>(_ExtVector<2, int> address_space<1> *, _Constant int, _Constant int)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_global_load_monitor_b128 : AMDGPUBuiltin<"_ExtVector<4, int>(_ExtVector<4, int> address_space<1> *, _Constant int, _Constant int)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_global_load_monitor_b64 : AMDGPUBuiltin<"_Vector<2, int>(_Vector<2, int> address_space<1> *, _Constant int, _Constant int)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_global_load_monitor_b128 : AMDGPUBuiltin<"_Vector<4, int>(_Vector<4, int> address_space<1> *, _Constant int, _Constant int)", [Const], "gfx1250-insts">;
def __builtin_amdgcn_flat_load_monitor_b32 : AMDGPUBuiltin<"int(int address_space<0> *, _Constant int, _Constant int)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_flat_load_monitor_b64 : AMDGPUBuiltin<"_ExtVector<2, int>(_ExtVector<2, int> address_space<0> *, _Constant int, _Constant int)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_flat_load_monitor_b128 : AMDGPUBuiltin<"_ExtVector<4, int>(_ExtVector<4, int> address_space<0> *, _Constant int, _Constant int)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_flat_load_monitor_b64 : AMDGPUBuiltin<"_Vector<2, int>(_Vector<2, int> address_space<0> *, _Constant int, _Constant int)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_flat_load_monitor_b128 : AMDGPUBuiltin<"_Vector<4, int>(_Vector<4, int> address_space<0> *, _Constant int, _Constant int)", [Const], "gfx1250-insts">;
def __builtin_amdgcn_cluster_load_b32 : AMDGPUBuiltin<"int(int address_space<1> *, _Constant int, int)", [Const], "mcast-load-insts,wavefrontsize32">;
-def __builtin_amdgcn_cluster_load_b64 : AMDGPUBuiltin<"_ExtVector<2, int>(_ExtVector<2, int> address_space<1> *, _Constant int, int)", [Const], "mcast-load-insts,wavefrontsize32">;
-def __builtin_amdgcn_cluster_load_b128 : AMDGPUBuiltin<"_ExtVector<4, int>(_ExtVector<4, int> address_space<1> *, _Constant int, int)", [Const], "mcast-load-insts,wavefrontsize32">;
+def __builtin_amdgcn_cluster_load_b64 : AMDGPUBuiltin<"_Vector<2, int>(_Vector<2, int> address_space<1> *, _Constant int, int)", [Const], "mcast-load-insts,wavefrontsize32">;
+def __builtin_amdgcn_cluster_load_b128 : AMDGPUBuiltin<"_Vector<4, int>(_Vector<4, int> address_space<1> *, _Constant int, int)", [Const], "mcast-load-insts,wavefrontsize32">;
def __builtin_amdgcn_cluster_load_async_to_lds_b8 : AMDGPUBuiltin<"void(char address_space<1> *, char address_space<3> *, _Constant int, _Constant int, int)", [Const], "mcast-load-insts,wavefrontsize32">;
def __builtin_amdgcn_cluster_load_async_to_lds_b32 : AMDGPUBuiltin<"void(int address_space<1> *, int address_space<3> *, _Constant int, _Constant int, int)", [Const], "mcast-load-insts,wavefrontsize32">;
-def __builtin_amdgcn_cluster_load_async_to_lds_b64 : AMDGPUBuiltin<"void(_ExtVector<2, int> address_space<1> *, _ExtVector<2, int> address_space<3> *, _Constant int, _Constant int, int)", [Const], "mcast-load-insts,wavefrontsize32">;
-def __builtin_amdgcn_cluster_load_async_to_lds_b128 : AMDGPUBuiltin<"void(_ExtVector<4, int> address_space<1> *, _ExtVector<4, int> address_space<3> *, _Constant int, _Constant int, int)", [Const], "mcast-load-insts,wavefrontsize32">;
+def __builtin_amdgcn_cluster_load_async_to_lds_b64 : AMDGPUBuiltin<"void(_Vector<2, int> address_space<1> *, _Vector<2, int> address_space<3> *, _Constant int, _Constant int, int)", [Const], "mcast-load-insts,wavefrontsize32">;
+def __builtin_amdgcn_cluster_load_async_to_lds_b128 : AMDGPUBuiltin<"void(_Vector<4, int> address_space<1> *, _Vector<4, int> address_space<3> *, _Constant int, _Constant int, int)", [Const], "mcast-load-insts,wavefrontsize32">;
def __builtin_amdgcn_global_load_async_to_lds_b8 : AMDGPUBuiltin<"void(char address_space<1> *, char address_space<3> *, _Constant int, _Constant int)", [Const], "async-load-to-lds-insts,wavefrontsize32">;
def __builtin_amdgcn_global_load_async_to_lds_b32 : AMDGPUBuiltin<"void(int address_space<1> *, int address_space<3> *, _Constant int, _Constant int)", [Const], "async-load-to-lds-insts,wavefrontsize32">;
-def __builtin_amdgcn_global_load_async_to_lds_b64 : AMDGPUBuiltin<"void(_ExtVector<2, int> address_space<1> *, _ExtVector<2, int> address_space<3> *, _Constant int, _Constant int)", [Const], "async-load-to-lds-insts,wavefrontsize32">;
-def __builtin_amdgcn_global_load_async_to_lds_b128 : AMDGPUBuiltin<"void(_ExtVector<4, int> address_space<1> *, _ExtVector<4, int> address_space<3> *, _Constant int, _Constant int)", [Const], "async-load-to-lds-insts,wavefrontsize32">;
+def __builtin_amdgcn_global_load_async_to_lds_b64 : AMDGPUBuiltin<"void(_Vector<2, int> address_space<1> *, _Vector<2, int> address_space<3> *, _Constant int, _Constant int)", [Const], "async-load-to-lds-insts,wavefrontsize32">;
+def __builtin_amdgcn_global_load_async_to_lds_b128 : AMDGPUBuiltin<"void(_Vector<4, int> address_space<1> *, _Vector<4, int> address_space<3> *, _Constant int, _Constant int)", [Const], "async-load-to-lds-insts,wavefrontsize32">;
def __builtin_amdgcn_global_store_async_from_lds_b8 : AMDGPUBuiltin<"void(char address_space<1> *, char address_space<3> *, _Constant int, _Constant int)", [Const], "async-store-from-lds-insts">;
def __builtin_amdgcn_global_store_async_from_lds_b32 : AMDGPUBuiltin<"void(int address_space<1> *, int address_space<3> *, _Constant int, _Constant int)", [Const], "async-store-from-lds-insts">;
-def __builtin_amdgcn_global_store_async_from_lds_b64 : AMDGPUBuiltin<"void(_ExtVector<2, int> address_space<1> *, _ExtVector<2, int> address_space<3> *, _Constant int, _Constant int)", [Const], "async-store-from-lds-insts">;
-def __builtin_amdgcn_global_store_async_from_lds_b128 : AMDGPUBuiltin<"void(_ExtVector<4, int> address_space<1> *, _ExtVector<4, int> address_space<3> *, _Constant int, _Constant int)", [Const], "async-store-from-lds-insts">;
+def __builtin_amdgcn_global_store_async_from_lds_b64 : AMDGPUBuiltin<"void(_Vector<2, int> address_space<1> *, _Vector<2, int> address_space<3> *, _Constant int, _Constant int)", [Const], "async-store-from-lds-insts">;
+def __builtin_amdgcn_global_store_async_from_lds_b128 : AMDGPUBuiltin<"void(_Vector<4, int> address_space<1> *, _Vector<4, int> address_space<3> *, _Constant int, _Constant int)", [Const], "async-store-from-lds-insts">;
def __builtin_amdgcn_ds_atomic_async_barrier_arrive_b64 : AMDGPUBuiltin<"void(long int address_space<3> *)", [Const], "gfx1250-insts">;
def __builtin_amdgcn_ds_atomic_barrier_arrive_rtn_b64 : AMDGPUBuiltin<"long int(long int address_space<3> *, long int)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_tensor_load_to_lds : AMDGPUBuiltin<"void(_ExtVector<4, unsigned int>, _ExtVector<8, int>, _ExtVector<4, int>, _ExtVector<4, int>, _ExtVector<8, int>, _Constant int)", [Const], "gfx1250-insts"> {
+def __builtin_amdgcn_tensor_load_to_lds : AMDGPUBuiltin<"void(_Vector<4, unsigned int>, _Vector<8, int>, _Vector<4, int>, _Vector<4, int>, _Vector<8, int>, _Constant int)", [Const], "gfx1250-insts"> {
let Documentation = [DocTensorLoadToLDS_GFX1250];
let ArgNames = ["D0", "D1", "D2", "D3", "D4", "cpol"];
}
-def __builtin_amdgcn_tensor_store_from_lds : AMDGPUBuiltin<"void(_ExtVector<4, unsigned int>, _ExtVector<8, int>, _ExtVector<4, int>, _ExtVector<4, int>, _ExtVector<8, int>, _Constant int)", [Const], "gfx1250-insts"> {
+def __builtin_amdgcn_tensor_store_from_lds : AMDGPUBuiltin<"void(_Vector<4, unsigned int>, _Vector<8, int>, _Vector<4, int>, _Vector<4, int>, _Vector<8, int>, _Constant int)", [Const], "gfx1250-insts"> {
let Documentation = [DocTensorStoreFromLDS_GFX1250];
let ArgNames = ["D0", "D1", "D2", "D3", "D4", "cpol"];
}
-def __builtin_amdgcn_global_load_tr4_b64_v2i32 : AMDGPUBuiltin<"_ExtVector<2, int>(_ExtVector<2, int> address_space<1> *)", [Const], "transpose-load-f4f6-insts,wavefrontsize32">;
-def __builtin_amdgcn_global_load_tr8_b64_v2i32 : AMDGPUBuiltin<"_ExtVector<2, int>(_ExtVector<2, int> address_space<1> *)", [Const], "gfx1250-insts,wavefrontsize32">;
-def __builtin_amdgcn_global_load_tr6_b96_v3i32 : AMDGPUBuiltin<"_ExtVector<3, int>(_ExtVector<3, int> address_space<1> *)", [Const], "transpose-load-f4f6-insts,wavefrontsize32">;
-def __builtin_amdgcn_global_load_tr16_b128_v8i16 : AMDGPUBuiltin<"_ExtVector<8, short>(_ExtVector<8, short> address_space<1> *)", [Const], "gfx1250-insts,wavefrontsize32">;
-def __builtin_amdgcn_global_load_tr16_b128_v8f16 : AMDGPUBuiltin<"_ExtVector<8, __fp16>(_ExtVector<8, __fp16> address_space<1> *)", [Const], "gfx1250-insts,wavefrontsize32">;
-def __builtin_amdgcn_global_load_tr16_b128_v8bf16 : AMDGPUBuiltin<"_ExtVector<8, __bf16>(_ExtVector<8, __bf16> address_space<1> *)", [Const], "gfx1250-insts,wavefrontsize32">;
-def __builtin_amdgcn_ds_load_tr4_b64_v2i32 : AMDGPUBuiltin<"_ExtVector<2, int>(_ExtVector<2, int> address_space<3> *)", [Const], "transpose-load-f4f6-insts,wavefrontsize32">;
-def __builtin_amdgcn_ds_load_tr8_b64_v2i32 : AMDGPUBuiltin<"_ExtVector<2, int>(_ExtVector<2, int> address_space<3> *)", [Const], "gfx1250-insts,wavefrontsize32">;
-def __builtin_amdgcn_ds_load_tr6_b96_v3i32 : AMDGPUBuiltin<"_ExtVector<3, int>(_ExtVector<3, int> address_space<3> *)", [Const], "transpose-load-f4f6-insts,wavefrontsize32">;
-def __builtin_amdgcn_ds_load_tr16_b128_v8i16 : AMDGPUBuiltin<"_ExtVector<8, short>(_ExtVector<8, short> address_space<3> *)", [Const], "gfx1250-insts,wavefrontsize32">;
-def __builtin_amdgcn_ds_load_tr16_b128_v8f16 : AMDGPUBuiltin<"_ExtVector<8, __fp16>(_ExtVector<8, __fp16> address_space<3> *)", [Const], "gfx1250-insts,wavefrontsize32">;
-def __builtin_amdgcn_ds_load_tr16_b128_v8bf16 : AMDGPUBuiltin<"_ExtVector<8, __bf16>(_ExtVector<8, __bf16> address_space<3> *)", [Const], "gfx1250-insts,wavefrontsize32">;
+def __builtin_amdgcn_global_load_tr4_b64_v2i32 : AMDGPUBuiltin<"_Vector<2, int>(_Vector<2, int> address_space<1> *)", [Const], "transpose-load-f4f6-insts,wavefrontsize32">;
+def __builtin_amdgcn_global_load_tr8_b64_v2i32 : AMDGPUBuiltin<"_Vector<2, int>(_Vector<2, int> address_space<1> *)", [Const], "gfx1250-insts,wavefrontsize32">;
+def __builtin_amdgcn_global_load_tr6_b96_v3i32 : AMDGPUBuiltin<"_Vector<3, int>(_Vector<3, int> address_space<1> *)", [Const], "transpose-load-f4f6-insts,wavefrontsize32">;
+def __builtin_amdgcn_global_load_tr16_b128_v8i16 : AMDGPUBuiltin<"_Vector<8, short>(_Vector<8, short> address_space<1> *)", [Const], "gfx1250-insts,wavefrontsize32">;
+def __builtin_amdgcn_global_load_tr16_b128_v8f16 : AMDGPUBuiltin<"_Vector<8, __fp16>(_Vector<8, __fp16> address_space<1> *)", [Const], "gfx1250-insts,wavefrontsize32">;
+def __builtin_amdgcn_global_load_tr16_b128_v8bf16 : AMDGPUBuiltin<"_Vector<8, __bf16>(_Vector<8, __bf16> address_space<1> *)", [Const], "gfx1250-insts,wavefrontsize32">;
+def __builtin_amdgcn_ds_load_tr4_b64_v2i32 : AMDGPUBuiltin<"_Vector<2, int>(_Vector<2, int> address_space<3> *)", [Const], "transpose-load-f4f6-insts,wavefrontsize32">;
+def __builtin_amdgcn_ds_load_tr8_b64_v2i32 : AMDGPUBuiltin<"_Vector<2, int>(_Vector<2, int> address_space<3> *)", [Const], "gfx1250-insts,wavefrontsize32">;
+def __builtin_amdgcn_ds_load_tr6_b96_v3i32 : AMDGPUBuiltin<"_Vector<3, int>(_Vector<3, int> address_space<3> *)", [Const], "transpose-load-f4f6-insts,wavefrontsize32">;
+def __builtin_amdgcn_ds_load_tr16_b128_v8i16 : AMDGPUBuiltin<"_Vector<8, short>(_Vector<8, short> address_space<3> *)", [Const], "gfx1250-insts,wavefrontsize32">;
+def __builtin_amdgcn_ds_load_tr16_b128_v8f16 : AMDGPUBuiltin<"_Vector<8, __fp16>(_Vector<8, __fp16> address_space<3> *)", [Const], "gfx1250-insts,wavefrontsize32">;
+def __builtin_amdgcn_ds_load_tr16_b128_v8bf16 : AMDGPUBuiltin<"_Vector<8, __bf16>(_Vector<8, __bf16> address_space<3> *)", [Const], "gfx1250-insts,wavefrontsize32">;
def __builtin_amdgcn_s_setprio_inc_wg : AMDGPUBuiltin<"void(_Constant short)", [], "setprio-inc-wg-inst">;
def __builtin_amdgcn_s_monitor_sleep : AMDGPUBuiltin<"void(_Constant short)", [], "gfx1250-insts">;
@@ -1055,61 +1055,61 @@ def __builtin_amdgcn_exp2_bf16 : AMDGPUBuiltin<"__bf16(__bf16)", [Const], "bf16-
def __builtin_amdgcn_sin_bf16 : AMDGPUBuiltin<"__bf16(__bf16)", [Const], "bf16-trans-insts">;
def __builtin_amdgcn_cos_bf16 : AMDGPUBuiltin<"__bf16(__bf16)", [Const], "bf16-trans-insts">;
-def __builtin_amdgcn_cvt_sr_pk_bf16_f32 : AMDGPUBuiltin<"_ExtVector<2, __bf16>(float, float, int)", [Const], "cvt-sr-pk-bf16-f32-inst">;
-def __builtin_amdgcn_cvt_sr_pk_f16_f32 : AMDGPUBuiltin<"_ExtVector<2, _Float16>(float, float, int)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_sr_pk_bf16_f32 : AMDGPUBuiltin<"_Vector<2, __bf16>(float, float, int)", [Const], "cvt-sr-pk-bf16-f32-inst">;
+def __builtin_amdgcn_cvt_sr_pk_f16_f32 : AMDGPUBuiltin<"_Vector<2, _Float16>(float, float, int)", [Const], "gfx1250-insts">;
def __builtin_amdgcn_cvt_f16_fp8 : AMDGPUBuiltin<"_Float16(int, _Constant int)", [Const], "gfx1250-insts">;
def __builtin_amdgcn_cvt_f16_bf8 : AMDGPUBuiltin<"_Float16(int, _Constant int)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_pk_f16_fp8 : AMDGPUBuiltin<"_ExtVector<2, _Float16>(short)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_pk_f16_bf8 : AMDGPUBuiltin<"_ExtVector<2, _Float16>(short)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_pk_fp8_f16 : AMDGPUBuiltin<"short(_ExtVector<2, _Float16>)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_pk_bf8_f16 : AMDGPUBuiltin<"short(_ExtVector<2, _Float16>)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_pk_f16_fp8 : AMDGPUBuiltin<"_Vector<2, _Float16>(short)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_pk_f16_bf8 : AMDGPUBuiltin<"_Vector<2, _Float16>(short)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_pk_fp8_f16 : AMDGPUBuiltin<"short(_Vector<2, _Float16>)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_pk_bf8_f16 : AMDGPUBuiltin<"short(_Vector<2, _Float16>)", [Const], "gfx1250-insts">;
def __builtin_amdgcn_cvt_sr_fp8_f16 : AMDGPUBuiltin<"int(_Float16, int, unsigned int, _Constant int)", [Const], "gfx1250-insts">;
def __builtin_amdgcn_cvt_sr_bf8_f16 : AMDGPUBuiltin<"int(_Float16, int, unsigned int, _Constant int)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scale_pk8_f16_fp8 : AMDGPUBuiltin<"_ExtVector<8, _Float16>(_ExtVector<2, unsigned int>, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scale_pk8_bf16_fp8 : AMDGPUBuiltin<"_ExtVector<8, __bf16>(_ExtVector<2, unsigned int>, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scale_pk8_f16_bf8 : AMDGPUBuiltin<"_ExtVector<8, _Float16>(_ExtVector<2, unsigned int>, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scale_pk8_bf16_bf8 : AMDGPUBuiltin<"_ExtVector<8, __bf16>(_ExtVector<2, unsigned int>, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scale_pk8_f16_fp4 : AMDGPUBuiltin<"_ExtVector<8, _Float16>(unsigned int, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scale_pk8_bf16_fp4 : AMDGPUBuiltin<"_ExtVector<8, __bf16>(unsigned int, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scale_pk8_f32_fp8 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<2, unsigned int>, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scale_pk8_f32_bf8 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<2, unsigned int>, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scale_pk8_f32_fp4 : AMDGPUBuiltin<"_ExtVector<8, float>(unsigned int, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scale_pk16_f16_fp6 : AMDGPUBuiltin<"_ExtVector<16, _Float16>(_ExtVector<3, unsigned int>, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scale_pk16_bf16_fp6 : AMDGPUBuiltin<"_ExtVector<16, __bf16>(_ExtVector<3, unsigned int>, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scale_pk16_f16_bf6 : AMDGPUBuiltin<"_ExtVector<16, _Float16>(_ExtVector<3, unsigned int>, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scale_pk16_bf16_bf6 : AMDGPUBuiltin<"_ExtVector<16, __bf16>(_ExtVector<3, unsigned int>, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scale_pk16_f32_fp6 : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<3, unsigned int>, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scale_pk16_f32_bf6 : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<3, unsigned int>, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk8_fp8_bf16 : AMDGPUBuiltin<"_ExtVector<2, unsigned int>(_ExtVector<8, __bf16>, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk8_bf8_bf16 : AMDGPUBuiltin<"_ExtVector<2, unsigned int>(_ExtVector<8, __bf16>, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk8_fp8_f16 : AMDGPUBuiltin<"_ExtVector<2, unsigned int>(_ExtVector<8, _Float16>, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk8_bf8_f16 : AMDGPUBuiltin<"_ExtVector<2, unsigned int>(_ExtVector<8, _Float16>, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk8_fp8_f32 : AMDGPUBuiltin<"_ExtVector<2, unsigned int>(_ExtVector<8, float>, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk8_bf8_f32 : AMDGPUBuiltin<"_ExtVector<2, unsigned int>(_ExtVector<8, float>, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk8_fp4_f32 : AMDGPUBuiltin<"unsigned int(_ExtVector<8, float>, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk8_fp4_f16 : AMDGPUBuiltin<"unsigned int(_ExtVector<8, _Float16>, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk8_fp4_bf16 : AMDGPUBuiltin<"unsigned int(_ExtVector<8, __bf16>, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk16_fp6_f32 : AMDGPUBuiltin<"_ExtVector<3, unsigned int>(_ExtVector<16, float>, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk16_bf6_f32 : AMDGPUBuiltin<"_ExtVector<3, unsigned int>(_ExtVector<16, float>, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk16_fp6_f16 : AMDGPUBuiltin<"_ExtVector<3, unsigned int>(_ExtVector<16, _Float16>, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk16_bf6_f16 : AMDGPUBuiltin<"_ExtVector<3, unsigned int>(_ExtVector<16, _Float16>, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk16_fp6_bf16 : AMDGPUBuiltin<"_ExtVector<3, unsigned int>(_ExtVector<16, __bf16>, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_pk16_bf6_bf16 : AMDGPUBuiltin<"_ExtVector<3, unsigned int>(_ExtVector<16, __bf16>, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_sr_pk8_fp8_bf16 : AMDGPUBuiltin<"_ExtVector<2, unsigned int>(_ExtVector<8, __bf16>, unsigned int, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_sr_pk8_bf8_bf16 : AMDGPUBuiltin<"_ExtVector<2, unsigned int>(_ExtVector<8, __bf16>, unsigned int, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_sr_pk8_fp8_f16 : AMDGPUBuiltin<"_ExtVector<2, unsigned int>(_ExtVector<8, _Float16>, unsigned int, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_sr_pk8_bf8_f16 : AMDGPUBuiltin<"_ExtVector<2, unsigned int>(_ExtVector<8, _Float16>, unsigned int, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_sr_pk8_fp8_f32 : AMDGPUBuiltin<"_ExtVector<2, unsigned int>(_ExtVector<8, float>, unsigned int, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_sr_pk8_bf8_f32 : AMDGPUBuiltin<"_ExtVector<2, unsigned int>(_ExtVector<8, float>, unsigned int, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_sr_pk8_fp4_f32 : AMDGPUBuiltin<"unsigned int(_ExtVector<8, float>, unsigned int, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_sr_pk8_fp4_f16 : AMDGPUBuiltin<"unsigned int(_ExtVector<8, _Float16>, unsigned int, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_sr_pk8_fp4_bf16 : AMDGPUBuiltin<"unsigned int(_ExtVector<8, __bf16>, unsigned int, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_sr_pk16_bf6_bf16 : AMDGPUBuiltin<"_ExtVector<3, unsigned int>(_ExtVector<16, __bf16>, unsigned int, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_sr_pk16_bf6_f16 : AMDGPUBuiltin<"_ExtVector<3, unsigned int>(_ExtVector<16, _Float16>, unsigned int, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_sr_pk16_bf6_f32 : AMDGPUBuiltin<"_ExtVector<3, unsigned int>(_ExtVector<16, float>, unsigned int, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_sr_pk16_fp6_bf16 : AMDGPUBuiltin<"_ExtVector<3, unsigned int>(_ExtVector<16, __bf16>, unsigned int, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_sr_pk16_fp6_f16 : AMDGPUBuiltin<"_ExtVector<3, unsigned int>(_ExtVector<16, _Float16>, unsigned int, float)", [Const], "gfx1250-insts">;
-def __builtin_amdgcn_cvt_scalef32_sr_pk16_fp6_f32 : AMDGPUBuiltin<"_ExtVector<3, unsigned int>(_ExtVector<16, float>, unsigned int, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scale_pk8_f16_fp8 : AMDGPUBuiltin<"_Vector<8, _Float16>(_Vector<2, unsigned int>, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scale_pk8_bf16_fp8 : AMDGPUBuiltin<"_Vector<8, __bf16>(_Vector<2, unsigned int>, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scale_pk8_f16_bf8 : AMDGPUBuiltin<"_Vector<8, _Float16>(_Vector<2, unsigned int>, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scale_pk8_bf16_bf8 : AMDGPUBuiltin<"_Vector<8, __bf16>(_Vector<2, unsigned int>, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scale_pk8_f16_fp4 : AMDGPUBuiltin<"_Vector<8, _Float16>(unsigned int, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scale_pk8_bf16_fp4 : AMDGPUBuiltin<"_Vector<8, __bf16>(unsigned int, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scale_pk8_f32_fp8 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<2, unsigned int>, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scale_pk8_f32_bf8 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<2, unsigned int>, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scale_pk8_f32_fp4 : AMDGPUBuiltin<"_Vector<8, float>(unsigned int, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scale_pk16_f16_fp6 : AMDGPUBuiltin<"_Vector<16, _Float16>(_Vector<3, unsigned int>, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scale_pk16_bf16_fp6 : AMDGPUBuiltin<"_Vector<16, __bf16>(_Vector<3, unsigned int>, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scale_pk16_f16_bf6 : AMDGPUBuiltin<"_Vector<16, _Float16>(_Vector<3, unsigned int>, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scale_pk16_bf16_bf6 : AMDGPUBuiltin<"_Vector<16, __bf16>(_Vector<3, unsigned int>, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scale_pk16_f32_fp6 : AMDGPUBuiltin<"_Vector<16, float>(_Vector<3, unsigned int>, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scale_pk16_f32_bf6 : AMDGPUBuiltin<"_Vector<16, float>(_Vector<3, unsigned int>, unsigned int, _Constant unsigned int)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk8_fp8_bf16 : AMDGPUBuiltin<"_Vector<2, unsigned int>(_Vector<8, __bf16>, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk8_bf8_bf16 : AMDGPUBuiltin<"_Vector<2, unsigned int>(_Vector<8, __bf16>, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk8_fp8_f16 : AMDGPUBuiltin<"_Vector<2, unsigned int>(_Vector<8, _Float16>, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk8_bf8_f16 : AMDGPUBuiltin<"_Vector<2, unsigned int>(_Vector<8, _Float16>, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk8_fp8_f32 : AMDGPUBuiltin<"_Vector<2, unsigned int>(_Vector<8, float>, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk8_bf8_f32 : AMDGPUBuiltin<"_Vector<2, unsigned int>(_Vector<8, float>, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk8_fp4_f32 : AMDGPUBuiltin<"unsigned int(_Vector<8, float>, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk8_fp4_f16 : AMDGPUBuiltin<"unsigned int(_Vector<8, _Float16>, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk8_fp4_bf16 : AMDGPUBuiltin<"unsigned int(_Vector<8, __bf16>, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk16_fp6_f32 : AMDGPUBuiltin<"_Vector<3, unsigned int>(_Vector<16, float>, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk16_bf6_f32 : AMDGPUBuiltin<"_Vector<3, unsigned int>(_Vector<16, float>, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk16_fp6_f16 : AMDGPUBuiltin<"_Vector<3, unsigned int>(_Vector<16, _Float16>, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk16_bf6_f16 : AMDGPUBuiltin<"_Vector<3, unsigned int>(_Vector<16, _Float16>, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk16_fp6_bf16 : AMDGPUBuiltin<"_Vector<3, unsigned int>(_Vector<16, __bf16>, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_pk16_bf6_bf16 : AMDGPUBuiltin<"_Vector<3, unsigned int>(_Vector<16, __bf16>, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_sr_pk8_fp8_bf16 : AMDGPUBuiltin<"_Vector<2, unsigned int>(_Vector<8, __bf16>, unsigned int, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_sr_pk8_bf8_bf16 : AMDGPUBuiltin<"_Vector<2, unsigned int>(_Vector<8, __bf16>, unsigned int, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_sr_pk8_fp8_f16 : AMDGPUBuiltin<"_Vector<2, unsigned int>(_Vector<8, _Float16>, unsigned int, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_sr_pk8_bf8_f16 : AMDGPUBuiltin<"_Vector<2, unsigned int>(_Vector<8, _Float16>, unsigned int, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_sr_pk8_fp8_f32 : AMDGPUBuiltin<"_Vector<2, unsigned int>(_Vector<8, float>, unsigned int, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_sr_pk8_bf8_f32 : AMDGPUBuiltin<"_Vector<2, unsigned int>(_Vector<8, float>, unsigned int, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_sr_pk8_fp4_f32 : AMDGPUBuiltin<"unsigned int(_Vector<8, float>, unsigned int, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_sr_pk8_fp4_f16 : AMDGPUBuiltin<"unsigned int(_Vector<8, _Float16>, unsigned int, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_sr_pk8_fp4_bf16 : AMDGPUBuiltin<"unsigned int(_Vector<8, __bf16>, unsigned int, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_sr_pk16_bf6_bf16 : AMDGPUBuiltin<"_Vector<3, unsigned int>(_Vector<16, __bf16>, unsigned int, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_sr_pk16_bf6_f16 : AMDGPUBuiltin<"_Vector<3, unsigned int>(_Vector<16, _Float16>, unsigned int, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_sr_pk16_bf6_f32 : AMDGPUBuiltin<"_Vector<3, unsigned int>(_Vector<16, float>, unsigned int, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_sr_pk16_fp6_bf16 : AMDGPUBuiltin<"_Vector<3, unsigned int>(_Vector<16, __bf16>, unsigned int, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_sr_pk16_fp6_f16 : AMDGPUBuiltin<"_Vector<3, unsigned int>(_Vector<16, _Float16>, unsigned int, float)", [Const], "gfx1250-insts">;
+def __builtin_amdgcn_cvt_scalef32_sr_pk16_fp6_f32 : AMDGPUBuiltin<"_Vector<3, unsigned int>(_Vector<16, float>, unsigned int, float)", [Const], "gfx1250-insts">;
def __builtin_amdgcn_cvt_pk_fp8_f32_e5m3 : AMDGPUBuiltin<"int(float, float, int, _Constant bool)", [Const], "fp8e5m3-insts">;
def __builtin_amdgcn_cvt_sr_fp8_f32_e5m3 : AMDGPUBuiltin<"int(float, int, int, _Constant int)", [Const], "fp8e5m3-insts">;
def __builtin_amdgcn_sat_pk4_i4_i8 : AMDGPUBuiltin<"unsigned short(unsigned int)", [Const], "gfx1250-insts">;
@@ -1121,153 +1121,153 @@ def __builtin_amdgcn_permlane_down : AMDGPUBuiltin<"int(int, int, int)", [Const]
def __builtin_amdgcn_permlane_xor : AMDGPUBuiltin<"int(int, int, int)", [Const], "gfx1250-insts,wavefrontsize32">;
def __builtin_amdgcn_permlane_idx_gen : AMDGPUBuiltin<"int(int, int)", [Const], "gfx1250-insts,wavefrontsize32">;
-def __builtin_amdgcn_perm_pk16_b4_u4 : AMDGPUBuiltin<"_ExtVector<2, unsigned int>(unsigned int, unsigned int, _ExtVector<2, unsigned int>)", [Const], "tensor-cvt-lut-insts">;
-def __builtin_amdgcn_perm_pk16_b6_u4 : AMDGPUBuiltin<"_ExtVector<3, unsigned int>(unsigned int, unsigned long int, _ExtVector<2, unsigned int>)", [Const], "tensor-cvt-lut-insts">;
-def __builtin_amdgcn_perm_pk16_b8_u4 : AMDGPUBuiltin<"_ExtVector<4, unsigned int>(unsigned long int, unsigned long int, _ExtVector<2, unsigned int>)", [Const], "tensor-cvt-lut-insts">;
+def __builtin_amdgcn_perm_pk16_b4_u4 : AMDGPUBuiltin<"_Vector<2, unsigned int>(unsigned int, unsigned int, _Vector<2, unsigned int>)", [Const], "tensor-cvt-lut-insts">;
+def __builtin_amdgcn_perm_pk16_b6_u4 : AMDGPUBuiltin<"_Vector<3, unsigned int>(unsigned int, unsigned long int, _Vector<2, unsigned int>)", [Const], "tensor-cvt-lut-insts">;
+def __builtin_amdgcn_perm_pk16_b8_u4 : AMDGPUBuiltin<"_Vector<4, unsigned int>(unsigned long int, unsigned long int, _Vector<2, unsigned int>)", [Const], "tensor-cvt-lut-insts">;
def __builtin_amdgcn_add_max_i32 : AMDGPUBuiltin<"int(int, int, int, _Constant bool)", [Const], "add-min-max-insts">;
def __builtin_amdgcn_add_max_u32 : AMDGPUBuiltin<"unsigned int(unsigned int, unsigned int, unsigned int, _Constant bool)", [Const], "add-min-max-insts">;
def __builtin_amdgcn_add_min_i32 : AMDGPUBuiltin<"int(int, int, int, _Constant bool)", [Const], "add-min-max-insts">;
def __builtin_amdgcn_add_min_u32 : AMDGPUBuiltin<"unsigned int(unsigned int, unsigned int, unsigned int, _Constant bool)", [Const], "add-min-max-insts">;
-def __builtin_amdgcn_pk_add_max_i16 : AMDGPUBuiltin<"_ExtVector<2, short>(_ExtVector<2, short>, _ExtVector<2, short>, _ExtVector<2, short>, _Constant bool)", [Const], "pk-add-min-max-insts">;
-def __builtin_amdgcn_pk_add_max_u16 : AMDGPUBuiltin<"_ExtVector<2, unsigned short>(_ExtVector<2, unsigned short>, _ExtVector<2, unsigned short>, _ExtVector<2, unsigned short>, _Constant bool)", [Const], "pk-add-min-max-insts">;
-def __builtin_amdgcn_pk_add_min_i16 : AMDGPUBuiltin<"_ExtVector<2, short>(_ExtVector<2, short>, _ExtVector<2, short>, _ExtVector<2, short>, _Constant bool)", [Const], "pk-add-min-max-insts">;
-def __builtin_amdgcn_pk_add_min_u16 : AMDGPUBuiltin<"_ExtVector<2, unsigned short>(_ExtVector<2, unsigned short>, _ExtVector<2, unsigned short>, _ExtVector<2, unsigned short>, _Constant bool)", [Const], "pk-add-min-max-insts">;
+def __builtin_amdgcn_pk_add_max_i16 : AMDGPUBuiltin<"_Vector<2, short>(_Vector<2, short>, _Vector<2, short>, _Vector<2, short>, _Constant bool)", [Const], "pk-add-min-max-insts">;
+def __builtin_amdgcn_pk_add_max_u16 : AMDGPUBuiltin<"_Vector<2, unsigned short>(_Vector<2, unsigned short>, _Vector<2, unsigned short>, _Vector<2, unsigned short>, _Constant bool)", [Const], "pk-add-min-max-insts">;
+def __builtin_amdgcn_pk_add_min_i16 : AMDGPUBuiltin<"_Vector<2, short>(_Vector<2, short>, _Vector<2, short>, _Vector<2, short>, _Constant bool)", [Const], "pk-add-min-max-insts">;
+def __builtin_amdgcn_pk_add_min_u16 : AMDGPUBuiltin<"_Vector<2, unsigned short>(_Vector<2, unsigned short>, _Vector<2, unsigned short>, _Vector<2, unsigned short>, _Constant bool)", [Const], "pk-add-min-max-insts">;
// GFX1250 WMMA builtins
-def __builtin_amdgcn_wmma_f32_16x16x4_f32 : AMDGPUBuiltin<"_ExtVector<8, float>(_Constant bool, _ExtVector<2, float>, _Constant bool, _ExtVector<2, float>, _Constant short, _ExtVector<8, float>, _Constant bool, _Constant bool)", [Const], "gfx1250-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f32_16x16x4_f32 : AMDGPUBuiltin<"_Vector<8, float>(_Constant bool, _Vector<2, float>, _Constant bool, _Vector<2, float>, _Constant short, _Vector<8, float>, _Constant bool, _Constant bool)", [Const], "gfx1250-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_f32_16x16x4_f32_GFX1250];
let ArgNames = ["a_neg", "a", "b_neg", "b", "c_mod", "c", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_wmma_f32_16x16x32_bf16 : AMDGPUBuiltin<"_ExtVector<8, float>(_Constant bool, _ExtVector<16, __bf16>, _Constant bool, _ExtVector<16, __bf16>, _Constant short, _ExtVector<8, float>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f32_16x16x32_bf16 : AMDGPUBuiltin<"_Vector<8, float>(_Constant bool, _Vector<16, __bf16>, _Constant bool, _Vector<16, __bf16>, _Constant short, _Vector<8, float>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_f32_16x16x32_GFX1250];
let ArgNames = ["a_neg", "a", "b_neg", "b", "c_mod", "c", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_wmma_bf16_16x16x32_bf16 : AMDGPUBuiltin<"_ExtVector<8, __bf16>(_Constant bool, _ExtVector<16, __bf16>, _Constant bool, _ExtVector<16, __bf16>, _Constant short, _ExtVector<8, __bf16>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_bf16_16x16x32_bf16 : AMDGPUBuiltin<"_Vector<8, __bf16>(_Constant bool, _Vector<16, __bf16>, _Constant bool, _Vector<16, __bf16>, _Constant short, _Vector<8, __bf16>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_half_16x16x32_GFX1250];
let ArgNames = ["a_neg", "a", "b_neg", "b", "c_mod", "c", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_wmma_bf16f32_16x16x32_bf16 : AMDGPUBuiltin<"_ExtVector<8, __bf16>(_Constant bool, _ExtVector<16, __bf16>, _Constant bool, _ExtVector<16, __bf16>, _Constant short, _ExtVector<8, float>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_bf16f32_16x16x32_bf16 : AMDGPUBuiltin<"_Vector<8, __bf16>(_Constant bool, _Vector<16, __bf16>, _Constant bool, _Vector<16, __bf16>, _Constant short, _Vector<8, float>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_bf16f32_16x16x32_GFX1250];
let ArgNames = ["a_neg", "a", "b_neg", "b", "c_mod", "c", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_wmma_f32_16x16x64_fp8_fp8 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<8, int>, _ExtVector<8, int>, _Constant short, _ExtVector<8, float>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f32_16x16x64_fp8_fp8 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<8, int>, _Vector<8, int>, _Constant short, _Vector<8, float>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_fp8_16x16x64_GFX1250];
let ArgNames = ["a", "b", "c_mod", "c", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_wmma_f32_16x16x64_fp8_bf8 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<8, int>, _ExtVector<8, int>, _Constant short, _ExtVector<8, float>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f32_16x16x64_fp8_bf8 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<8, int>, _Vector<8, int>, _Constant short, _Vector<8, float>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_fp8_16x16x64_GFX1250];
let ArgNames = ["a", "b", "c_mod", "c", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_wmma_f32_16x16x64_bf8_fp8 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<8, int>, _ExtVector<8, int>, _Constant short, _ExtVector<8, float>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f32_16x16x64_bf8_fp8 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<8, int>, _Vector<8, int>, _Constant short, _Vector<8, float>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_fp8_16x16x64_GFX1250];
let ArgNames = ["a", "b", "c_mod", "c", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_wmma_f32_16x16x64_bf8_bf8 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<8, int>, _ExtVector<8, int>, _Constant short, _ExtVector<8, float>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f32_16x16x64_bf8_bf8 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<8, int>, _Vector<8, int>, _Constant short, _Vector<8, float>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_fp8_16x16x64_GFX1250];
let ArgNames = ["a", "b", "c_mod", "c", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_wmma_f16_16x16x64_fp8_fp8 : AMDGPUBuiltin<"_ExtVector<8, _Float16>(_ExtVector<8, int>, _ExtVector<8, int>, _Constant short, _ExtVector<8, _Float16>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f16_16x16x64_fp8_fp8 : AMDGPUBuiltin<"_Vector<8, _Float16>(_Vector<8, int>, _Vector<8, int>, _Constant short, _Vector<8, _Float16>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_fp8_16x16x64_GFX1250];
let ArgNames = ["a", "b", "c_mod", "c", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_wmma_f16_16x16x64_fp8_bf8 : AMDGPUBuiltin<"_ExtVector<8, _Float16>(_ExtVector<8, int>, _ExtVector<8, int>, _Constant short, _ExtVector<8, _Float16>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f16_16x16x64_fp8_bf8 : AMDGPUBuiltin<"_Vector<8, _Float16>(_Vector<8, int>, _Vector<8, int>, _Constant short, _Vector<8, _Float16>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_fp8_16x16x64_GFX1250];
let ArgNames = ["a", "b", "c_mod", "c", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_wmma_f16_16x16x64_bf8_fp8 : AMDGPUBuiltin<"_ExtVector<8, _Float16>(_ExtVector<8, int>, _ExtVector<8, int>, _Constant short, _ExtVector<8, _Float16>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f16_16x16x64_bf8_fp8 : AMDGPUBuiltin<"_Vector<8, _Float16>(_Vector<8, int>, _Vector<8, int>, _Constant short, _Vector<8, _Float16>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_fp8_16x16x64_GFX1250];
let ArgNames = ["a", "b", "c_mod", "c", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_wmma_f16_16x16x64_bf8_bf8 : AMDGPUBuiltin<"_ExtVector<8, _Float16>(_ExtVector<8, int>, _ExtVector<8, int>, _Constant short, _ExtVector<8, _Float16>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f16_16x16x64_bf8_bf8 : AMDGPUBuiltin<"_Vector<8, _Float16>(_Vector<8, int>, _Vector<8, int>, _Constant short, _Vector<8, _Float16>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_fp8_16x16x64_GFX1250];
let ArgNames = ["a", "b", "c_mod", "c", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_wmma_i32_16x16x64_iu8 : AMDGPUBuiltin<"_ExtVector<8, int>(_Constant bool, _ExtVector<8, int>, _Constant bool, _ExtVector<8, int>, _ExtVector<8, int>, _Constant bool, _Constant bool, ...)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_i32_16x16x64_iu8 : AMDGPUBuiltin<"_Vector<8, int>(_Constant bool, _Vector<8, int>, _Constant bool, _Vector<8, int>, _Vector<8, int>, _Constant bool, _Constant bool, ...)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_i32_16x16x64_iu8_GFX1250];
let ArgNames = ["a_sign", "a", "b_sign", "b", "c", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_wmma_f16_16x16x128_fp8_fp8 : AMDGPUBuiltin<"_ExtVector<8, _Float16>(_ExtVector<16, int>, _ExtVector<16, int>, _Constant short, _ExtVector<8, _Float16>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f16_16x16x128_fp8_fp8 : AMDGPUBuiltin<"_Vector<8, _Float16>(_Vector<16, int>, _Vector<16, int>, _Constant short, _Vector<8, _Float16>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_fp8_16x16x128_GFX1250];
let ArgNames = ["a", "b", "c_mod", "c", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_wmma_f16_16x16x128_fp8_bf8 : AMDGPUBuiltin<"_ExtVector<8, _Float16>(_ExtVector<16, int>, _ExtVector<16, int>, _Constant short, _ExtVector<8, _Float16>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f16_16x16x128_fp8_bf8 : AMDGPUBuiltin<"_Vector<8, _Float16>(_Vector<16, int>, _Vector<16, int>, _Constant short, _Vector<8, _Float16>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_fp8_16x16x128_GFX1250];
let ArgNames = ["a", "b", "c_mod", "c", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_wmma_f16_16x16x128_bf8_fp8 : AMDGPUBuiltin<"_ExtVector<8, _Float16>(_ExtVector<16, int>, _ExtVector<16, int>, _Constant short, _ExtVector<8, _Float16>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f16_16x16x128_bf8_fp8 : AMDGPUBuiltin<"_Vector<8, _Float16>(_Vector<16, int>, _Vector<16, int>, _Constant short, _Vector<8, _Float16>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_fp8_16x16x128_GFX1250];
let ArgNames = ["a", "b", "c_mod", "c", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_wmma_f16_16x16x128_bf8_bf8 : AMDGPUBuiltin<"_ExtVector<8, _Float16>(_ExtVector<16, int>, _ExtVector<16, int>, _Constant short, _ExtVector<8, _Float16>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f16_16x16x128_bf8_bf8 : AMDGPUBuiltin<"_Vector<8, _Float16>(_Vector<16, int>, _Vector<16, int>, _Constant short, _Vector<8, _Float16>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_fp8_16x16x128_GFX1250];
let ArgNames = ["a", "b", "c_mod", "c", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_wmma_f32_16x16x128_f8f6f4 : AMDGPUBuiltin<"_ExtVector<8, float>(_Constant int, _ExtVector<16, int>, _Constant int, _ExtVector<16, int>, _Constant short, _ExtVector<8, float>)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f32_16x16x128_f8f6f4 : AMDGPUBuiltin<"_Vector<8, float>(_Constant int, _Vector<16, int>, _Constant int, _Vector<16, int>, _Constant short, _Vector<8, float>)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_f8f6f4_GFX1250];
let ArgNames = ["matrix_a_fmt", "a", "matrix_b_fmt", "b", "c_mod", "c"];
}
-def __builtin_amdgcn_wmma_f32_16x16x128_fp8_fp8 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<16, int>, _ExtVector<16, int>, _Constant short, _ExtVector<8, float>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f32_16x16x128_fp8_fp8 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<16, int>, _Vector<16, int>, _Constant short, _Vector<8, float>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_fp8_16x16x128_GFX1250];
let ArgNames = ["a", "b", "c_mod", "c", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_wmma_f32_16x16x128_fp8_bf8 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<16, int>, _ExtVector<16, int>, _Constant short, _ExtVector<8, float>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f32_16x16x128_fp8_bf8 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<16, int>, _Vector<16, int>, _Constant short, _Vector<8, float>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_fp8_16x16x128_GFX1250];
let ArgNames = ["a", "b", "c_mod", "c", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_wmma_f32_16x16x128_bf8_fp8 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<16, int>, _ExtVector<16, int>, _Constant short, _ExtVector<8, float>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f32_16x16x128_bf8_fp8 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<16, int>, _Vector<16, int>, _Constant short, _Vector<8, float>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_fp8_16x16x128_GFX1250];
let ArgNames = ["a", "b", "c_mod", "c", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_wmma_f32_16x16x128_bf8_bf8 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<16, int>, _ExtVector<16, int>, _Constant short, _ExtVector<8, float>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f32_16x16x128_bf8_bf8 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<16, int>, _Vector<16, int>, _Constant short, _Vector<8, float>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_fp8_16x16x128_GFX1250];
let ArgNames = ["a", "b", "c_mod", "c", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_wmma_scale_f32_16x16x128_f8f6f4 : AMDGPUBuiltin<"_ExtVector<8, float>(_Constant int, _ExtVector<16, int>, _Constant int, _ExtVector<16, int>, _Constant short, _ExtVector<8, float>, _Constant int, _Constant int, int, _Constant int, _Constant int, int, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_scale_f32_16x16x128_f8f6f4 : AMDGPUBuiltin<"_Vector<8, float>(_Constant int, _Vector<16, int>, _Constant int, _Vector<16, int>, _Constant short, _Vector<8, float>, _Constant int, _Constant int, int, _Constant int, _Constant int, int, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_scale_GFX1250];
let ArgNames = ["matrix_a_fmt", "a", "matrix_b_fmt", "b", "c_mod", "c", "matrix_a_scale", "matrix_a_scale_fmt", "matrix_a_scale_exp", "matrix_b_scale", "matrix_b_scale_fmt", "matrix_b_scale_exp", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_wmma_scale16_f32_16x16x128_f8f6f4 : AMDGPUBuiltin<"_ExtVector<8, float>(_Constant int, _ExtVector<16, int>, _Constant int, _ExtVector<16, int>, _Constant short, _ExtVector<8, float>, _Constant int, _Constant int, long int, _Constant int, _Constant int, long int, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_scale16_f32_16x16x128_f8f6f4 : AMDGPUBuiltin<"_Vector<8, float>(_Constant int, _Vector<16, int>, _Constant int, _Vector<16, int>, _Constant short, _Vector<8, float>, _Constant int, _Constant int, long int, _Constant int, _Constant int, long int, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_scale16_GFX1250];
let ArgNames = ["matrix_a_fmt", "a", "matrix_b_fmt", "b", "c_mod", "c", "matrix_a_scale", "matrix_a_scale_fmt", "matrix_a_scale_exp", "matrix_b_scale", "matrix_b_scale_fmt", "matrix_b_scale_exp", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_wmma_f32_16x16x32_f16 : AMDGPUBuiltin<"_ExtVector<8, float>(_Constant bool, _ExtVector<16, _Float16>, _Constant bool, _ExtVector<16, _Float16>, _Constant short, _ExtVector<8, float>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f32_16x16x32_f16 : AMDGPUBuiltin<"_Vector<8, float>(_Constant bool, _Vector<16, _Float16>, _Constant bool, _Vector<16, _Float16>, _Constant short, _Vector<8, float>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_f32_16x16x32_GFX1250];
let ArgNames = ["a_neg", "a", "b_neg", "b", "c_mod", "c", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_wmma_f16_16x16x32_f16 : AMDGPUBuiltin<"_ExtVector<8, _Float16>(_Constant bool, _ExtVector<16, _Float16>, _Constant bool, _ExtVector<16, _Float16>, _Constant short, _ExtVector<8, _Float16>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f16_16x16x32_f16 : AMDGPUBuiltin<"_Vector<8, _Float16>(_Constant bool, _Vector<16, _Float16>, _Constant bool, _Vector<16, _Float16>, _Constant short, _Vector<8, _Float16>, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_half_16x16x32_GFX1250];
let ArgNames = ["a_neg", "a", "b_neg", "b", "c_mod", "c", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_wmma_f32_32x16x128_f4 : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<16, int>, _ExtVector<8, int>, _Constant short, _ExtVector<16, float>)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f32_32x16x128_f4 : AMDGPUBuiltin<"_Vector<16, float>(_Vector<16, int>, _Vector<8, int>, _Constant short, _Vector<16, float>)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_f4_GFX1250];
let ArgNames = ["a", "b", "c_mod", "c"];
}
-def __builtin_amdgcn_wmma_scale_f32_32x16x128_f4 : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<16, int>, _ExtVector<8, int>, _Constant short, _ExtVector<16, float>, _Constant int, _Constant int, int, _Constant int, _Constant int, int, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_scale_f32_32x16x128_f4 : AMDGPUBuiltin<"_Vector<16, float>(_Vector<16, int>, _Vector<8, int>, _Constant short, _Vector<16, float>, _Constant int, _Constant int, int, _Constant int, _Constant int, int, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_scale_GFX1250];
let ArgNames = ["a", "b", "c_mod", "c", "matrix_a_scale", "matrix_a_scale_fmt", "matrix_a_scale_exp", "matrix_b_scale", "matrix_b_scale_fmt", "matrix_b_scale_exp", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_wmma_scale16_f32_32x16x128_f4 : AMDGPUBuiltin<"_ExtVector<16, float>(_ExtVector<16, int>, _ExtVector<8, int>, _Constant short, _ExtVector<16, float>, _Constant int, _Constant int, long int, _Constant int, _Constant int, long int, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_scale16_f32_32x16x128_f4 : AMDGPUBuiltin<"_Vector<16, float>(_Vector<16, int>, _Vector<8, int>, _Constant short, _Vector<16, float>, _Constant int, _Constant int, long int, _Constant int, _Constant int, long int, _Constant bool, _Constant bool)", [Const], "wmma-n16-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_scale16_GFX1250];
let ArgNames = ["a", "b", "c_mod", "c", "matrix_a_scale", "matrix_a_scale_fmt", "matrix_a_scale_exp", "matrix_b_scale", "matrix_b_scale_fmt", "matrix_b_scale_exp", "matrix_a_reuse", "matrix_b_reuse"];
}
-def __builtin_amdgcn_swmmac_f32_16x16x64_bf16 : AMDGPUBuiltin<"_ExtVector<8, float>(_Constant bool, _ExtVector<16, __bf16>, _Constant bool, _ExtVector<32, __bf16>, _ExtVector<8, float>, int, _Constant bool, _Constant bool)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
-def __builtin_amdgcn_swmmac_bf16_16x16x64_bf16 : AMDGPUBuiltin<"_ExtVector<8, __bf16>(_Constant bool, _ExtVector<16, __bf16>, _Constant bool, _ExtVector<32, __bf16>, _ExtVector<8, __bf16>, int, _Constant bool, _Constant bool)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
-def __builtin_amdgcn_swmmac_bf16f32_16x16x64_bf16 : AMDGPUBuiltin<"_ExtVector<8, float>(_Constant bool, _ExtVector<16, __bf16>, _Constant bool, _ExtVector<32, __bf16>, _ExtVector<8, float>, int, _Constant bool, _Constant bool)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
-def __builtin_amdgcn_swmmac_f32_16x16x128_fp8_fp8 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<8, int>, _ExtVector<16, int>, _ExtVector<8, float>, _ExtVector<2, int>, _Constant bool, _Constant bool)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
-def __builtin_amdgcn_swmmac_f32_16x16x128_fp8_bf8 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<8, int>, _ExtVector<16, int>, _ExtVector<8, float>, _ExtVector<2, int>, _Constant bool, _Constant bool)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
-def __builtin_amdgcn_swmmac_f32_16x16x128_bf8_fp8 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<8, int>, _ExtVector<16, int>, _ExtVector<8, float>, _ExtVector<2, int>, _Constant bool, _Constant bool)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
-def __builtin_amdgcn_swmmac_f32_16x16x128_bf8_bf8 : AMDGPUBuiltin<"_ExtVector<8, float>(_ExtVector<8, int>, _ExtVector<16, int>, _ExtVector<8, float>, _ExtVector<2, int>, _Constant bool, _Constant bool)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
-def __builtin_amdgcn_swmmac_f16_16x16x128_fp8_fp8 : AMDGPUBuiltin<"_ExtVector<8, _Float16>(_ExtVector<8, int>, _ExtVector<16, int>, _ExtVector<8, _Float16>, _ExtVector<2, int>, _Constant bool, _Constant bool)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
-def __builtin_amdgcn_swmmac_f16_16x16x128_fp8_bf8 : AMDGPUBuiltin<"_ExtVector<8, _Float16>(_ExtVector<8, int>, _ExtVector<16, int>, _ExtVector<8, _Float16>, _ExtVector<2, int>, _Constant bool, _Constant bool)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
-def __builtin_amdgcn_swmmac_f16_16x16x128_bf8_fp8 : AMDGPUBuiltin<"_ExtVector<8, _Float16>(_ExtVector<8, int>, _ExtVector<16, int>, _ExtVector<8, _Float16>, _ExtVector<2, int>, _Constant bool, _Constant bool)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
-def __builtin_amdgcn_swmmac_f16_16x16x128_bf8_bf8 : AMDGPUBuiltin<"_ExtVector<8, _Float16>(_ExtVector<8, int>, _ExtVector<16, int>, _ExtVector<8, _Float16>, _ExtVector<2, int>, _Constant bool, _Constant bool)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
-def __builtin_amdgcn_swmmac_i32_16x16x128_iu8 : AMDGPUBuiltin<"_ExtVector<8, int>(_Constant bool, _ExtVector<8, int>, _Constant bool, _ExtVector<16, int>, _ExtVector<8, int>, _ExtVector<2, int>, _Constant bool, _Constant bool, ...)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
-def __builtin_amdgcn_swmmac_f32_16x16x64_f16 : AMDGPUBuiltin<"_ExtVector<8, float>(_Constant bool, _ExtVector<16, _Float16>, _Constant bool, _ExtVector<32, _Float16>, _ExtVector<8, float>, int, _Constant bool, _Constant bool)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
-def __builtin_amdgcn_swmmac_f16_16x16x64_f16 : AMDGPUBuiltin<"_ExtVector<8, _Float16>(_Constant bool, _ExtVector<16, _Float16>, _Constant bool, _ExtVector<32, _Float16>, _ExtVector<8, _Float16>, int, _Constant bool, _Constant bool)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
+def __builtin_amdgcn_swmmac_f32_16x16x64_bf16 : AMDGPUBuiltin<"_Vector<8, float>(_Constant bool, _Vector<16, __bf16>, _Constant bool, _Vector<32, __bf16>, _Vector<8, float>, int, _Constant bool, _Constant bool)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
+def __builtin_amdgcn_swmmac_bf16_16x16x64_bf16 : AMDGPUBuiltin<"_Vector<8, __bf16>(_Constant bool, _Vector<16, __bf16>, _Constant bool, _Vector<32, __bf16>, _Vector<8, __bf16>, int, _Constant bool, _Constant bool)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
+def __builtin_amdgcn_swmmac_bf16f32_16x16x64_bf16 : AMDGPUBuiltin<"_Vector<8, float>(_Constant bool, _Vector<16, __bf16>, _Constant bool, _Vector<32, __bf16>, _Vector<8, float>, int, _Constant bool, _Constant bool)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
+def __builtin_amdgcn_swmmac_f32_16x16x128_fp8_fp8 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<8, int>, _Vector<16, int>, _Vector<8, float>, _Vector<2, int>, _Constant bool, _Constant bool)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
+def __builtin_amdgcn_swmmac_f32_16x16x128_fp8_bf8 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<8, int>, _Vector<16, int>, _Vector<8, float>, _Vector<2, int>, _Constant bool, _Constant bool)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
+def __builtin_amdgcn_swmmac_f32_16x16x128_bf8_fp8 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<8, int>, _Vector<16, int>, _Vector<8, float>, _Vector<2, int>, _Constant bool, _Constant bool)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
+def __builtin_amdgcn_swmmac_f32_16x16x128_bf8_bf8 : AMDGPUBuiltin<"_Vector<8, float>(_Vector<8, int>, _Vector<16, int>, _Vector<8, float>, _Vector<2, int>, _Constant bool, _Constant bool)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
+def __builtin_amdgcn_swmmac_f16_16x16x128_fp8_fp8 : AMDGPUBuiltin<"_Vector<8, _Float16>(_Vector<8, int>, _Vector<16, int>, _Vector<8, _Float16>, _Vector<2, int>, _Constant bool, _Constant bool)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
+def __builtin_amdgcn_swmmac_f16_16x16x128_fp8_bf8 : AMDGPUBuiltin<"_Vector<8, _Float16>(_Vector<8, int>, _Vector<16, int>, _Vector<8, _Float16>, _Vector<2, int>, _Constant bool, _Constant bool)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
+def __builtin_amdgcn_swmmac_f16_16x16x128_bf8_fp8 : AMDGPUBuiltin<"_Vector<8, _Float16>(_Vector<8, int>, _Vector<16, int>, _Vector<8, _Float16>, _Vector<2, int>, _Constant bool, _Constant bool)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
+def __builtin_amdgcn_swmmac_f16_16x16x128_bf8_bf8 : AMDGPUBuiltin<"_Vector<8, _Float16>(_Vector<8, int>, _Vector<16, int>, _Vector<8, _Float16>, _Vector<2, int>, _Constant bool, _Constant bool)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
+def __builtin_amdgcn_swmmac_i32_16x16x128_iu8 : AMDGPUBuiltin<"_Vector<8, int>(_Constant bool, _Vector<8, int>, _Constant bool, _Vector<16, int>, _Vector<8, int>, _Vector<2, int>, _Constant bool, _Constant bool, ...)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
+def __builtin_amdgcn_swmmac_f32_16x16x64_f16 : AMDGPUBuiltin<"_Vector<8, float>(_Constant bool, _Vector<16, _Float16>, _Constant bool, _Vector<32, _Float16>, _Vector<8, float>, int, _Constant bool, _Constant bool)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
+def __builtin_amdgcn_swmmac_f16_16x16x64_f16 : AMDGPUBuiltin<"_Vector<8, _Float16>(_Constant bool, _Vector<16, _Float16>, _Constant bool, _Vector<32, _Float16>, _Vector<8, _Float16>, int, _Constant bool, _Constant bool)", [Const], "swmmac-gfx1250-insts,wavefrontsize32">;
// GFX1251 WMMA builtins
-def __builtin_amdgcn_wmma_f64_16x16x4_f64 : AMDGPUBuiltin<"_ExtVector<8, double>(_Constant bool, _ExtVector<2, double>, _Constant bool, _ExtVector<2, double>, _Constant short, _ExtVector<8, double>, _Constant bool, _Constant bool)", [Const], "gfx1251-gemm-insts,wavefrontsize32"> {
+def __builtin_amdgcn_wmma_f64_16x16x4_f64 : AMDGPUBuiltin<"_Vector<8, double>(_Constant bool, _Vector<2, double>, _Constant bool, _Vector<2, double>, _Constant short, _Vector<8, double>, _Constant bool, _Constant bool)", [Const], "gfx1251-gemm-insts,wavefrontsize32"> {
let Documentation = [DocWMMA_f64_16x16x4_f64_GFX1251];
let ArgNames = ["a_neg", "a", "b_neg", "b", "c_mod", "c", "matrix_a_reuse", "matrix_b_reuse"];
}
@@ -1276,11 +1276,11 @@ def __builtin_amdgcn_wmma_f64_16x16x4_f64 : AMDGPUBuiltin<"_ExtVector<8, double>
def __builtin_amdgcn_cooperative_atomic_load_32x4B : AMDGPUBuiltin<"int(int *, _Constant int, char const *)", [Const], "gfx1250-insts,wavefrontsize32">;
def __builtin_amdgcn_cooperative_atomic_store_32x4B : AMDGPUBuiltin<"void(int *, int, _Constant int, char const *)", [Const], "gfx1250-insts,wavefrontsize32">;
-def __builtin_amdgcn_cooperative_atomic_load_16x8B : AMDGPUBuiltin<"_ExtVector<2, int>(_ExtVector<2, int> *, _Constant int, char const *)", [Const], "gfx1250-insts,wavefrontsize32">;
-def __builtin_amdgcn_cooperative_atomic_store_16x8B : AMDGPUBuiltin<"void(_ExtVector<2, int> *, _ExtVector<2, int>, _Constant int, char const *)", [Const], "gfx1250-insts,wavefrontsize32">;
+def __builtin_amdgcn_cooperative_atomic_load_16x8B : AMDGPUBuiltin<"_Vector<2, int>(_Vector<2, int> *, _Constant int, char const *)", [Const], "gfx1250-insts,wavefrontsize32">;
+def __builtin_amdgcn_cooperative_atomic_store_16x8B : AMDGPUBuiltin<"void(_Vector<2, int> *, _Vector<2, int>, _Constant int, char const *)", [Const], "gfx1250-insts,wavefrontsize32">;
-def __builtin_amdgcn_cooperative_atomic_load_8x16B : AMDGPUBuiltin<"_ExtVector<4, int>(_ExtVector<4, int> *, _Constant int, char const *)", [Const], "gfx1250-insts,wavefrontsize32">;
-def __builtin_amdgcn_cooperative_atomic_store_8x16B : AMDGPUBuiltin<"void(_ExtVector<4, int> *, _ExtVector<4, int>, _Constant int, char const *)", [Const], "gfx1250-insts,wavefrontsize32">;
+def __builtin_amdgcn_cooperative_atomic_load_8x16B : AMDGPUBuiltin<"_Vector<4, int>(_Vector<4, int> *, _Constant int, char const *)", [Const], "gfx1250-insts,wavefrontsize32">;
+def __builtin_amdgcn_cooperative_atomic_store_8x16B : AMDGPUBuiltin<"void(_Vector<4, int> *, _Vector<4, int>, _Constant int, char const *)", [Const], "gfx1250-insts,wavefrontsize32">;
//===----------------------------------------------------------------------===//
// GFX13+ only builtins.
@@ -1297,115 +1297,115 @@ def __builtin_amdgcn_cvt_scalef32_pk32_fp6_f32 : AMDGPUBuiltin<"_Vector<6, unsig
//===----------------------------------------------------------------------===//
// Image builtins
//===----------------------------------------------------------------------===//
-def __builtin_amdgcn_image_load_1d_v4f32_i32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_load_1d_v4f16_i32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_load_1darray_v4f32_i32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_load_1darray_v4f16_i32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_load_1d_v4f32_i32 : AMDGPUBuiltin<"_Vector<4, float>(int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_load_1d_v4f16_i32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_load_1darray_v4f32_i32 : AMDGPUBuiltin<"_Vector<4, float>(int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_load_1darray_v4f16_i32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
def __builtin_amdgcn_image_load_2d_f32_i32 : AMDGPUBuiltin<"float(int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_load_2d_v4f32_i32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_load_2d_v4f16_i32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_load_2d_v4f32_i32 : AMDGPUBuiltin<"_Vector<4, float>(int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_load_2d_v4f16_i32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
def __builtin_amdgcn_image_load_2darray_f32_i32 : AMDGPUBuiltin<"float(int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_load_2darray_v4f32_i32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_load_2darray_v4f16_i32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_load_3d_v4f32_i32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_load_3d_v4f16_i32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_load_cube_v4f32_i32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_load_cube_v4f16_i32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_load_mip_1d_v4f32_i32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_load_mip_1d_v4f16_i32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_load_mip_1darray_v4f32_i32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_load_mip_1darray_v4f16_i32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_load_2darray_v4f32_i32 : AMDGPUBuiltin<"_Vector<4, float>(int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_load_2darray_v4f16_i32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_load_3d_v4f32_i32 : AMDGPUBuiltin<"_Vector<4, float>(int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_load_3d_v4f16_i32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_load_cube_v4f32_i32 : AMDGPUBuiltin<"_Vector<4, float>(int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_load_cube_v4f16_i32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_load_mip_1d_v4f32_i32 : AMDGPUBuiltin<"_Vector<4, float>(int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_load_mip_1d_v4f16_i32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_load_mip_1darray_v4f32_i32 : AMDGPUBuiltin<"_Vector<4, float>(int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_load_mip_1darray_v4f16_i32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
def __builtin_amdgcn_image_load_mip_2d_f32_i32 : AMDGPUBuiltin<"float(int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_load_mip_2d_v4f32_i32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_load_mip_2d_v4f16_i32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_load_mip_2d_v4f32_i32 : AMDGPUBuiltin<"_Vector<4, float>(int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_load_mip_2d_v4f16_i32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
def __builtin_amdgcn_image_load_mip_2darray_f32_i32 : AMDGPUBuiltin<"float(int, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_load_mip_2darray_v4f32_i32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_load_mip_2darray_v4f16_i32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_load_mip_3d_v4f32_i32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_load_mip_3d_v4f16_i32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_load_mip_cube_v4f32_i32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_load_mip_cube_v4f16_i32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_store_1d_v4f32_i32 : AMDGPUBuiltin<"void(_ExtVector<4, float>, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_store_1d_v4f16_i32 : AMDGPUBuiltin<"void(_ExtVector<4, _Float16>, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_store_1darray_v4f32_i32 : AMDGPUBuiltin<"void(_ExtVector<4, float>, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_store_1darray_v4f16_i32 : AMDGPUBuiltin<"void(_ExtVector<4, _Float16>, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_load_mip_2darray_v4f32_i32 : AMDGPUBuiltin<"_Vector<4, float>(int, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_load_mip_2darray_v4f16_i32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_load_mip_3d_v4f32_i32 : AMDGPUBuiltin<"_Vector<4, float>(int, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_load_mip_3d_v4f16_i32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_load_mip_cube_v4f32_i32 : AMDGPUBuiltin<"_Vector<4, float>(int, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_load_mip_cube_v4f16_i32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_store_1d_v4f32_i32 : AMDGPUBuiltin<"void(_Vector<4, float>, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_store_1d_v4f16_i32 : AMDGPUBuiltin<"void(_Vector<4, _Float16>, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_store_1darray_v4f32_i32 : AMDGPUBuiltin<"void(_Vector<4, float>, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_store_1darray_v4f16_i32 : AMDGPUBuiltin<"void(_Vector<4, _Float16>, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
def __builtin_amdgcn_image_store_2d_f32_i32 : AMDGPUBuiltin<"void(float, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_store_2d_v4f32_i32 : AMDGPUBuiltin<"void(_ExtVector<4, float>, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_store_2d_v4f16_i32 : AMDGPUBuiltin<"void(_ExtVector<4, _Float16>, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_store_2d_v4f32_i32 : AMDGPUBuiltin<"void(_Vector<4, float>, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_store_2d_v4f16_i32 : AMDGPUBuiltin<"void(_Vector<4, _Float16>, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
def __builtin_amdgcn_image_store_2darray_f32_i32 : AMDGPUBuiltin<"void(float, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_store_2darray_v4f32_i32 : AMDGPUBuiltin<"void(_ExtVector<4, float>, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_store_2darray_v4f16_i32 : AMDGPUBuiltin<"void(_ExtVector<4, _Float16>, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_store_3d_v4f32_i32 : AMDGPUBuiltin<"void(_ExtVector<4, float>, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_store_3d_v4f16_i32 : AMDGPUBuiltin<"void(_ExtVector<4, _Float16>, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_store_cube_v4f32_i32 : AMDGPUBuiltin<"void(_ExtVector<4, float>, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_store_cube_v4f16_i32 : AMDGPUBuiltin<"void(_ExtVector<4, _Float16>, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_store_mip_1d_v4f32_i32 : AMDGPUBuiltin<"void(_ExtVector<4, float>, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_store_mip_1d_v4f16_i32 : AMDGPUBuiltin<"void(_ExtVector<4, _Float16>, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_store_mip_1darray_v4f32_i32 : AMDGPUBuiltin<"void(_ExtVector<4, float>, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_store_mip_1darray_v4f16_i32 : AMDGPUBuiltin<"void(_ExtVector<4, _Float16>, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_store_2darray_v4f32_i32 : AMDGPUBuiltin<"void(_Vector<4, float>, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_store_2darray_v4f16_i32 : AMDGPUBuiltin<"void(_Vector<4, _Float16>, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_store_3d_v4f32_i32 : AMDGPUBuiltin<"void(_Vector<4, float>, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_store_3d_v4f16_i32 : AMDGPUBuiltin<"void(_Vector<4, _Float16>, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_store_cube_v4f32_i32 : AMDGPUBuiltin<"void(_Vector<4, float>, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_store_cube_v4f16_i32 : AMDGPUBuiltin<"void(_Vector<4, _Float16>, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_store_mip_1d_v4f32_i32 : AMDGPUBuiltin<"void(_Vector<4, float>, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_store_mip_1d_v4f16_i32 : AMDGPUBuiltin<"void(_Vector<4, _Float16>, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_store_mip_1darray_v4f32_i32 : AMDGPUBuiltin<"void(_Vector<4, float>, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_store_mip_1darray_v4f16_i32 : AMDGPUBuiltin<"void(_Vector<4, _Float16>, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
def __builtin_amdgcn_image_store_mip_2d_f32_i32 : AMDGPUBuiltin<"void(float, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_store_mip_2d_v4f32_i32 : AMDGPUBuiltin<"void(_ExtVector<4, float>, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_store_mip_2d_v4f16_i32 : AMDGPUBuiltin<"void(_ExtVector<4, _Float16>, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_store_mip_2d_v4f32_i32 : AMDGPUBuiltin<"void(_Vector<4, float>, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_store_mip_2d_v4f16_i32 : AMDGPUBuiltin<"void(_Vector<4, _Float16>, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
def __builtin_amdgcn_image_store_mip_2darray_f32_i32 : AMDGPUBuiltin<"void(float, int, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_store_mip_2darray_v4f32_i32 : AMDGPUBuiltin<"void(_ExtVector<4, float>, int, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_store_mip_2darray_v4f16_i32 : AMDGPUBuiltin<"void(_ExtVector<4, _Float16>, int, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_store_mip_3d_v4f32_i32 : AMDGPUBuiltin<"void(_ExtVector<4, float>, int, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_store_mip_3d_v4f16_i32 : AMDGPUBuiltin<"void(_ExtVector<4, _Float16>, int, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_store_mip_cube_v4f32_i32 : AMDGPUBuiltin<"void(_ExtVector<4, float>, int, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_store_mip_cube_v4f16_i32 : AMDGPUBuiltin<"void(_ExtVector<4, _Float16>, int, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_sample_1d_v4f32_f32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_sample_1d_v4f16_f32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_sample_1darray_v4f32_f32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_sample_1darray_v4f16_f32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_sample_2d_f32_f32 : AMDGPUBuiltin<"float(int, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_sample_2d_v4f32_f32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_sample_2d_v4f16_f32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_sample_2darray_f32_f32 : AMDGPUBuiltin<"float(int, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_sample_2darray_v4f32_f32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_sample_2darray_v4f16_f32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_sample_3d_v4f32_f32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_sample_3d_v4f16_f32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_sample_cube_v4f32_f32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "image-insts">;
-def __builtin_amdgcn_image_sample_cube_v4f16_f32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_sample_lz_1d_v4f32_f32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
-def __builtin_amdgcn_image_sample_lz_1d_v4f16_f32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_sample_lz_1darray_v4f32_f32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
-def __builtin_amdgcn_image_sample_lz_1darray_v4f16_f32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_sample_lz_2d_f32_f32 : AMDGPUBuiltin<"float(int, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
-def __builtin_amdgcn_image_sample_lz_2d_v4f32_f32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
-def __builtin_amdgcn_image_sample_lz_2d_v4f16_f32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_sample_lz_2darray_f32_f32 : AMDGPUBuiltin<"float(int, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
-def __builtin_amdgcn_image_sample_lz_2darray_v4f32_f32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
-def __builtin_amdgcn_image_sample_lz_2darray_v4f16_f32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_sample_lz_3d_v4f32_f32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
-def __builtin_amdgcn_image_sample_lz_3d_v4f16_f32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_sample_lz_cube_v4f32_f32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
-def __builtin_amdgcn_image_sample_lz_cube_v4f16_f32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_sample_l_1d_v4f32_f32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
-def __builtin_amdgcn_image_sample_l_1d_v4f16_f32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_sample_l_1darray_v4f32_f32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
-def __builtin_amdgcn_image_sample_l_1darray_v4f16_f32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_sample_l_2d_f32_f32 : AMDGPUBuiltin<"float(int, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
-def __builtin_amdgcn_image_sample_l_2d_v4f32_f32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
-def __builtin_amdgcn_image_sample_l_2d_v4f16_f32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_sample_l_2darray_f32_f32 : AMDGPUBuiltin<"float(int, float, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
-def __builtin_amdgcn_image_sample_l_2darray_v4f32_f32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, float, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
-def __builtin_amdgcn_image_sample_l_2darray_v4f16_f32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, float, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_sample_l_3d_v4f32_f32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, float, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
-def __builtin_amdgcn_image_sample_l_3d_v4f16_f32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, float, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_sample_l_cube_v4f32_f32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, float, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
-def __builtin_amdgcn_image_sample_l_cube_v4f16_f32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, float, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_sample_d_1d_v4f32_f32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
-def __builtin_amdgcn_image_sample_d_1d_v4f16_f32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_sample_d_1darray_v4f32_f32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, float, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
-def __builtin_amdgcn_image_sample_d_1darray_v4f16_f32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, float, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_sample_d_2d_f32_f32 : AMDGPUBuiltin<"float(int, float, float, float, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
-def __builtin_amdgcn_image_sample_d_2d_v4f32_f32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, float, float, float, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
-def __builtin_amdgcn_image_sample_d_2d_v4f16_f32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, float, float, float, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_sample_d_2darray_f32_f32 : AMDGPUBuiltin<"float(int, float, float, float, float, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
-def __builtin_amdgcn_image_sample_d_2darray_v4f32_f32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, float, float, float, float, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
-def __builtin_amdgcn_image_sample_d_2darray_v4f16_f32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, float, float, float, float, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_sample_d_3d_v4f32_f32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, float, float, float, float, float, float, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
-def __builtin_amdgcn_image_sample_d_3d_v4f16_f32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, float, float, float, float, float, float, float, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
-def __builtin_amdgcn_image_gather4_lz_2d_v4f32_f32 : AMDGPUBuiltin<"_ExtVector<4, float>(int, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
-def __builtin_amdgcn_image_gather4_lz_2d_v4f16_f32 : AMDGPUBuiltin<"_ExtVector<4, _Float16>(int, float, float, __amdgpu_texture_t, _ExtVector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_store_mip_2darray_v4f32_i32 : AMDGPUBuiltin<"void(_Vector<4, float>, int, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_store_mip_2darray_v4f16_i32 : AMDGPUBuiltin<"void(_Vector<4, _Float16>, int, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_store_mip_3d_v4f32_i32 : AMDGPUBuiltin<"void(_Vector<4, float>, int, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_store_mip_3d_v4f16_i32 : AMDGPUBuiltin<"void(_Vector<4, _Float16>, int, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_store_mip_cube_v4f32_i32 : AMDGPUBuiltin<"void(_Vector<4, float>, int, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_store_mip_cube_v4f16_i32 : AMDGPUBuiltin<"void(_Vector<4, _Float16>, int, int, int, int, int, __amdgpu_texture_t, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_sample_1d_v4f32_f32 : AMDGPUBuiltin<"_Vector<4, float>(int, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_sample_1d_v4f16_f32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_sample_1darray_v4f32_f32 : AMDGPUBuiltin<"_Vector<4, float>(int, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_sample_1darray_v4f16_f32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_sample_2d_f32_f32 : AMDGPUBuiltin<"float(int, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_sample_2d_v4f32_f32 : AMDGPUBuiltin<"_Vector<4, float>(int, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_sample_2d_v4f16_f32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_sample_2darray_f32_f32 : AMDGPUBuiltin<"float(int, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_sample_2darray_v4f32_f32 : AMDGPUBuiltin<"_Vector<4, float>(int, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_sample_2darray_v4f16_f32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_sample_3d_v4f32_f32 : AMDGPUBuiltin<"_Vector<4, float>(int, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_sample_3d_v4f16_f32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_sample_cube_v4f32_f32 : AMDGPUBuiltin<"_Vector<4, float>(int, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "image-insts">;
+def __builtin_amdgcn_image_sample_cube_v4f16_f32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_sample_lz_1d_v4f32_f32 : AMDGPUBuiltin<"_Vector<4, float>(int, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
+def __builtin_amdgcn_image_sample_lz_1d_v4f16_f32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_sample_lz_1darray_v4f32_f32 : AMDGPUBuiltin<"_Vector<4, float>(int, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
+def __builtin_amdgcn_image_sample_lz_1darray_v4f16_f32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_sample_lz_2d_f32_f32 : AMDGPUBuiltin<"float(int, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
+def __builtin_amdgcn_image_sample_lz_2d_v4f32_f32 : AMDGPUBuiltin<"_Vector<4, float>(int, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
+def __builtin_amdgcn_image_sample_lz_2d_v4f16_f32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_sample_lz_2darray_f32_f32 : AMDGPUBuiltin<"float(int, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
+def __builtin_amdgcn_image_sample_lz_2darray_v4f32_f32 : AMDGPUBuiltin<"_Vector<4, float>(int, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
+def __builtin_amdgcn_image_sample_lz_2darray_v4f16_f32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_sample_lz_3d_v4f32_f32 : AMDGPUBuiltin<"_Vector<4, float>(int, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
+def __builtin_amdgcn_image_sample_lz_3d_v4f16_f32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_sample_lz_cube_v4f32_f32 : AMDGPUBuiltin<"_Vector<4, float>(int, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
+def __builtin_amdgcn_image_sample_lz_cube_v4f16_f32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_sample_l_1d_v4f32_f32 : AMDGPUBuiltin<"_Vector<4, float>(int, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
+def __builtin_amdgcn_image_sample_l_1d_v4f16_f32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_sample_l_1darray_v4f32_f32 : AMDGPUBuiltin<"_Vector<4, float>(int, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
+def __builtin_amdgcn_image_sample_l_1darray_v4f16_f32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_sample_l_2d_f32_f32 : AMDGPUBuiltin<"float(int, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
+def __builtin_amdgcn_image_sample_l_2d_v4f32_f32 : AMDGPUBuiltin<"_Vector<4, float>(int, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
+def __builtin_amdgcn_image_sample_l_2d_v4f16_f32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_sample_l_2darray_f32_f32 : AMDGPUBuiltin<"float(int, float, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
+def __builtin_amdgcn_image_sample_l_2darray_v4f32_f32 : AMDGPUBuiltin<"_Vector<4, float>(int, float, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
+def __builtin_amdgcn_image_sample_l_2darray_v4f16_f32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, float, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_sample_l_3d_v4f32_f32 : AMDGPUBuiltin<"_Vector<4, float>(int, float, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
+def __builtin_amdgcn_image_sample_l_3d_v4f16_f32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, float, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_sample_l_cube_v4f32_f32 : AMDGPUBuiltin<"_Vector<4, float>(int, float, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
+def __builtin_amdgcn_image_sample_l_cube_v4f16_f32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, float, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_sample_d_1d_v4f32_f32 : AMDGPUBuiltin<"_Vector<4, float>(int, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
+def __builtin_amdgcn_image_sample_d_1d_v4f16_f32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_sample_d_1darray_v4f32_f32 : AMDGPUBuiltin<"_Vector<4, float>(int, float, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
+def __builtin_amdgcn_image_sample_d_1darray_v4f16_f32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, float, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_sample_d_2d_f32_f32 : AMDGPUBuiltin<"float(int, float, float, float, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
+def __builtin_amdgcn_image_sample_d_2d_v4f32_f32 : AMDGPUBuiltin<"_Vector<4, float>(int, float, float, float, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
+def __builtin_amdgcn_image_sample_d_2d_v4f16_f32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, float, float, float, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_sample_d_2darray_f32_f32 : AMDGPUBuiltin<"float(int, float, float, float, float, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
+def __builtin_amdgcn_image_sample_d_2darray_v4f32_f32 : AMDGPUBuiltin<"_Vector<4, float>(int, float, float, float, float, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
+def __builtin_amdgcn_image_sample_d_2darray_v4f16_f32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, float, float, float, float, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_sample_d_3d_v4f32_f32 : AMDGPUBuiltin<"_Vector<4, float>(int, float, float, float, float, float, float, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
+def __builtin_amdgcn_image_sample_d_3d_v4f16_f32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, float, float, float, float, float, float, float, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
+def __builtin_amdgcn_image_gather4_lz_2d_v4f32_f32 : AMDGPUBuiltin<"_Vector<4, float>(int, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts">;
+def __builtin_amdgcn_image_gather4_lz_2d_v4f16_f32 : AMDGPUBuiltin<"_Vector<4, _Float16>(int, float, float, __amdgpu_texture_t, _Vector<4, int>, bool, int, int)", [Const], "extended-image-insts,16-bit-insts">;
diff --git a/clang/include/clang/Options/Options.td b/clang/include/clang/Options/Options.td
index 3b88dce9c822b..038e3a05078a1 100644
--- a/clang/include/clang/Options/Options.td
+++ b/clang/include/clang/Options/Options.td
@@ -3554,14 +3554,20 @@ def fno_lto : Flag<["-"], "fno-lto">,
Visibility<[ClangOption, CLOption, DXCOption, CC1Option, FlangOption]>, Group<f_Group>,
HelpText<"Disable LTO mode (default)">;
def foffload_lto_EQ : Joined<["-"], "foffload-lto=">,
- Visibility<[ClangOption, CLOption]>, Group<f_Group>,
+ Visibility<[ClangOption, CLOption, FlangOption]>, Group<f_Group>,
HelpText<"Set LTO mode for offload compilation">, Values<"thin,full">;
def foffload_lto : Flag<["-"], "foffload-lto">,
- Visibility<[ClangOption, CLOption]>, Group<f_Group>,
+ Visibility<[ClangOption, CLOption, FlangOption]>, Group<f_Group>,
Alias<foffload_lto_EQ>, AliasArgs<["full"]>, HelpText<"Enable LTO in 'full' mode for offload compilation">;
def fno_offload_lto : Flag<["-"], "fno-offload-lto">,
- Visibility<[ClangOption, CLOption]>, Group<f_Group>,
+ Visibility<[ClangOption, CLOption, FlangOption]>, Group<f_Group>,
HelpText<"Disable LTO mode (default) for offload compilation">;
+def foffload_object_linking : Flag<["-"], "foffload-object-linking">,
+ Visibility<[ClangOption]>, Group<f_Group>,
+ HelpText<"Enable object linking for offload compilation (AMDGPU)">;
+def fno_offload_object_linking : Flag<["-"], "fno-offload-object-linking">,
+ Visibility<[ClangOption]>, Group<f_Group>,
+ HelpText<"Disable object linking for offload compilation (default)">;
def flto_jobs_EQ : Joined<["-"], "flto-jobs=">,
Visibility<[ClangOption, CC1Option]>, Group<f_Group>,
HelpText<"Controls the backend parallelism of -flto=thin (default "
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index f06f1c5a75ff8..5dfe19b6f4513 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -1069,6 +1069,21 @@ static TripleSet inferOffloadToolchains(Compilation &C,
void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
InputList &Inputs) {
+ bool IsOpenMP =
+ C.getInputArgs().hasFlag(options::OPT_fopenmp, options::OPT_fopenmp_EQ,
+ options::OPT_fno_openmp, false);
+ const Arg *ObjectLinkingArg =
+ C.getInputArgs().getLastArg(options::OPT_foffload_object_linking,
+ options::OPT_fno_offload_object_linking);
+ bool IsExplicitObjectLinking =
+ C.getInputArgs().hasFlag(options::OPT_foffload_object_linking,
+ options::OPT_fno_offload_object_linking, false);
+ if (IsOpenMP && ObjectLinkingArg) {
+ Diag(clang::diag::err_drv_unsupported_opt_for_language_mode)
+ << ObjectLinkingArg->getSpelling() << "OpenMP";
+ return;
+ }
+
bool IsCuda =
llvm::any_of(Inputs, [](std::pair<types::ID, const llvm::opt::Arg *> &I) {
return types::isCuda(I.first);
@@ -1079,15 +1094,14 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
return types::isHIP(I.first);
}) ||
C.getInputArgs().hasArg(options::OPT_hip_link) ||
- C.getInputArgs().hasArg(options::OPT_hipstdpar));
+ C.getInputArgs().hasArg(options::OPT_hipstdpar) ||
+ IsExplicitObjectLinking);
bool IsSYCL = C.getInputArgs().hasFlag(options::OPT_fsycl,
options::OPT_fno_sycl, false);
bool IsOpenMPOffloading =
- (C.getInputArgs().hasFlag(options::OPT_fopenmp, options::OPT_fopenmp_EQ,
- options::OPT_fno_openmp, false) &&
- (C.getInputArgs().hasArg(options::OPT_offload_targets_EQ) ||
- (C.getInputArgs().hasArg(options::OPT_offload_arch_EQ) &&
- !(IsCuda || IsHIP))));
+ IsOpenMP && (C.getInputArgs().hasArg(options::OPT_offload_targets_EQ) ||
+ (C.getInputArgs().hasArg(options::OPT_offload_arch_EQ) &&
+ !(IsCuda || IsHIP)));
llvm::SmallSet<Action::OffloadKind, 4> Kinds;
const std::pair<bool, Action::OffloadKind> ActiveKinds[] = {
@@ -5198,8 +5212,20 @@ Driver::BuildOffloadingActions(Compilation &C, llvm::opt::DerivedArgList &Args,
// Propagate the ToolChain so we can use it in ConstructPhaseAction.
A->propagateDeviceOffloadInfo(Kind, TCAndArch->second,
TCAndArch->first);
- A = ConstructPhaseAction(C, Args, Phase, A, Kind,
- TCAndArch->first->getLTOMode(Args, Kind));
+ // ThinLTO backends emit relocatable objects for final object linking.
+ // For other LTO modes, explicit object linking replaces LTO.
+ LTOKind RequestedDeviceLTOMode =
+ TCAndArch->first->getLTOMode(Args, Kind);
+ bool IsThinLTOObjectLinking =
+ TCAndArch->first->getTriple().isAMDGPU() &&
+ RequestedDeviceLTOMode == LTOK_Thin;
+ bool IsObjectLinking = Args.hasFlag(
+ options::OPT_foffload_object_linking,
+ options::OPT_fno_offload_object_linking, IsThinLTOObjectLinking);
+ LTOKind DeviceLTOMode = IsObjectLinking && !IsThinLTOObjectLinking
+ ? LTOK_None
+ : RequestedDeviceLTOMode;
+ A = ConstructPhaseAction(C, Args, Phase, A, Kind, DeviceLTOMode);
if (isa<CompileJobAction>(A) && isa<CompileJobAction>(HostAction) &&
Kind == Action::OFK_OpenMP &&
diff --git a/clang/lib/Driver/ToolChains/AMDGPU.cpp b/clang/lib/Driver/ToolChains/AMDGPU.cpp
index 6bc6a53732881..691ad1717f4db 100644
--- a/clang/lib/Driver/ToolChains/AMDGPU.cpp
+++ b/clang/lib/Driver/ToolChains/AMDGPU.cpp
@@ -935,6 +935,13 @@ void AMDGPUToolChain::addClangTargetOptions(
}
}
+ LTOKind DeviceLTOMode = getLTOMode(DriverArgs, DeviceOffloadingKind);
+ bool IsThinLTOObjectLinking = DeviceLTOMode == LTOK_Thin;
+ if (DriverArgs.hasFlag(options::OPT_foffload_object_linking,
+ options::OPT_fno_offload_object_linking,
+ IsThinLTOObjectLinking))
+ CC1Args.append({"-mllvm", "-amdgpu-enable-object-linking"});
+
DriverArgs.AddLastArg(CC1Args, options::OPT_gpu_max_threads_per_block_EQ);
// Default to "hidden" visibility, as object level linking will not be
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 072664e6040f3..9361fb3138fcd 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -9981,28 +9981,39 @@ void LinkerWrapper::ConstructJob(Compilation &C, const JobAction &JA,
// Forward the LTO mode for this toolchain.
auto DeviceLTOMode = TC->getLTOMode(ToolChainArgs, Kind);
- if (DeviceLTOMode == LTOK_Full)
+ bool IsThinLTOObjectLinking =
+ TC->getTriple().isAMDGPU() && DeviceLTOMode == LTOK_Thin;
+ bool HasObjectLinking = Args.hasFlag(
+ options::OPT_foffload_object_linking,
+ options::OPT_fno_offload_object_linking, IsThinLTOObjectLinking);
+
+ if (HasObjectLinking && DeviceLTOMode == LTOK_Full &&
+ Args.hasArg(options::OPT_foffload_lto,
+ options::OPT_foffload_lto_EQ)) {
+ C.getDriver().Diag(diag::err_drv_cannot_mix_options)
+ << "-foffload-object-linking" << "-foffload-lto";
+ }
+
+ bool UseObjectLinkingOnly =
+ HasObjectLinking && DeviceLTOMode != LTOK_Thin;
+ if (HasObjectLinking && DeviceLTOMode == LTOK_Thin &&
+ TC->getTriple().isAMDGPU()) {
+ CmdArgs.push_back(
+ Args.MakeArgString("--device-linker=" + TC->getTripleString() +
+ "=-plugin-opt=-amdgpu-enable-object-linking"));
+ }
+
+ if (UseObjectLinkingOnly) {
+ // Explicit object linking without ThinLTO uses ISA-level linking.
+ } else if (DeviceLTOMode == LTOK_Full) {
CmdArgs.push_back(Args.MakeArgString(
"--device-compiler=" + TC->getTripleString() + "=-flto=full"));
- else if (DeviceLTOMode == LTOK_Thin) {
+ } else if (DeviceLTOMode == LTOK_Thin) {
CmdArgs.push_back(Args.MakeArgString(
"--device-compiler=" + TC->getTripleString() + "=-flto=thin"));
- if (TC->getTriple().isAMDGPU()) {
- CmdArgs.push_back(
- Args.MakeArgString("--device-linker=" + TC->getTripleString() +
- "=-plugin-opt=-force-import-all"));
- CmdArgs.push_back(
- Args.MakeArgString("--device-linker=" + TC->getTripleString() +
- "=-plugin-opt=-avail-extern-to-local"));
- CmdArgs.push_back(Args.MakeArgString(
- "--device-linker=" + TC->getTripleString() +
- "=-plugin-opt=-avail-extern-gv-in-addrspace-to-local=3"));
- if (Kind == Action::OFK_OpenMP) {
- CmdArgs.push_back(
- Args.MakeArgString("--device-linker=" + TC->getTripleString() +
- "=-plugin-opt=-amdgpu-internalize-symbols"));
- }
- }
+ } else if (TC->getTriple().isAMDGPU()) {
+ CmdArgs.push_back(Args.MakeArgString(
+ "--device-compiler=" + TC->getTripleString() + "=-flto"));
}
}
}
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 7562206f93438..734181626a972 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -384,9 +384,10 @@ void Flang::addCodegenOptions(const ArgList &Args,
CmdArgs.push_back("-fcoarray");
}
-void Flang::addLTOOptions(const ArgList &Args, ArgStringList &CmdArgs) const {
+void Flang::addLTOOptions(const ArgList &Args, ArgStringList &CmdArgs,
+ Action::OffloadKind DeviceOffloadKind) const {
const ToolChain &TC = getToolChain();
- LTOKind LTOMode = TC.getLTOMode(Args);
+ LTOKind LTOMode = TC.getLTOMode(Args, DeviceOffloadKind);
// LTO mode is parsed by the Clang driver library.
assert(LTOMode != LTOK_Unknown && "Unknown LTO mode.");
if (LTOMode == LTOK_Full)
@@ -1347,7 +1348,7 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
handleColorDiagnosticsArgs(D, Args, CmdArgs);
- addLTOOptions(Args, CmdArgs);
+ addLTOOptions(Args, CmdArgs, JA.getOffloadingDeviceKind());
// -fPIC and related options.
addPicOptions(Args, CmdArgs);
diff --git a/clang/lib/Driver/ToolChains/Flang.h b/clang/lib/Driver/ToolChains/Flang.h
index a887301c703cd..c1897f39951ce 100644
--- a/clang/lib/Driver/ToolChains/Flang.h
+++ b/clang/lib/Driver/ToolChains/Flang.h
@@ -45,8 +45,10 @@ class LLVM_LIBRARY_VISIBILITY Flang : public Tool {
///
/// \param [in] Args The list of input driver arguments
/// \param [out] CmdArgs The list of output command arguments
+ /// \param [in] DeviceOffloadKind The offload kind
void addLTOOptions(const llvm::opt::ArgList &Args,
- llvm::opt::ArgStringList &CmdArgs) const;
+ llvm::opt::ArgStringList &CmdArgs,
+ Action::OffloadKind DeviceOffloadKind) const;
/// Extract PIC options from the driver arguments and add them to
/// the command arguments.
diff --git a/clang/test/Driver/hip-offload-object-linking.hip b/clang/test/Driver/hip-offload-object-linking.hip
new file mode 100644
index 0000000000000..d0d5b1bd1d906
--- /dev/null
+++ b/clang/test/Driver/hip-offload-object-linking.hip
@@ -0,0 +1,81 @@
+// REQUIRES: amdgpu-registered-target
+
+// Test -foffload-object-linking flag behavior for HIP.
+
+//--- Device compilation: -foffload-object-linking passes -mllvm -amdgpu-enable-object-linking
+// RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -nogpuinc \
+// RUN: --offload-arch=gfx90a -fgpu-rdc -foffload-object-linking \
+// RUN: -c %s 2>&1 | FileCheck -check-prefix=MLLVM %s
+// MLLVM: "-cc1" "-triple" "amdgpu9.0a-amd-amdhsa"
+// MLLVM-SAME: "-emit-obj"
+// MLLVM-NOT: "-flto
+// MLLVM-SAME: "-mllvm" "-amdgpu-enable-object-linking"
+
+//--- Device compilation: without -foffload-object-linking, no -amdgpu-enable-object-linking
+// RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -nogpuinc \
+// RUN: --offload-arch=gfx90a -fgpu-rdc \
+// RUN: -c %s 2>&1 | FileCheck -check-prefix=NO-MLLVM %s
+// NO-MLLVM: "-cc1" "-triple" "amdgpu9.0a-amd-amdhsa"
+// NO-MLLVM-NOT: "-amdgpu-enable-object-linking"
+
+//--- Device compilation: -fno-offload-object-linking overrides -foffload-object-linking
+// RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -nogpuinc \
+// RUN: --offload-arch=gfx90a -fgpu-rdc \
+// RUN: -foffload-object-linking -fno-offload-object-linking \
+// RUN: -c %s 2>&1 | FileCheck -check-prefix=NEGATED %s
+// NEGATED: "-cc1" "-triple" "amdgpu9.0a-amd-amdhsa"
+// NEGATED-NOT: "-amdgpu-enable-object-linking"
+
+//--- Link step: -foffload-object-linking does not forward -flto to linker wrapper
+// RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -nogpuinc \
+// RUN: --offload-arch=gfx90a -fgpu-rdc -foffload-object-linking \
+// RUN: %s 2>&1 | FileCheck -check-prefix=NO-LTO %s
+// NO-LTO: clang-linker-wrapper
+// NO-LTO-NOT: "--device-compiler=amdgpu-amd-amdhsa=-flto"
+// NO-LTO-NOT: "--device-compiler=amdgpu-amd-amdhsa=-flto-partitions=
+
+//--- Link step: without -foffload-object-linking, -flto is forwarded
+// RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -nogpuinc \
+// RUN: --offload-arch=gfx90a -fgpu-rdc \
+// RUN: %s 2>&1 | FileCheck -check-prefix=LTO-DEFAULT %s
+// LTO-DEFAULT: clang-linker-wrapper
+// LTO-DEFAULT-SAME: "--device-compiler=amdgpu-amd-amdhsa=-flto=full"
+
+//--- Link step: -foffload-object-linking invokes clang-linker-wrapper for .o inputs
+// RUN: touch %t1.o %t2.o
+// RUN: %clang -### --target=x86_64-linux-gnu \
+// RUN: --offload-arch=gfx90a -fgpu-rdc -foffload-object-linking \
+// RUN: %t1.o %t2.o 2>&1 | FileCheck -check-prefix=WRAPPER %s
+// WRAPPER: clang-linker-wrapper
+// WRAPPER-SAME: "--should-extract=gfx90a"
+
+//--- Host compilation: -foffload-object-linking does not affect host cc1
+// RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -nogpuinc \
+// RUN: --offload-arch=gfx90a -fgpu-rdc -foffload-object-linking \
+// RUN: -c %s 2>&1 | FileCheck -check-prefix=HOST %s
+// HOST: "-cc1" "-triple" "x86_64-unknown-linux-gnu"
+// HOST-NOT: "-amdgpu-enable-object-linking"
+// HOST-SAME: "-emit-obj"
+
+//--- Error: -foffload-object-linking conflicts with full -foffload-lto
+// RUN: not %clang -### --target=x86_64-linux-gnu -nogpulib -nogpuinc \
+// RUN: --offload-arch=gfx90a -fgpu-rdc -foffload-object-linking \
+// RUN: -foffload-lto %s 2>&1 | FileCheck -check-prefix=CONFLICT %s
+// CONFLICT: error: cannot specify '-foffload-lto' along with '-foffload-object-linking'
+
+//--- ThinLTO implies object linking; an explicit object-linking flag is redundant
+// RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -nogpuinc \
+// RUN: --offload-arch=gfx90a -fgpu-rdc -foffload-object-linking \
+// RUN: -foffload-lto=thin -c %s 2>&1 | FileCheck -check-prefix=THIN-OBJECT %s
+// THIN-OBJECT: "-cc1" "-triple" "amdgpu9.0a-amd-amdhsa"
+// THIN-OBJECT-SAME: "-emit-llvm-bc"
+// THIN-OBJECT-SAME: "-flto=thin"
+// THIN-OBJECT-SAME: "-mllvm" "-amdgpu-enable-object-linking"
+
+//--- Link step: ThinLTO forwards object-linking mode to the LTO backend.
+// RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -nogpuinc \
+// RUN: --offload-arch=gfx90a -fgpu-rdc -foffload-lto=thin %s 2>&1 \
+// RUN: | FileCheck -check-prefix=THIN-LINK %s
+// THIN-LINK: clang-linker-wrapper
+// THIN-LINK-SAME: "--device-linker=amdgpu-amd-amdhsa=-plugin-opt=-amdgpu-enable-object-linking"
+// THIN-LINK-SAME: "--device-compiler=amdgpu-amd-amdhsa=-flto=thin"
diff --git a/clang/test/Driver/hip-options.hip b/clang/test/Driver/hip-options.hip
index 90cd051c98271..c1846c2292b12 100644
--- a/clang/test/Driver/hip-options.hip
+++ b/clang/test/Driver/hip-options.hip
@@ -84,9 +84,13 @@
// Ensure we don't error about -fwhole-program-vtables for the non-device offload compile.
// HIPTHINLTO-NOT: error: invalid argument '-fwhole-program-vtables' only allowed with '-flto'
// HIPTHINLTO-NOT: "-cc1"{{.*}} "-triple" "x86_64-unknown-linux-gnu" {{.*}} "-flto-unit"
-// HIPTHINLTO: "-cc1"{{.*}} "-triple" "amdgpu9.06-amd-amdhsa" {{.*}} "-flto=thin" "-flto-unit" {{.*}} "-fwhole-program-vtables"
+// HIPTHINLTO: "-cc1"{{.*}} "-triple" "amdgpu9.06-amd-amdhsa" {{.*}} "-emit-llvm-bc"
+// HIPTHINLTO-SAME: "-flto=thin"
+// HIPTHINLTO-SAME: "-mllvm" "-amdgpu-enable-object-linking"
// HIPTHINLTO-NOT: "-cc1"{{.*}} "-triple" "x86_64-unknown-linux-gnu" {{.*}} "-flto-unit"
-// HIPTHINLTO: clang-linker-wrapper{{.*}} "--device-compiler=amdgpu-amd-amdhsa=-flto=thin"
+// HIPTHINLTO: clang-linker-wrapper
+// HIPTHINLTO-SAME: "--device-linker=amdgpu-amd-amdhsa=-plugin-opt=-amdgpu-enable-object-linking"
+// HIPTHINLTO-SAME: "--device-compiler=amdgpu-amd-amdhsa=-flto=thin"
// Check -no-canonical-prefixes is propagated correctly.
diff --git a/clang/test/Driver/hip-thinlto.hip b/clang/test/Driver/hip-thinlto.hip
index 8ec0ff5130fcc..7c5305c5ac8e9 100644
--- a/clang/test/Driver/hip-thinlto.hip
+++ b/clang/test/Driver/hip-thinlto.hip
@@ -1,9 +1,13 @@
// RUN: %clang -foffload-lto=thin -nogpulib -nogpuinc %s -### 2>&1 | FileCheck %s
-// CHECK: "--device-compiler=amdgpu-amd-amdhsa=-flto=thin"
-// CHECK-SAME: "--device-linker=amdgpu-amd-amdhsa=-plugin-opt=-force-import-all"
-// CHECK-SAME: "--device-linker=amdgpu-amd-amdhsa=-plugin-opt=-avail-extern-to-local"
-// CHECK-SAME: "--device-linker=amdgpu-amd-amdhsa=-plugin-opt=-avail-extern-gv-in-addrspace-to-local=3"
+// CHECK: "-cc1" "-triple" "amdgpu9.06-amd-amdhsa"
+// CHECK-SAME: "-emit-llvm-bc"
+// CHECK-SAME: "-flto=thin"
+// CHECK-SAME: "-mllvm" "-amdgpu-enable-object-linking"
+// CHECK: clang-linker-wrapper
+// CHECK-SAME: "--device-linker=amdgpu-amd-amdhsa=-plugin-opt=-amdgpu-enable-object-linking"
+// CHECK-SAME: "--device-compiler=amdgpu-amd-amdhsa=-flto=thin"
+// CHECK-NOT: "--device-linker=amdgpu-amd-amdhsa=-plugin-opt=-force-import-all"
int main(int, char *[]) {
return 0;
}
diff --git a/clang/test/Driver/openmp-offload-gpu.c b/clang/test/Driver/openmp-offload-gpu.c
index 3c159130adeca..7e00267f2d1e2 100644
--- a/clang/test/Driver/openmp-offload-gpu.c
+++ b/clang/test/Driver/openmp-offload-gpu.c
@@ -391,11 +391,14 @@
// RUN: %clang -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp \
// RUN: --offload-arch=gfx906 -foffload-lto=thin -nogpulib -nogpuinc %s 2>&1 \
// RUN: | FileCheck --check-prefix=THINLTO-GFX906 %s
-// THINLTO-GFX906: --device-compiler=amdgpu-amd-amdhsa=-flto=thin
-// THINLTO-GFX906-SAME: --device-linker=amdgpu-amd-amdhsa=-plugin-opt=-force-import-all
-// THINLTO-GFX906-SAME: --device-linker=amdgpu-amd-amdhsa=-plugin-opt=-avail-extern-to-local
-// THINLTO-GFX906-SAME: --device-linker=amdgpu-amd-amdhsa=-plugin-opt=-avail-extern-gv-in-addrspace-to-local=3
-// THINLTO-GFX906-SAME: --device-linker=amdgpu-amd-amdhsa=-plugin-opt=-amdgpu-internalize-symbols
+// THINLTO-GFX906: "-cc1" "-triple" "amdgpu9.06-amd-amdhsa"
+// THINLTO-GFX906-SAME: "-emit-llvm-bc"
+// THINLTO-GFX906-SAME: "-flto=thin"
+// THINLTO-GFX906-SAME: "-mllvm" "-amdgpu-enable-object-linking"
+// THINLTO-GFX906: --device-linker=amdgpu-amd-amdhsa=-plugin-opt=-amdgpu-enable-object-linking
+// THINLTO-GFX906-SAME: --device-compiler=amdgpu-amd-amdhsa=-flto=thin
+// THINLTO-GFX906-NOT: --device-linker=amdgpu-amd-amdhsa=-plugin-opt=-force-import-all
+// THINLTO-GFX906-NOT: --device-linker=amdgpu-amd-amdhsa=-plugin-opt=-amdgpu-internalize-symbols
//
// RUN: %clang -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp \
// RUN: --offload-arch=sm_52 -foffload-lto=thin -nogpulib -nogpuinc %s 2>&1 \
@@ -437,7 +440,6 @@
//
// PROFILE-SUBARCH: clang-linker-wrapper{{.*}}--device-compiler=amdgpu10.30-amd-amdhsa-llvm=-fprofile-generate
-
// RUN: %clang -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp \
// RUN: -resource-dir=%S/Inputs/resource_dir \
// RUN: --offload-arch=gfx906 -fprofile-generate -nogpulib -nogpuinc %s 2>&1 \
diff --git a/clang/test/Driver/openmp-offload-object-linking.c b/clang/test/Driver/openmp-offload-object-linking.c
new file mode 100644
index 0000000000000..0be259a8f8a41
--- /dev/null
+++ b/clang/test/Driver/openmp-offload-object-linking.c
@@ -0,0 +1,21 @@
+// REQUIRES: amdgpu-registered-target
+
+// OpenMP supports full LTO and ThinLTO, but not explicit object-linking
+// options.
+// RUN: not %clang -### --target=x86_64-unknown-linux-gnu -fopenmp \
+// RUN: --offload-arch=gfx906 -foffload-object-linking \
+// RUN: -nogpulib -nogpuinc %s 2>&1 \
+// RUN: | FileCheck %s --check-prefix=ENABLE
+// RUN: not %clang -### --target=x86_64-unknown-linux-gnu -fopenmp \
+// RUN: --offload-arch=gfx906 -foffload-lto=thin \
+// RUN: -foffload-object-linking -nogpulib -nogpuinc %s 2>&1 \
+// RUN: | FileCheck %s --check-prefix=ENABLE
+// ENABLE: error: unsupported option '-foffload-object-linking' for language mode 'OpenMP'
+
+// RUN: not %clang -### --target=x86_64-unknown-linux-gnu -fopenmp \
+// RUN: --offload-arch=gfx906 -foffload-lto=thin \
+// RUN: -fno-offload-object-linking -nogpulib -nogpuinc %s 2>&1 \
+// RUN: | FileCheck %s --check-prefix=DISABLE
+// DISABLE: error: unsupported option '-fno-offload-object-linking' for language mode 'OpenMP'
+
+int main(void) { return 0; }
diff --git a/clang/test/OffloadTools/clang-linker-wrapper/linker-wrapper.c b/clang/test/OffloadTools/clang-linker-wrapper/linker-wrapper.c
index 550cab4b0604d..c5caa35b258ed 100644
--- a/clang/test/OffloadTools/clang-linker-wrapper/linker-wrapper.c
+++ b/clang/test/OffloadTools/clang-linker-wrapper/linker-wrapper.c
@@ -141,9 +141,10 @@ __attribute__((visibility("protected"), used)) int x;
// RUN: clang-linker-wrapper --dry-run --host-triple=x86_64-unknown-linux-gnu \
// RUN: --linker-path=/usr/bin/ld --device-linker=foo=bar --device-linker=a \
// RUN: --device-linker=nvptx64-nvidia-cuda=b --device-compiler=foo\
+// RUN: --device-linker=amdgpu-amd-amdhsa=-plugin-opt=-amdgpu-enable-object-linking \
// RUN: %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=LINKER-ARGS
-// LINKER-ARGS: clang{{.*}}--target=amdgpu9.08-amd-amdhsa{{.*}}-Xlinker foo=bar{{.*}}-Xlinker a{{.*}}foo
+// LINKER-ARGS: clang{{.*}}--target=amdgpu9.08-amd-amdhsa{{.*}}-Xlinker foo=bar{{.*}}-Xlinker a{{.*}}-Xlinker -plugin-opt=-amdgpu-enable-object-linking{{.*}}foo
// LINKER-ARGS: clang{{.*}}--target=nvptx64-nvidia-cuda{{.*}}-Xlinker foo=bar{{.*}}-Xlinker a -Xlinker b{{.*}}foo
// RUN: not clang-linker-wrapper --dry-run --host-triple=x86_64-unknown-linux-gnu \
diff --git a/flang/test/Driver/omp-offload-object-linking.f90 b/flang/test/Driver/omp-offload-object-linking.f90
new file mode 100644
index 0000000000000..bfaea48da5267
--- /dev/null
+++ b/flang/test/Driver/omp-offload-object-linking.f90
@@ -0,0 +1,17 @@
+! REQUIRES: amdgpu-registered-target
+
+! AMDGPU ThinLTO uses object linking in the frontend and linker wrapper.
+! RUN: %flang -### --target=x86_64-unknown-linux-gnu -fopenmp \
+! RUN: --offload-arch=gfx906 -foffload-lto=thin -nogpulib %s 2>&1 \
+! RUN: | FileCheck %s --check-prefix=THINLTO
+! THINLTO: "{{[^"]*}}flang{{[^"]*}}" "-fc1" "-triple" "amdgpu9.06-amd-amdhsa"
+! THINLTO-SAME: "-emit-llvm-bc"
+! THINLTO-SAME: "-flto=thin"
+! THINLTO-SAME: "-mllvm" "-amdgpu-enable-object-linking"
+! THINLTO: "{{[^"]*}}clang-linker-wrapper"
+! THINLTO-SAME: "--device-linker=amdgpu-amd-amdhsa=-plugin-opt=-amdgpu-enable-object-linking"
+! THINLTO-SAME: "--device-compiler=amdgpu-amd-amdhsa=-flto=thin"
+! THINLTO-NOT: --device-linker=amdgpu-amd-amdhsa=-plugin-opt=-force-import-all
+! THINLTO-NOT: --device-linker=amdgpu-amd-amdhsa=-plugin-opt=-amdgpu-internalize-symbols
+
+end
diff --git a/lld/ELF/AMDGPUObjectLinking.cpp b/lld/ELF/AMDGPUObjectLinking.cpp
new file mode 100644
index 0000000000000..8fd866c37363c
--- /dev/null
+++ b/lld/ELF/AMDGPUObjectLinking.cpp
@@ -0,0 +1,1951 @@
+//===- AMDGPUObjectLinking.cpp - AMDGPU link-time resolution --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implements link-time resolution and patching for AMDGPU object linking.
+//
+// The linker:
+// 1. Uses the linked image's resolved target e_flags
+// 2. Collects SHN_AMDGPU_LDS and named-barrier symbols
+// 3. Parses .amdgpu.info to build the cross-TU call graph, type-ID
+// signatures, LDS and named-barrier uses, and per-function resource usage
+// 4. Resolves function aliases, indirect call edges, and kernel entries
+// 5. Validates call-edge wave-size compatibility
+// 6. Builds a shared SCC condensation graph for kernel-reachable functions
+// 7. Computes per-kernel LDS and named-barrier reachability
+// 8. Assigns LDS offsets and named-barrier IDs
+// 9. Propagates resource usage across the SCC graph (MAX for registers, OR
+// for flags, caller scratch plus maximum callee scratch path)
+// 10. Validates required ABI occupancy and wave-size metadata, call-edge ABI
+// compatibility, and each kernel's LDS usage against its occupancy
+//
+// After resolution, the linker patches kernel descriptors and HSA metadata
+// with the resolved LDS size, named-barrier count, propagated register usage,
+// scratch size, dynamic-stack flag, and related resource fields.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUObjectLinking.h"
+#include "Config.h"
+#include "InputFiles.h"
+#include "InputSection.h"
+#include "SymbolTable.h"
+#include "Symbols.h"
+#include "lld/Common/ErrorHandler.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/BinaryFormat/AMDGPUMetadataVerifier.h"
+#include "llvm/BinaryFormat/ELF.h"
+#include "llvm/BinaryFormat/MsgPackDocument.h"
+#include "llvm/Support/AMDGPUObjLinkingInfo.h"
+#include "llvm/Support/AMDHSAKernelDescriptor.h"
+#include "llvm/Support/Alignment.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/TimeProfiler.h"
+#include "llvm/TargetParser/AMDGPUTargetParser.h"
+
+#define DEBUG_TYPE "amdgpu-object-linking"
+
+using namespace llvm;
+using namespace llvm::amdhsa;
+using namespace llvm::support::endian;
+using namespace llvm::object;
+using namespace llvm::ELF;
+using namespace lld;
+using namespace lld::elf;
+
+namespace {
+
+// One SHN_AMDGPU_LDS symbol after collection and before it is rewritten to a
+// concrete offset.
+struct LDSSymbolInfo {
+ Symbol *sym = nullptr;
+ uint64_t size = 0;
+ Align alignment;
+ uint64_t assignedOffset = 0;
+};
+
+// One named-barrier pseudo-symbol. The linker assigns each barrier a hardware
+// ID range and rewrites the symbol to the encoded barrier address.
+struct NamedBarrierInfo {
+ Symbol *sym = nullptr;
+ uint32_t slotCount = 0;
+ uint32_t assignedBarId = 0;
+};
+
+// Raw resource usage values propagated across the call graph by the linker.
+// No ISA-specific encoding has been applied.
+struct RawLinkingResources {
+ uint32_t numArchVGPR = 0;
+ uint32_t numAccVGPR = 0;
+ uint32_t numSGPR = 0;
+ uint32_t scratchSize = 0;
+ uint32_t ldsSize = 0;
+ uint32_t numNamedBarrier = 0;
+ bool usesVCC = false;
+ bool usesFlatScratch = false;
+ bool hasDynSizedStack = false;
+};
+
+// Fully resolved and encoded resource values ready to write into a kernel
+// descriptor or HSA metadata.
+struct ResolvedLinkingResources {
+ uint32_t totalVGPR = 0;
+ uint32_t totalSGPR = 0;
+ uint32_t numAccVGPR = 0;
+ uint32_t scratchSize = 0;
+ uint32_t ldsSize = 0;
+ uint32_t accumOffset = 0;
+ uint32_t encodedNamedBarrierCount = 0;
+ uint32_t sgprBlocks = 0;
+ bool scratchEnable = false;
+ bool usesDynamicStack = false;
+};
+
+// Per-function resource usage parsed from .amdgpu.info, extended with
+// linker-only validation fields.
+struct CGResourceInfo : RawLinkingResources {
+ uint32_t occupancy = 0;
+ uint32_t waveSize = 0;
+ bool isCuMode = true;
+};
+
+struct CGNode;
+
+// One function body in the device call graph, plus the data computed for that
+// function when it is a kernel entry. Multiple ELF symbols can point at the
+// same function body, and their CGNodes share this object.
+struct CGFunctionInfo {
+ bool isKernel = false;
+
+ CGResourceInfo localRes;
+ CGResourceInfo propagatedRes;
+
+ SmallVector<CGNode *, 4> callees;
+ SmallVector<size_t, 2> ldsUseIndices;
+ DenseSet<size_t> reachableLDS;
+ uint32_t ldsSize = 0;
+
+ SmallVector<size_t, 2> barrierUseIndices;
+ DenseSet<size_t> reachableBarriers;
+ uint32_t numNamedBarrier = 0;
+};
+
+// One symbol in the device call graph.
+struct CGNode {
+ Symbol *sym = nullptr;
+ CGFunctionInfo *info = nullptr;
+};
+
+// Functions and indirect callers that share the same type-id encoding.
+struct SignatureInfo {
+ SmallVector<CGNode *, 4> functions;
+ SmallVector<CGNode *, 4> indirectCallers;
+};
+
+// One strongly connected component in the kernel-reachable call graph. The SCC
+// carries direct and propagated reachability/resource data so LDS,
+// named-barrier, and resource passes can share the same condensed graph.
+struct CallGraphSCC {
+ SmallVector<CGNode *, 4> members;
+ SmallVector<unsigned, 4> callees;
+
+ DenseSet<size_t> reachableLDS;
+ DenseSet<size_t> reachableBarriers;
+
+ CGResourceInfo localRes;
+ CGResourceInfo propagatedRes;
+
+ bool isRecursive = false;
+};
+
+// Cross-object device call graph built from .amdgpu.info records. The graph
+// keeps kernel order stable for diagnostics and later per-kernel patching.
+// After construction, the graph can be condensed into an SCC DAG for
+// call ABI validation, reachability, and resource propagation.
+class AMDGPUCallGraph {
+ SpecificBumpPtrAllocator<CGNode> alloc;
+ SpecificBumpPtrAllocator<CGFunctionInfo> funcInfoAlloc;
+ DenseMap<Symbol *, CGNode *> symToNode;
+ SmallVector<CGNode *> kernelNodes;
+ bool hasLDSUseEntries = false;
+ bool hasBarrierUseEntries = false;
+
+ DenseSet<CGNode *> addressTakenNodes;
+ StringMap<SignatureInfo> signatureMap;
+
+ // SCC condensation state (populated by buildCondensedGraph).
+ SmallVector<CallGraphSCC, 16> sccs;
+ DenseMap<CGFunctionInfo *, unsigned> infoToSCC;
+
+public:
+ CGNode &getOrCreate(Symbol *sym) {
+ auto [it, inserted] = symToNode.try_emplace(sym, nullptr);
+ if (inserted) {
+ it->second = new (alloc.Allocate()) CGNode();
+ it->second->sym = sym;
+ }
+ return *it->second;
+ }
+
+ CGFunctionInfo &getOrCreateInfo(CGNode &node) {
+ if (!node.info)
+ node.info = new (funcInfoAlloc.Allocate()) CGFunctionInfo();
+ return *node.info;
+ }
+
+ CGNode *lookup(Symbol *sym) const {
+ auto it = symToNode.find(sym);
+ return it != symToNode.end() ? it->second : nullptr;
+ }
+
+ void addKernel(CGNode &node) {
+ CGFunctionInfo &info = getOrCreateInfo(node);
+ if (info.isKernel)
+ return;
+ info.isKernel = true;
+ kernelNodes.push_back(&node);
+ }
+
+ void markAddressTaken(CGNode *node) { addressTakenNodes.insert(node); }
+
+ void addIndirectCall(CGNode *caller, StringRef encoding) {
+ signatureMap[encoding].indirectCallers.push_back(caller);
+ }
+
+ void addSignature(CGNode *node, StringRef encoding) {
+ signatureMap[encoding].functions.push_back(node);
+ }
+
+ // After aliases are resolved, build indirect call edges by matching signature
+ // encodings: for each indirect call encoding, the potential callees are
+ // address-taken functions with the same encoding.
+ void buildIndirectEdges() {
+ DenseSet<CGFunctionInfo *> addressTakenInfos;
+ for (CGNode *node : addressTakenNodes) {
+ assert(node->info && "missing resource usage after alias resolution");
+ addressTakenInfos.insert(node->info);
+ }
+
+ for (auto &[encoding, info] : signatureMap) {
+ if (info.indirectCallers.empty())
+ continue;
+
+ DenseSet<CGFunctionInfo *> seenCallees;
+ SmallVector<CGNode *, 4> potentialCallees;
+ for (CGNode *func : info.functions) {
+ assert(func->info && "missing resource usage after alias resolution");
+ if (!addressTakenInfos.count(func->info) ||
+ !seenCallees.insert(func->info).second)
+ continue;
+ potentialCallees.push_back(func);
+ }
+
+ if (potentialCallees.empty())
+ continue;
+
+ for (CGNode *caller : info.indirectCallers) {
+ assert(caller->info && "missing resource usage after alias resolution");
+ for (CGNode *callee : potentialCallees) {
+ LLVM_DEBUG(dbgs() << " indirect edge: " << caller->sym->getName()
+ << " -> " << callee->sym->getName()
+ << " (sig=" << encoding << ")\n");
+ caller->info->callees.push_back(callee);
+ }
+ }
+ }
+ }
+
+ // Resolve function aliases: multiple ELF symbols can point to the same
+ // address (e.g. weak/strong pairs or constructor variants). The compiler
+ // emits .amdgpu.info only for one of them. Make alias nodes share the
+ // canonical node's function info so later graph construction sees the same
+ // ABI, resource, and call metadata through either symbol.
+ void resolveAliases() {
+ DenseMap<std::pair<SectionBase *, uint64_t>, CGNode *> addrToNode;
+ for (auto &[sym, node] : symToNode) {
+ if (!node->info)
+ continue;
+ auto *d = dyn_cast<Defined>(sym);
+ if (!d || !d->section)
+ continue;
+ addrToNode[{d->section, d->value}] = node;
+ }
+
+ for (auto &[sym, node] : symToNode) {
+ if (node->info)
+ continue;
+ auto *d = dyn_cast<Defined>(sym);
+ if (!d || !d->section)
+ continue;
+ auto it = addrToNode.find({d->section, d->value});
+ if (it != addrToNode.end() && it->second != node) {
+ LLVM_DEBUG(dbgs() << " alias: " << sym->getName() << " -> "
+ << it->second->sym->getName() << "\n");
+ node->info = it->second->info;
+ }
+ }
+ }
+
+ void setHasLDSUses() { hasLDSUseEntries = true; }
+ bool hasLDSUses() const { return hasLDSUseEntries; }
+
+ void setHasBarrierUses() { hasBarrierUseEntries = true; }
+ bool hasBarrierUses() const { return hasBarrierUseEntries; }
+
+ ArrayRef<CGNode *> kernels() const { return kernelNodes; }
+
+ using const_iterator = DenseMap<Symbol *, CGNode *>::const_iterator;
+ const_iterator begin() const { return symToNode.begin(); }
+ const_iterator end() const { return symToNode.end(); }
+ bool empty() const { return symToNode.empty(); }
+
+ // Build the SCC DAG used by call ABI validation and propagation. Tarjan emits
+ // SCCs in reverse topological order so every outgoing edge points to an
+ // earlier SCC and consumers can use a single forward sweep.
+ bool buildCondensedGraph(Ctx &ctx);
+
+ // Propagate transitive LDS/barrier reachability over the SCC DAG into each
+ // kernel node.
+ void computeKernelReachability();
+
+ // Propagate resource usage over the SCC DAG. Since SCCs are in reverse
+ // topological order, each callee has already been propagated when its
+ // caller is visited.
+ void propagateResourceUsage();
+
+private:
+ bool buildSCCs(Ctx &ctx, CGNode *node,
+ DenseMap<CGFunctionInfo *, unsigned> &nodeIndex,
+ DenseMap<CGFunctionInfo *, unsigned> &lowLink,
+ DenseSet<CGFunctionInfo *> &onStack,
+ SmallVectorImpl<CGNode *> &stack, unsigned &nextIndex);
+
+ void buildSCCEdges();
+};
+
+// The relocation payload needed to resolve a tagged .amdgpu.info entry.
+struct RelocInfo {
+ uint32_t symIdx = 0;
+ int64_t addend = 0;
+};
+
+struct LDSAllocationTraits {
+ bool hasUses(const AMDGPUCallGraph &cg) const { return cg.hasLDSUses(); }
+
+ const DenseSet<size_t> &reachableResources(const CGNode &kernel) const {
+ return kernel.info->reachableLDS;
+ }
+
+ uint64_t initialValue() const { return 0; }
+ uint64_t size(const LDSSymbolInfo &lds) const { return lds.size; }
+ Align alignment(const LDSSymbolInfo &lds) const { return lds.alignment; }
+ uint64_t assigned(const LDSSymbolInfo &lds) const {
+ return lds.assignedOffset;
+ }
+
+ void setAssigned(LDSSymbolInfo &lds, uint64_t value) const {
+ lds.assignedOffset = value;
+ }
+
+ bool compare(const LDSSymbolInfo &a, const LDSSymbolInfo &b) const {
+ if (a.alignment != b.alignment)
+ return a.alignment > b.alignment;
+ if (a.size != b.size)
+ return a.size > b.size;
+ return a.sym->getName() < b.sym->getName();
+ }
+
+ void printAllocation(const LDSSymbolInfo &lds, uint64_t offset,
+ size_t users) const {
+ LLVM_DEBUG(dbgs() << " allocate " << lds.sym->getName() << " at "
+ << offset << " size=" << lds.size << " users=" << users
+ << "\n");
+ }
+};
+
+struct NamedBarrierAllocationTraits {
+ bool hasUses(const AMDGPUCallGraph &cg) const { return cg.hasBarrierUses(); }
+
+ const DenseSet<size_t> &reachableResources(const CGNode &kernel) const {
+ return kernel.info->reachableBarriers;
+ }
+
+ uint64_t initialValue() const { return 1; }
+ uint64_t size(const NamedBarrierInfo &bar) const { return bar.slotCount; }
+ Align alignment(const NamedBarrierInfo &) const { return Align(1); }
+ uint64_t assigned(const NamedBarrierInfo &bar) const {
+ return bar.assignedBarId;
+ }
+
+ void setAssigned(NamedBarrierInfo &bar, uint64_t value) const {
+ bar.assignedBarId = static_cast<uint32_t>(value);
+ }
+
+ bool compare(const NamedBarrierInfo &a, const NamedBarrierInfo &b) const {
+ if (a.slotCount != b.slotCount)
+ return a.slotCount > b.slotCount;
+ return a.sym->getName() < b.sym->getName();
+ }
+
+ void printAllocation(const NamedBarrierInfo &bar, uint64_t barId,
+ size_t users) const {
+ LLVM_DEBUG(dbgs() << " allocate " << bar.sym->getName() << " at "
+ << barId << " slots=" << bar.slotCount
+ << " users=" << users << "\n");
+ }
+};
+
+} // namespace
+
+//===----------------------------------------------------------------------===//
+// Resource allocation helpers
+//===----------------------------------------------------------------------===//
+
+// Return the placement frontier shared by all kernels in users. Each kernel has
+// an independent frontier, and a resource used by several kernels must be
+// placed after every one of those kernels' existing live resources.
+// initialValue is the resource-specific base frontier, e.g. LDS offsets start
+// at 0 while named-barrier IDs start at 1.
+static uint64_t
+getMaxKernelFrontier(ArrayRef<CGNode *> users,
+ const DenseMap<CGNode *, uint64_t> &kernelFrontiers,
+ uint64_t initialValue) {
+ uint64_t frontier = initialValue;
+ for (CGNode *user : users) {
+ auto it = kernelFrontiers.find(user);
+ assert(it != kernelFrontiers.end() && "missing kernel frontier");
+ frontier = std::max(frontier, it->second);
+ }
+ return frontier;
+}
+
+// Assign one packed, deterministic layout used by every kernel. This is the
+// fallback when the input objects do not provide .amdgpu.info reachability for
+// a resource kind. Non-zero-sized resources advance a single global frontier.
+// Zero-sized resources are assigned to the first suitably aligned address after
+// the fixed layout, matching the dynamic-LDS convention while also giving
+// zero-slot barriers a stable value if they ever appear.
+template <typename ResourceT, typename Traits>
+static void assignUniversalResources(SmallVectorImpl<ResourceT> &resources,
+ const Traits &traits) {
+ llvm::sort(resources, [&](const ResourceT &a, const ResourceT &b) {
+ return traits.compare(a, b);
+ });
+ uint64_t frontier = traits.initialValue();
+ for (ResourceT &resource : resources) {
+ if (traits.size(resource) == 0)
+ continue;
+ frontier = alignTo(frontier, traits.alignment(resource));
+ traits.setAssigned(resource, frontier);
+ frontier += traits.size(resource);
+ }
+ Align maxZeroAlign(1);
+ for (ResourceT &resource : resources) {
+ if (traits.size(resource) == 0)
+ maxZeroAlign = std::max(maxZeroAlign, traits.alignment(resource));
+ }
+ uint64_t zeroBase = alignTo(frontier, maxZeroAlign);
+ for (ResourceT &resource : resources) {
+ if (traits.size(resource) == 0)
+ traits.setAssigned(resource, zeroBase);
+ }
+}
+
+#ifndef NDEBUG
+template <typename ResourceT, typename Traits>
+static void assertResourceLayoutInvariants(
+ ArrayRef<ResourceT> resources, ArrayRef<CGNode *> kernels,
+ const DenseMap<size_t, SmallVector<CGNode *, 4>> &resourceToUsers,
+ const DenseSet<size_t> &assigned, const Traits &traits) {
+ assert(assigned.size() == resources.size() && "not all resources assigned");
+ for (size_t idx = 0, e = resources.size(); idx < e; ++idx)
+ assert(assigned.count(idx) && "resource not assigned");
+
+ for (const auto &[idx, users] : resourceToUsers) {
+ uint64_t start = traits.assigned(resources[idx]);
+ for (CGNode *user : users) {
+ assert(traits.reachableResources(*user).count(idx) &&
+ "inconsistent resource user map");
+ assert(traits.assigned(resources[idx]) == start &&
+ "resource has non-uniform assignment across kernels");
+ }
+ }
+
+ for (CGNode *kernel : kernels) {
+ SmallVector<std::pair<uint64_t, uint64_t>, 8> intervals;
+ for (size_t idx : traits.reachableResources(*kernel)) {
+ uint64_t size = traits.size(resources[idx]);
+ if (size == 0)
+ continue;
+ uint64_t begin = traits.assigned(resources[idx]);
+ intervals.push_back({begin, begin + size});
+ }
+
+ llvm::sort(intervals);
+ for (size_t i = 1, e = intervals.size(); i < e; ++i)
+ assert(intervals[i - 1].second <= intervals[i].first &&
+ "kernel has overlapping resource intervals");
+ }
+}
+#endif
+
+// Assign a single global resource range to each resource while minimizing the
+// per-kernel high-water mark. ResourceT is the linker-side record for one
+// allocatable entity. It is currently either LDSSymbolInfo, where the assigned
+// value is an LDS byte offset, or NamedBarrierInfo, where the assigned value is
+// the first hardware named-barrier ID in the resource's ID range.
+//
+// The algorithm uses a per-kernel frontier. For a resource used by kernels K,
+// place it at the maximum current frontier of K, rounded up to the resource's
+// alignment, then advance only those kernels by the resource size. This keeps
+// one uniform assignment per resource, while allowing resources reached by
+// disjoint kernel sets to overlap. Resources are assigned in descending number
+// of reaching kernels, with Traits::compare breaking ties in a
+// resource-specific way.
+//
+// Traits supplies the resource-specific policy:
+// - hasUses(cg): whether .amdgpu.info reachability data is available.
+// - reachableResources(kernel): the DenseSet<size_t> of resource indices
+// reachable from the kernel.
+// - initialValue(): the first valid frontier value, e.g. 0 for LDS offsets
+// and 1 for named-barrier IDs.
+// - size(resource): the amount by which a non-zero resource advances a
+// kernel frontier.
+// - alignment(resource): required alignment of the assigned value.
+// - assigned(resource) / setAssigned(resource, value): accessors for the
+// resource's final assignment.
+// - compare(a, b): deterministic allocation priority for equal user counts.
+// - printAllocation(resource, value, users): debug logging for claimed
+// resources.
+//
+// If reachability is absent, all resources are laid out universally using the
+// same Traits policy. If a resource is unreachable from every kernel, it still
+// receives a valid assignment for relocation resolution, but it does not affect
+// any kernel frontier.
+template <typename ResourceT, typename Traits>
+static void assignGroupedResources(SmallVectorImpl<ResourceT> &resources,
+ AMDGPUCallGraph &cg, const Traits &traits) {
+ if (!traits.hasUses(cg)) {
+ assignUniversalResources(resources, traits);
+ return;
+ }
+
+ SmallVector<CGNode *, 8> kernels(cg.kernels().begin(), cg.kernels().end());
+ DenseMap<size_t, SmallVector<CGNode *, 4>> resourceToUsers;
+ DenseMap<CGNode *, uint64_t> kernelFrontiers;
+ for (CGNode *kernel : kernels) {
+ kernelFrontiers[kernel] = traits.initialValue();
+ for (size_t idx : traits.reachableResources(*kernel))
+ resourceToUsers[idx].push_back(kernel);
+ }
+
+ auto idxCmp = [&](size_t a, size_t b) {
+ return traits.compare(resources[a], resources[b]);
+ };
+ auto getUserCount = [&](size_t idx) {
+ auto it = resourceToUsers.find(idx);
+ return it == resourceToUsers.end() ? 0 : it->second.size();
+ };
+ auto claimedCmp = [&](size_t a, size_t b) {
+ size_t aUsers = getUserCount(a);
+ size_t bUsers = getUserCount(b);
+ if (aUsers != bUsers)
+ return aUsers > bUsers;
+ return idxCmp(a, b);
+ };
+
+ DenseSet<size_t> assigned;
+ // Split resources by two independent properties:
+ // - claimed resources are reachable from at least one kernel and therefore
+ // participate in per-kernel frontier allocation; unclaimed resources only
+ // need a valid standalone assignment for relocation resolution.
+ // - non-zero resources consume frontier space; zero-sized resources get a
+ // stable aligned assignment but must not advance any kernel frontier.
+ // In practice, zero-sized resources are dynamic LDS symbols. Named
+ // barriers should have nonzero slot counts, but the shared allocator
+ // keeps the zero-size handling generic.
+ SmallVector<size_t, 8> claimedNonZero;
+ SmallVector<size_t, 4> claimedZeroSize;
+ SmallVector<size_t, 8> unclaimedNonZero;
+ SmallVector<size_t, 4> unclaimedZeroSize;
+
+ for (size_t i = 0, e = resources.size(); i < e; ++i) {
+ bool hasUsers = getUserCount(i) != 0;
+ if (hasUsers && traits.size(resources[i]) != 0)
+ claimedNonZero.push_back(i);
+ else if (hasUsers)
+ claimedZeroSize.push_back(i);
+ else if (traits.size(resources[i]) == 0)
+ unclaimedZeroSize.push_back(i);
+ else
+ unclaimedNonZero.push_back(i);
+ }
+
+ llvm::sort(claimedNonZero, claimedCmp);
+ llvm::sort(claimedZeroSize, claimedCmp);
+
+ auto assignResource = [&](size_t idx, uint64_t start) {
+ traits.setAssigned(resources[idx], start);
+ [[maybe_unused]] bool inserted = assigned.insert(idx).second;
+ assert(inserted && "resource assigned multiple times");
+ };
+
+ auto allocateClaimed = [&](size_t idx, bool advanceFrontier) {
+ ArrayRef<CGNode *> users = resourceToUsers[idx];
+ uint64_t start = alignTo(
+ getMaxKernelFrontier(users, kernelFrontiers, traits.initialValue()),
+ traits.alignment(resources[idx]));
+ assignResource(idx, start);
+ if (!advanceFrontier)
+ return;
+
+ uint64_t next = start + traits.size(resources[idx]);
+ traits.printAllocation(resources[idx], start, users.size());
+ for (CGNode *user : users)
+ kernelFrontiers[user] = next;
+ };
+
+ for (size_t idx : claimedNonZero)
+ allocateClaimed(idx, /*advanceFrontier=*/true);
+
+ for (size_t idx : claimedZeroSize)
+ allocateClaimed(idx, /*advanceFrontier=*/false);
+
+ // Resources not reachable from any kernel still need valid addresses for
+ // relocation resolution. Assign them independently from the initial frontier
+ // because no kernel's resource total depends on them.
+ if (!unclaimedNonZero.empty() || !unclaimedZeroSize.empty()) {
+ llvm::sort(unclaimedNonZero, idxCmp);
+ llvm::sort(unclaimedZeroSize, idxCmp);
+ uint64_t frontier = traits.initialValue();
+ for (size_t idx : unclaimedNonZero) {
+ frontier = alignTo(frontier, traits.alignment(resources[idx]));
+ assignResource(idx, frontier);
+ frontier += traits.size(resources[idx]);
+ }
+ if (!unclaimedZeroSize.empty()) {
+ Align maxZeroAlign(1);
+ for (size_t idx : unclaimedZeroSize)
+ maxZeroAlign = std::max(maxZeroAlign, traits.alignment(resources[idx]));
+ uint64_t zeroBase = alignTo(frontier, maxZeroAlign);
+ for (size_t idx : unclaimedZeroSize)
+ assignResource(idx, zeroBase);
+ }
+ }
+
+#ifndef NDEBUG
+ assertResourceLayoutInvariants(ArrayRef<ResourceT>(resources), kernels,
+ resourceToUsers, assigned, traits);
+#endif
+}
+
+//===----------------------------------------------------------------------===//
+// Section parsing
+//===----------------------------------------------------------------------===//
+
+// Build relocation map: byte offset -> {ELF symbol index, addend}.
+// The addend is needed to resolve STT_SECTION symbols: ELF assemblers
+// canonically convert relocations for local symbols in their own sections
+// into section_symbol + addend form, so we must track the addend to map
+// back to the actual function symbol (see resolveSecSymbol).
+// Extract explicit RELA/CREL addends and treat REL entries as addend zero.
+template <class RelTy> static int64_t getAddend(const RelTy &rel) {
+ if constexpr (RelTy::HasAddend)
+ return rel.r_addend;
+ return 0;
+}
+
+// Build the relocation lookup for one .amdgpu.info section.
+template <class ELFT>
+static DenseMap<uint64_t, RelocInfo> buildRelocMap(ObjFile<ELFT> *obj,
+ uint32_t secIndex) {
+ ArrayRef<typename ELFT::Shdr> objSections = obj->template getELFShdrs<ELFT>();
+ const ELFFile<ELFT> &elfObj = obj->getObj();
+ DenseMap<uint64_t, RelocInfo> relocMap;
+ for (size_t i = 0, e = objSections.size(); i < e; ++i) {
+ const auto &relSec = objSections[i];
+ if (relSec.sh_info != secIndex)
+ continue;
+ if (relSec.sh_type == SHT_RELA) {
+ for (const auto &rel :
+ CHECK(elfObj.relas(relSec), "could not read rela section"))
+ relocMap[rel.r_offset] = {rel.getSymbol(false), getAddend(rel)};
+ break;
+ }
+ if (relSec.sh_type == SHT_REL) {
+ for (const auto &rel :
+ CHECK(elfObj.rels(relSec), "could not read rel section"))
+ relocMap[rel.r_offset] = {rel.getSymbol(false), 0};
+ break;
+ }
+ if (relSec.sh_type == SHT_CREL) {
+ auto crels = CHECK(elfObj.crels(relSec), "could not read crel section");
+ for (const auto &rel : crels.first)
+ relocMap[rel.r_offset] = {rel.getSymbol(false), getAddend(rel)};
+ for (const auto &rel : crels.second)
+ relocMap[rel.r_offset] = {rel.getSymbol(false), getAddend(rel)};
+ break;
+ }
+ }
+ return relocMap;
+}
+
+// Map function definition addresses back to symbols so section-symbol
+// relocations can recover the original named function.
+template <class ELFT>
+static DenseMap<std::pair<SectionBase *, uint64_t>, Symbol *>
+buildSymbolAddressMap(ObjFile<ELFT> *obj) {
+ DenseMap<std::pair<SectionBase *, uint64_t>, Symbol *> addrToSym;
+ for (Symbol *sym : obj->getSymbols()) {
+ if (!sym || sym->getName().empty())
+ continue;
+ auto *def = dyn_cast<Defined>(sym);
+ if (!def || !def->section)
+ continue;
+ addrToSym.try_emplace({def->section, def->value}, sym);
+ }
+ return addrToSym;
+}
+
+// Resolve an STT_SECTION symbol + addend to the named function symbol at that
+// address. ELF assemblers replace relocations targeting local symbols with
+// section_symbol + addend (standard ELF canonicalization). Since section
+// symbols have empty names in LLD, the .amdgpu.info parser cannot identify
+// the function from the section symbol alone. Use a per-object address map to
+// find the named Defined symbol at the matching section and offset.
+static Symbol *resolveSecSymbol(
+ Symbol &secSym, int64_t addend,
+ const DenseMap<std::pair<SectionBase *, uint64_t>, Symbol *> &addrToSym) {
+ auto *secDef = dyn_cast<Defined>(&secSym);
+ if (!secDef || !secDef->section)
+ return nullptr;
+ auto it = addrToSym.find({secDef->section, static_cast<uint64_t>(addend)});
+ return it == addrToSym.end() ? nullptr : it->second;
+}
+
+using namespace llvm::AMDGPU;
+
+// Resolve a relocation attached to a .amdgpu.info payload field. The result is
+// either the referenced symbol or the named function represented by a
+// section-symbol relocation.
+template <class ELFT>
+static Symbol *resolveRelocSym(
+ ObjFile<ELFT> *obj, const DenseMap<uint64_t, RelocInfo> &relocMap,
+ const DenseMap<std::pair<SectionBase *, uint64_t>, Symbol *> &addrToSym,
+ uint64_t off, StringRef diagName) {
+ auto it = relocMap.find(off);
+ if (it == relocMap.end())
+ return nullptr;
+ Symbol *sym = &obj->getSymbol(it->second.symIdx);
+ if (sym->getName().empty()) {
+ sym = resolveSecSymbol(*sym, it->second.addend, addrToSym);
+ if (!sym) {
+ Warn(obj->ctx) << obj->getName() << ": .amdgpu.info " << diagName
+ << " at offset " << off
+ << " references a section symbol that could not be "
+ "resolved to a named function symbol";
+ }
+ }
+ return sym;
+}
+
+// Return a string from .amdgpu.strtab without reading past the section if
+// malformed input omits the trailing NUL.
+static StringRef getInfoString(StringRef strtab, uint32_t offset) {
+ if (offset >= strtab.size())
+ return StringRef();
+ StringRef tail = strtab.substr(offset);
+ size_t nul = tail.find('\0');
+ return tail.substr(0, nul);
+}
+
+// .amdgpu.info is emitted as a standalone section, so records from losing weak
+// definitions or discarded COMDAT groups can survive. Only the prevailing
+// definition may contribute local resources and edges. Non-prevailing scopes
+// may still contribute type IDs for address-taken declarations.
+static bool isPrevailingInfoFunction(ELFFileBase *obj, Symbol *sym) {
+ auto *def = dyn_cast<Defined>(sym);
+ if (!def || !def->section || def->section == &InputSection::discarded)
+ return false;
+ return def->section->file == obj;
+}
+
+namespace {
+struct AMDGPUInfoSectionIndices {
+ uint32_t info = 0;
+ uint32_t strtab = 0;
+};
+
+template <class ELFT>
+static AMDGPUInfoSectionIndices findAMDGPUInfoSections(ObjFile<ELFT> *obj) {
+ AMDGPUInfoSectionIndices result;
+ ArrayRef<typename ELFT::Shdr> sections = obj->template getELFShdrs<ELFT>();
+ StringRef shstrtab = CHECK(obj->getObj().getSectionStringTable(sections),
+ "could not read section string table");
+ for (const auto &[index, sec] : llvm::enumerate(sections)) {
+ if (sec.sh_type != SHT_PROGBITS && sec.sh_type != SHT_STRTAB)
+ continue;
+ StringRef name = CHECK(obj->getObj().getSectionName(sec, shstrtab),
+ "could not read section name");
+ if (name == ".amdgpu.info")
+ result.info = static_cast<uint32_t>(index);
+ else if (name == ".amdgpu.strtab")
+ result.strtab = static_cast<uint32_t>(index);
+ }
+ return result;
+}
+} // namespace
+
+// Parse one object's .amdgpu.info and .amdgpu.strtab sections into the shared
+// call graph, direct resource records, and direct LDS/barrier uses.
+template <class ELFT>
+static void parseInfoSection(ObjFile<ELFT> *obj, AMDGPUCallGraph &cg,
+ const DenseMap<Symbol *, size_t> &ldsSymToIndex,
+ const DenseMap<Symbol *, size_t> &barSymToIndex) {
+ AMDGPUInfoSectionIndices indices = findAMDGPUInfoSections(obj);
+ if (indices.info == 0)
+ return;
+
+ ArrayRef<typename ELFT::Shdr> objSections = obj->template getELFShdrs<ELFT>();
+ const auto &sec = objSections[indices.info];
+ ArrayRef<uint8_t> data = CHECK(obj->getObj().getSectionContents(sec),
+ "could not read .amdgpu.info section");
+ if (data.empty())
+ return;
+
+ DenseMap<uint64_t, RelocInfo> relocMap = buildRelocMap(obj, indices.info);
+ DenseMap<std::pair<SectionBase *, uint64_t>, Symbol *> addrToSym =
+ buildSymbolAddressMap(obj);
+
+ StringRef strtab;
+ if (indices.strtab != 0) {
+ const auto &strtabSec = objSections[indices.strtab];
+ ArrayRef<uint8_t> strtabData =
+ CHECK(obj->getObj().getSectionContents(strtabSec),
+ "could not read .amdgpu.strtab section");
+ strtab = StringRef(reinterpret_cast<const char *>(strtabData.data()),
+ strtabData.size());
+ }
+
+ CGNode *curNode = nullptr;
+ bool curFuncIsPrevailing = false;
+ size_t pos = 0;
+ while (pos + 2 <= data.size()) {
+ uint8_t kind = data[pos];
+ uint8_t len = data[pos + 1];
+ size_t payloadOff = pos + 2;
+ if (payloadOff + len > data.size())
+ break;
+
+ switch (kind) {
+ case static_cast<uint8_t>(AMDGPU::InfoKind::INFO_FUNC): {
+ if (len < 8)
+ break;
+ Symbol *sym =
+ resolveRelocSym(obj, relocMap, addrToSym, payloadOff, "func");
+ if (!sym || sym->getName().empty()) {
+ curNode = nullptr;
+ curFuncIsPrevailing = false;
+ break;
+ }
+ curNode = &cg.getOrCreate(sym);
+ curFuncIsPrevailing = isPrevailingInfoFunction(obj, sym);
+ break;
+ }
+ case static_cast<uint8_t>(AMDGPU::InfoKind::INFO_FLAGS): {
+ if (!curNode || !curFuncIsPrevailing || len < 4)
+ break;
+ CGFunctionInfo &info = cg.getOrCreateInfo(*curNode);
+ AMDGPU::FuncInfoFlags flags = static_cast<AMDGPU::FuncInfoFlags>(
+ read32le(data.data() + payloadOff));
+ info.localRes.usesVCC = !!(flags & AMDGPU::FuncInfoFlags::FUNC_USES_VCC);
+ info.localRes.usesFlatScratch =
+ !!(flags & AMDGPU::FuncInfoFlags::FUNC_USES_FLAT_SCRATCH);
+ info.localRes.hasDynSizedStack =
+ !!(flags & AMDGPU::FuncInfoFlags::FUNC_HAS_DYN_STACK);
+ info.localRes.isCuMode = !(flags & AMDGPU::FuncInfoFlags::FUNC_WGP_MODE);
+ info.localRes.waveSize =
+ !!(flags & AMDGPU::FuncInfoFlags::FUNC_WAVE32) ? 32 : 64;
+ break;
+ }
+ case static_cast<uint8_t>(AMDGPU::InfoKind::INFO_NUM_VGPR): {
+ if (!curNode || !curFuncIsPrevailing || len < 4)
+ break;
+ cg.getOrCreateInfo(*curNode).localRes.numArchVGPR =
+ read32le(data.data() + payloadOff);
+ break;
+ }
+ case static_cast<uint8_t>(AMDGPU::InfoKind::INFO_NUM_AGPR): {
+ if (!curNode || !curFuncIsPrevailing || len < 4)
+ break;
+ cg.getOrCreateInfo(*curNode).localRes.numAccVGPR =
+ read32le(data.data() + payloadOff);
+ break;
+ }
+ case static_cast<uint8_t>(AMDGPU::InfoKind::INFO_NUM_SGPR): {
+ if (!curNode || !curFuncIsPrevailing || len < 4)
+ break;
+ cg.getOrCreateInfo(*curNode).localRes.numSGPR =
+ read32le(data.data() + payloadOff);
+ break;
+ }
+ case static_cast<uint8_t>(AMDGPU::InfoKind::INFO_PRIVATE_SEGMENT_SIZE): {
+ if (!curNode || !curFuncIsPrevailing || len < 4)
+ break;
+ cg.getOrCreateInfo(*curNode).localRes.scratchSize =
+ read32le(data.data() + payloadOff);
+ break;
+ }
+ case static_cast<uint8_t>(AMDGPU::InfoKind::INFO_OCCUPANCY): {
+ if (!curNode || !curFuncIsPrevailing || len < 4)
+ break;
+ cg.getOrCreateInfo(*curNode).localRes.occupancy =
+ read32le(data.data() + payloadOff);
+ break;
+ }
+ case static_cast<uint8_t>(AMDGPU::InfoKind::INFO_USE): {
+ if (!curNode || !curFuncIsPrevailing || len < 8)
+ break;
+ Symbol *resSym =
+ resolveRelocSym(obj, relocMap, addrToSym, payloadOff, "use");
+ if (!resSym || resSym->getName().empty())
+ break;
+ auto barIt = barSymToIndex.find(resSym);
+ if (barIt != barSymToIndex.end()) {
+ LLVM_DEBUG(dbgs() << " barrier use: " << curNode->sym->getName()
+ << " -> " << resSym->getName() << "\n");
+ cg.getOrCreateInfo(*curNode).barrierUseIndices.push_back(barIt->second);
+ cg.setHasBarrierUses();
+ break;
+ }
+ auto ldsIt = ldsSymToIndex.find(resSym);
+ if (ldsIt != ldsSymToIndex.end()) {
+ LLVM_DEBUG(dbgs() << " use: " << curNode->sym->getName() << " -> "
+ << resSym->getName() << "\n");
+ cg.getOrCreateInfo(*curNode).ldsUseIndices.push_back(ldsIt->second);
+ cg.setHasLDSUses();
+ }
+ break;
+ }
+ case static_cast<uint8_t>(AMDGPU::InfoKind::INFO_CALL): {
+ if (!curNode || !curFuncIsPrevailing || len < 8)
+ break;
+ Symbol *dstSym =
+ resolveRelocSym(obj, relocMap, addrToSym, payloadOff, "call");
+ if (!dstSym || dstSym->getName().empty())
+ break;
+ CGNode &dst = cg.getOrCreate(dstSym);
+ LLVM_DEBUG(dbgs() << " call: " << curNode->sym->getName() << " -> "
+ << dstSym->getName() << "\n");
+ cg.getOrCreateInfo(*curNode).callees.push_back(&dst);
+ break;
+ }
+ case static_cast<uint8_t>(AMDGPU::InfoKind::INFO_INDIRECT_CALL): {
+ if (!curNode || !curFuncIsPrevailing || len < 4)
+ break;
+ uint32_t typeIdOff = read32le(data.data() + payloadOff);
+ StringRef typeId = getInfoString(strtab, typeIdOff);
+ if (!typeId.empty()) {
+ LLVM_DEBUG(dbgs() << " indirect-call: " << curNode->sym->getName()
+ << " enc=" << typeId << "\n");
+ cg.addIndirectCall(curNode, typeId);
+ }
+ break;
+ }
+ case static_cast<uint8_t>(AMDGPU::InfoKind::INFO_TYPEID): {
+ if (!curNode || len < 4)
+ break;
+ cg.markAddressTaken(curNode);
+ uint32_t typeIdOff = read32le(data.data() + payloadOff);
+ StringRef typeId = getInfoString(strtab, typeIdOff);
+ if (!typeId.empty()) {
+ LLVM_DEBUG(dbgs() << " signature: " << curNode->sym->getName()
+ << " typeId=" << typeId << "\n");
+ cg.addSignature(curNode, typeId);
+ }
+ break;
+ }
+ default:
+ LLVM_DEBUG(dbgs() << " unknown info kind " << (unsigned)kind
+ << " (len=" << (unsigned)len << "), skipping\n");
+ break;
+ }
+
+ pos = payloadOff + len;
+ }
+
+ // Emit debug summary of parsed resources.
+ if (curNode && curNode->info) {
+ LLVM_DEBUG({
+ for (auto &[sym, node] : cg) {
+ if (!node->info)
+ continue;
+ const CGResourceInfo &info = node->info->localRes;
+ dbgs() << " resource: " << sym->getName()
+ << " vgpr=" << info.numArchVGPR << " agpr=" << info.numAccVGPR
+ << " sgpr=" << info.numSGPR << " scratch=" << info.scratchSize
+ << " occupancy=" << info.occupancy << "\n";
+ }
+ });
+ }
+}
+
+// Kernels are identified by the companion kernel descriptor symbol emitted as
+// <kernel>.kd.
+static bool hasKernelDescriptor(Ctx &ctx, Symbol *sym) {
+ std::string kdName = (sym->getName() + ".kd").str();
+ Symbol *kdSym = ctx.symtab->find(kdName);
+ Defined *kdDef = dyn_cast_or_null<Defined>(kdSym);
+ return kdDef && kdDef->section;
+}
+
+// Mark graph nodes with kernel descriptors as roots for reachability and
+// resource propagation.
+static void markKernelsWithDescriptors(Ctx &ctx, AMDGPUCallGraph &cg) {
+ for (auto &[sym, node] : cg) {
+ assert(node->info && "missing resource usage after alias resolution");
+ if (hasKernelDescriptor(ctx, sym)) {
+ LLVM_DEBUG(dbgs() << " kernel: " << sym->getName() << "\n");
+ cg.addKernel(*node);
+ }
+ }
+}
+
+//===----------------------------------------------------------------------===//
+// Call graph SCC condensation, reachability, and resource propagation
+//===----------------------------------------------------------------------===//
+
+static bool validateLocalABI(Ctx &ctx, const AMDGPUCallGraph &cg,
+ const AMDGPU::TargetID &targetInfo) {
+ // These checks do not depend on reachability or propagated resource usage.
+ StringRef arch = AMDGPU::getArchNameAMDGCN(targetInfo.getGPUKind());
+ DenseSet<CGFunctionInfo *> seen;
+ for (auto &[sym, node] : cg) {
+ assert(node->info && "missing resource usage after alias resolution");
+ if (!seen.insert(node->info).second)
+ continue;
+
+ const CGResourceInfo &localRes = node->info->localRes;
+ if (localRes.occupancy == 0) {
+ Err(ctx) << "AMDGPU: function '" << node->sym->getName()
+ << "' has invalid ABI occupancy 0";
+ LLVM_DEBUG(dbgs() << " resolve " << node->sym->getName()
+ << " (invalid occupancy)\n");
+ return false;
+ }
+
+ if (localRes.waveSize != 32 && localRes.waveSize != 64) {
+ Err(ctx) << "AMDGPU: function '" << node->sym->getName()
+ << "' has invalid ABI wave size " << localRes.waveSize;
+ LLVM_DEBUG(dbgs() << " resolve " << node->sym->getName()
+ << " (invalid wave size)\n");
+ return false;
+ }
+
+ if (localRes.waveSize == 32 &&
+ !AMDGPU::isObjectLinkingWaveSizeSupported(targetInfo, 32)) {
+ Err(ctx) << "AMDGPU: function '" << node->sym->getName()
+ << "' uses unsupported ABI wave32 for " << arch;
+ LLVM_DEBUG(dbgs() << " resolve " << node->sym->getName()
+ << " (unsupported wave32)\n");
+ return false;
+ }
+
+ if (localRes.waveSize == 64 &&
+ !AMDGPU::isObjectLinkingWaveSizeSupported(targetInfo, 64)) {
+ Err(ctx) << "AMDGPU: function '" << node->sym->getName()
+ << "' uses unsupported ABI wave64 for " << arch;
+ LLVM_DEBUG(dbgs() << " resolve " << node->sym->getName()
+ << " (unsupported wave64)\n");
+ return false;
+ }
+ }
+ return true;
+}
+
+// Validate ABI compatibility across call edges. A caller and callee must use
+// the same wavefront size and CU/WGP mode because the calling convention,
+// register layout, and instruction semantics depend on these properties.
+static bool validateCallABICompatibility(Ctx &ctx, const CGNode *caller,
+ const CGNode *callee) {
+ assert(caller->info && callee->info &&
+ "missing resource usage after alias resolution");
+ const CGResourceInfo &callerRes = caller->info->localRes;
+ const CGResourceInfo &calleeRes = callee->info->localRes;
+ uint32_t callerWaveSize = callerRes.waveSize;
+ uint32_t calleeWaveSize = calleeRes.waveSize;
+ if (callerWaveSize != calleeWaveSize) {
+ Err(ctx) << "AMDGPU object linking: wave size mismatch in call from '"
+ << caller->sym->getName() << "' (wave" << callerWaveSize
+ << ") to '" << callee->sym->getName() << "' (wave"
+ << calleeWaveSize << ")";
+ return false;
+ }
+
+ if (callerRes.isCuMode != calleeRes.isCuMode) {
+ Err(ctx) << "AMDGPU object linking: CU/WGP mode mismatch in call from '"
+ << caller->sym->getName() << "' ("
+ << (callerRes.isCuMode ? "CU" : "WGP") << ") to '"
+ << callee->sym->getName() << "' ("
+ << (calleeRes.isCuMode ? "CU" : "WGP") << ")";
+ return false;
+ }
+
+ return true;
+}
+
+// Merge resource usage from another CGResourceInfo into the result using
+// element-wise max for register/scratch counts and OR for boolean flags.
+static void mergeResourceInfo(CGResourceInfo &result,
+ const CGResourceInfo &info) {
+ result.numArchVGPR = std::max(result.numArchVGPR, info.numArchVGPR);
+ result.numAccVGPR = std::max(result.numAccVGPR, info.numAccVGPR);
+ result.numSGPR = std::max(result.numSGPR, info.numSGPR);
+ result.scratchSize = std::max(result.scratchSize, info.scratchSize);
+ result.usesVCC |= info.usesVCC;
+ result.usesFlatScratch |= info.usesFlatScratch;
+ result.hasDynSizedStack |= info.hasDynSizedStack;
+}
+
+bool AMDGPUCallGraph::buildCondensedGraph(Ctx &ctx) {
+ DenseMap<CGFunctionInfo *, unsigned> nodeIndex;
+ DenseMap<CGFunctionInfo *, unsigned> lowLink;
+ DenseSet<CGFunctionInfo *> onStack;
+ SmallVector<CGNode *, 16> stack;
+ unsigned nextIndex = 0;
+
+ // Start from kernels so resource diagnostics are limited to code that can
+ // execute. A validation failure aborts the build; callers do not inspect the
+ // partially constructed SCC state.
+ for (CGNode *kernel : kernels()) {
+ assert(kernel->info && "missing resource usage after alias resolution");
+ if (nodeIndex.count(kernel->info))
+ continue;
+ if (!buildSCCs(ctx, kernel, nodeIndex, lowLink, onStack, stack, nextIndex))
+ return false;
+ }
+
+ buildSCCEdges();
+ return true;
+}
+
+bool AMDGPUCallGraph::buildSCCs(Ctx &ctx, CGNode *node,
+ DenseMap<CGFunctionInfo *, unsigned> &nodeIndex,
+ DenseMap<CGFunctionInfo *, unsigned> &lowLink,
+ DenseSet<CGFunctionInfo *> &onStack,
+ SmallVectorImpl<CGNode *> &stack,
+ unsigned &nextIndex) {
+ assert(node->info && "missing resource usage after alias resolution");
+ CGFunctionInfo *func = node->info;
+
+ nodeIndex[func] = nextIndex;
+ lowLink[func] = nextIndex;
+ ++nextIndex;
+ stack.push_back(node);
+ onStack.insert(func);
+
+ CGFunctionInfo &info = *func;
+ const CGResourceInfo &localRes = info.localRes;
+ LLVM_DEBUG(dbgs() << " resolve " << node->sym->getName()
+ << " (has local res: vgpr=" << localRes.numArchVGPR
+ << " sgpr=" << localRes.numSGPR
+ << " scratch=" << localRes.scratchSize << ")\n");
+
+ if (!info.callees.empty()) {
+ LLVM_DEBUG({
+ dbgs() << " " << node->sym->getName() << " has " << info.callees.size()
+ << " callees:";
+ for (CGNode *c : info.callees)
+ dbgs() << " " << c->sym->getName();
+ dbgs() << "\n";
+ });
+
+ for (CGNode *callee : info.callees) {
+ assert(callee->info &&
+ "missing callee resource usage after alias resolution");
+ if (!validateCallABICompatibility(ctx, node, callee))
+ return false;
+
+ const CGResourceInfo &calleeRes = callee->info->localRes;
+ if (calleeRes.occupancy < localRes.occupancy) {
+ Err(ctx) << "AMDGPU: incompatible ABI occupancy: function '"
+ << node->sym->getName() << "' requires occupancy "
+ << localRes.occupancy << " but calls '"
+ << callee->sym->getName() << "' with occupancy "
+ << calleeRes.occupancy;
+ return false;
+ }
+
+ CGFunctionInfo *calleeFunc = callee->info;
+ auto calleeIndex = nodeIndex.find(calleeFunc);
+ if (calleeIndex == nodeIndex.end()) {
+ if (!buildSCCs(ctx, callee, nodeIndex, lowLink, onStack, stack,
+ nextIndex))
+ return false;
+ lowLink[func] = std::min(lowLink[func], lowLink[calleeFunc]);
+ } else if (onStack.count(calleeFunc)) {
+ lowLink[func] = std::min(lowLink[func], calleeIndex->second);
+ }
+ }
+ }
+
+ if (lowLink[func] != nodeIndex[func])
+ return true;
+
+ unsigned sccId = sccs.size();
+ sccs.emplace_back();
+ CallGraphSCC &scc = sccs.back();
+
+ for (;;) {
+ CGNode *member = stack.pop_back_val();
+ onStack.erase(member->info);
+ scc.members.push_back(member);
+ infoToSCC[member->info] = sccId;
+
+ CGFunctionInfo &info = *member->info;
+ if (scc.members.size() == 1)
+ scc.localRes = info.localRes;
+ else
+ mergeResourceInfo(scc.localRes, info.localRes);
+
+ for (size_t idx : info.ldsUseIndices)
+ scc.reachableLDS.insert(idx);
+
+ for (size_t idx : info.barrierUseIndices)
+ scc.reachableBarriers.insert(idx);
+
+ if (member == node)
+ break;
+ }
+
+ return true;
+}
+
+void AMDGPUCallGraph::buildSCCEdges() {
+ for (unsigned sccId = 0, e = sccs.size(); sccId != e; ++sccId) {
+ CallGraphSCC &scc = sccs[sccId];
+ if (scc.members.size() > 1) {
+ scc.isRecursive = true;
+ scc.localRes.hasDynSizedStack = true;
+ }
+
+ DenseSet<unsigned> seenCallees;
+ for (CGNode *member : scc.members) {
+ for (CGNode *callee : member->info->callees) {
+ auto it = infoToSCC.find(callee->info);
+ assert(it != infoToSCC.end() && "callee should be in call graph SCC");
+
+ unsigned calleeSCC = it->second;
+ if (calleeSCC == sccId) {
+ scc.isRecursive = true;
+ scc.localRes.hasDynSizedStack = true;
+ continue;
+ }
+
+ assert(calleeSCC < sccId &&
+ "sccs should be in reverse topological order");
+
+ if (seenCallees.insert(calleeSCC).second)
+ scc.callees.push_back(calleeSCC);
+ }
+ }
+ }
+}
+
+//===----------------------------------------------------------------------===//
+// LDS resolution
+//===----------------------------------------------------------------------===//
+
+// Collect every distinct SHN_AMDGPU_LDS common symbol that needs a final
+// link-time offset. Named barrier symbols (identified by the
+// __amdgpu_named_barrier prefix) are routed to a separate barriers array.
+template <class ELFT>
+static void collectLDSSymbols(Ctx &ctx,
+ SmallVectorImpl<LDSSymbolInfo> &ldsSymbols,
+ SmallVectorImpl<NamedBarrierInfo> &barriers) {
+ SmallVector<Symbol *, 16> candidates;
+ DenseSet<Symbol *> seenLDS;
+ DenseSet<Symbol *> regularCommons;
+ for (ELFFileBase *file : ctx.objectFiles) {
+ auto *obj = cast<ObjFile<ELFT>>(file);
+ ArrayRef<Symbol *> symbols = obj->getGlobalSymbols();
+ typename ELFT::SymRange elfSymbols = obj->template getGlobalELFSyms<ELFT>();
+ assert(symbols.size() == elfSymbols.size());
+ for (size_t i = 0, e = symbols.size(); i != e; ++i) {
+ Symbol *sym = symbols[i];
+ if (elfSymbols[i].st_shndx == ELF::SHN_AMDGPU_LDS) {
+ if (seenLDS.insert(sym).second)
+ candidates.push_back(sym);
+ } else if (elfSymbols[i].isCommon()) {
+ regularCommons.insert(sym);
+ }
+ }
+ }
+
+ for (Symbol *sym : candidates) {
+ if (regularCommons.contains(sym)) {
+ Err(ctx) << "symbol '" << sym->getName()
+ << "' is defined as both an AMDGPU LDS symbol and a regular "
+ "common symbol";
+ continue;
+ }
+
+ auto *cs = dyn_cast<CommonSymbol>(sym);
+ if (!cs)
+ continue;
+ if (!isPowerOf2_64(cs->alignment)) {
+ Err(ctx) << "symbol '" << sym->getName()
+ << "' has non-power-of-two AMDGPU LDS alignment "
+ << cs->alignment;
+ continue;
+ }
+
+ if (sym->getName().starts_with("__amdgpu_named_barrier")) {
+ LLVM_DEBUG(dbgs() << " collected named barrier: " << sym->getName()
+ << " size=" << cs->size << "\n");
+ barriers.push_back({sym, static_cast<uint32_t>(cs->size / 16)});
+ } else {
+ LLVM_DEBUG(dbgs() << " collected LDS symbol: " << sym->getName()
+ << " size=" << cs->size << " align=" << cs->alignment
+ << "\n");
+ ldsSymbols.push_back(
+ {sym, cs->size, Align(cs->alignment), /*assignedOffset=*/0});
+ }
+ }
+}
+
+void AMDGPUCallGraph::computeKernelReachability() {
+ for (unsigned sccId = 0, e = sccs.size(); sccId != e; ++sccId) {
+ CallGraphSCC &scc = sccs[sccId];
+ for (unsigned calleeSCC : scc.callees) {
+ assert(calleeSCC < sccId &&
+ "sccs should be in reverse topological order");
+ const CallGraphSCC &callee = sccs[calleeSCC];
+ scc.reachableLDS.insert(callee.reachableLDS.begin(),
+ callee.reachableLDS.end());
+ scc.reachableBarriers.insert(callee.reachableBarriers.begin(),
+ callee.reachableBarriers.end());
+ }
+ }
+
+ for (CGNode *kernel : kernels()) {
+ auto it = infoToSCC.find(kernel->info);
+ assert(it != infoToSCC.end() && "kernel should be in call graph SCC");
+ const CallGraphSCC &scc = sccs[it->second];
+ kernel->info->reachableLDS.clear();
+ kernel->info->reachableLDS.insert(scc.reachableLDS.begin(),
+ scc.reachableLDS.end());
+ kernel->info->reachableBarriers.clear();
+ kernel->info->reachableBarriers.insert(scc.reachableBarriers.begin(),
+ scc.reachableBarriers.end());
+ }
+}
+
+// Compute each kernel's fixed LDS usage from the final offsets of the LDS
+// objects reachable from that kernel.
+static void computeKernelLDSSizes(ArrayRef<LDSSymbolInfo> ldsSymbols,
+ AMDGPUCallGraph &cg) {
+ if (!cg.hasLDSUses()) {
+ uint32_t totalSize = 0;
+ for (const LDSSymbolInfo &lds : ldsSymbols)
+ totalSize = std::max<uint64_t>(totalSize, lds.assignedOffset + lds.size);
+ for (CGNode *kernel : cg.kernels())
+ kernel->info->ldsSize = totalSize;
+ return;
+ }
+
+ for (CGNode *kernel : cg.kernels()) {
+ uint32_t maxEnd = 0;
+ for (size_t idx : kernel->info->reachableLDS)
+ maxEnd = std::max<uint64_t>(maxEnd, ldsSymbols[idx].assignedOffset +
+ ldsSymbols[idx].size);
+ kernel->info->ldsSize = maxEnd;
+ }
+}
+
+// Assign LDS offsets, rewrite LDS symbols to those offsets, and record each
+// kernel's group segment size.
+static void resolveLDS(Ctx &ctx, SmallVectorImpl<LDSSymbolInfo> &ldsSymbols,
+ AMDGPUCallGraph &cg) {
+ LLVM_DEBUG(dbgs() << "AMDGPU LDS: assigning grouped offsets\n");
+ assignGroupedResources(ldsSymbols, cg, LDSAllocationTraits());
+ computeKernelLDSSizes(ldsSymbols, cg);
+
+ // Symbol::overwrite preserves the old symbol's visibility, so for shared-
+ // object links (AMDGPU code objects use -shared) we must explicitly force
+ // STV_HIDDEN to prevent the symbol from being preemptible, which would cause
+ // R_AMDGPU_ABS32_LO relocations to be rejected by the relocation scanner.
+ LLVM_DEBUG(dbgs() << "AMDGPU LDS: final symbol assignments:\n");
+ for (const LDSSymbolInfo &lds : ldsSymbols) {
+ LLVM_DEBUG(dbgs() << " " << lds.sym->getName() << " -> offset="
+ << lds.assignedOffset << " size=" << lds.size << "\n");
+ Defined(ctx, ctx.internalFile, lds.sym->getName(), lds.sym->binding,
+ STV_HIDDEN, STT_NOTYPE, lds.assignedOffset, lds.size, nullptr)
+ .overwrite(*lds.sym);
+ if (ctx.arg.shared)
+ lds.sym->stOther = (lds.sym->stOther & ~3) | STV_HIDDEN;
+ }
+
+ LLVM_DEBUG({
+ dbgs() << "AMDGPU LDS: per-kernel LDS sizes:\n";
+ for (CGNode *kernel : cg.kernels())
+ dbgs() << " " << kernel->sym->getName() << " -> "
+ << kernel->info->ldsSize << " bytes\n";
+ });
+}
+
+//===----------------------------------------------------------------------===//
+// Named barrier resolution
+//===----------------------------------------------------------------------===//
+
+static uint32_t computeMaxNamedBarrierEnd(ArrayRef<NamedBarrierInfo> barriers) {
+ uint32_t maxBarEnd = 0;
+ for (const NamedBarrierInfo &bar : barriers) {
+ if (bar.slotCount == 0)
+ continue;
+ maxBarEnd = std::max(maxBarEnd, bar.assignedBarId + bar.slotCount - 1);
+ }
+ return maxBarEnd;
+}
+
+static void computeKernelNamedBarrierCounts(ArrayRef<NamedBarrierInfo> barriers,
+ AMDGPUCallGraph &cg) {
+ if (!cg.hasBarrierUses()) {
+ uint32_t totalCount = computeMaxNamedBarrierEnd(barriers);
+ for (CGNode *kernel : cg.kernels())
+ kernel->info->numNamedBarrier = totalCount;
+ return;
+ }
+
+ for (CGNode *kernel : cg.kernels()) {
+ uint32_t maxBarEnd = 0;
+ for (size_t idx : kernel->info->reachableBarriers) {
+ if (barriers[idx].slotCount == 0)
+ continue;
+ uint32_t end = barriers[idx].assignedBarId + barriers[idx].slotCount - 1;
+ maxBarEnd = std::max(maxBarEnd, end);
+ }
+ kernel->info->numNamedBarrier = maxBarEnd;
+ }
+}
+
+static void checkNamedBarrierCounts(Ctx &ctx, AMDGPUCallGraph &cg) {
+ for (CGNode *kernel : cg.kernels()) {
+ if (kernel->info->numNamedBarrier <= 31)
+ continue;
+ Err(ctx) << "AMDGPU: named barrier ID overflow (max ID "
+ << kernel->info->numNamedBarrier
+ << " exceeds limit of 31) in kernel '" << kernel->sym->getName()
+ << "'";
+ }
+}
+
+// Assign named-barrier hardware IDs, rewrite named-barrier symbols to encoded
+// barrier addresses, and record each kernel's named-barrier count.
+static void resolveNamedBarriers(Ctx &ctx,
+ SmallVectorImpl<NamedBarrierInfo> &barriers,
+ AMDGPUCallGraph &cg) {
+ LLVM_DEBUG(dbgs() << "AMDGPU Named Barriers: assigning grouped IDs\n");
+ assignGroupedResources(barriers, cg, NamedBarrierAllocationTraits());
+ computeKernelNamedBarrierCounts(barriers, cg);
+ checkNamedBarrierCounts(ctx, cg);
+
+ constexpr uint32_t barScope = 0; // BARRIER_SCOPE_WORKGROUP
+ LLVM_DEBUG(dbgs() << "AMDGPU Named Barriers: final assignments:\n");
+ for (NamedBarrierInfo &bar : barriers) {
+ if (bar.assignedBarId == 0)
+ continue;
+ uint32_t addr = 0x802000u | (barScope << 9) | (bar.assignedBarId << 4);
+ LLVM_DEBUG(dbgs() << " " << bar.sym->getName()
+ << " -> barId=" << bar.assignedBarId << " addr=0x"
+ << Twine::utohexstr(addr) << "\n");
+ Defined(ctx, ctx.internalFile, bar.sym->getName(), bar.sym->binding,
+ STV_HIDDEN, STT_NOTYPE, addr, 0, nullptr)
+ .overwrite(*bar.sym);
+ if (ctx.arg.shared)
+ bar.sym->stOther = (bar.sym->stOther & ~3) | STV_HIDDEN;
+ }
+
+ for (CGNode *kernel : cg.kernels()) {
+ LLVM_DEBUG(dbgs() << " " << kernel->sym->getName() << " numNamedBarrier="
+ << kernel->info->numNamedBarrier << "\n");
+ }
+}
+
+// Check that each kernel's final LDS usage satisfies the ABI occupancy recorded
+// in .amdgpu.info.
+static bool validateKernelLDSOccupancy(Ctx &ctx, AMDGPUCallGraph &cg,
+ const AMDGPU::TargetID &targetInfo) {
+ for (CGNode *kernel : cg.kernels()) {
+ const CGResourceInfo &localRes = kernel->info->localRes;
+ assert(localRes.occupancy != 0 &&
+ "kernel occupancy should have been validated");
+ if (AMDGPU::isLDSSizeCompatibleWithOccupancy(
+ targetInfo, localRes.waveSize, localRes.isCuMode,
+ kernel->info->ldsSize, localRes.occupancy))
+ continue;
+
+ Err(ctx) << "AMDGPU: kernel '" << kernel->sym->getName() << "' uses "
+ << kernel->info->ldsSize
+ << " bytes of LDS, which does not meet ABI occupancy "
+ << localRes.occupancy;
+ return false;
+ }
+
+ return true;
+}
+
+//===----------------------------------------------------------------------===//
+// Resource usage propagation
+//===----------------------------------------------------------------------===//
+
+void AMDGPUCallGraph::propagateResourceUsage() {
+ for (unsigned sccId = 0, e = sccs.size(); sccId != e; ++sccId) {
+ CallGraphSCC &scc = sccs[sccId];
+ CGResourceInfo result = scc.localRes;
+ uint32_t maxCalleeScratch = 0;
+ for (unsigned calleeSCC : scc.callees) {
+ assert(calleeSCC < sccId &&
+ "sccs should be in reverse topological order");
+ const CallGraphSCC &callee = sccs[calleeSCC];
+ mergeResourceInfo(result, callee.propagatedRes);
+ maxCalleeScratch =
+ std::max(maxCalleeScratch, callee.propagatedRes.scratchSize);
+ }
+ // For recursive SCCs, scratchSize remains the finite fixed-frame
+ // contribution. The unbounded recursive depth is represented by
+ // hasDynSizedStack, which forces dynamic stack handling.
+ result.scratchSize = scc.localRes.scratchSize + maxCalleeScratch;
+
+ scc.propagatedRes = result;
+ for (CGNode *member : scc.members)
+ member->info->propagatedRes = result;
+ }
+
+ for (CGNode *kernel : kernels()) {
+ LLVM_DEBUG(dbgs() << " propagated " << kernel->sym->getName()
+ << ": vgpr=" << kernel->info->propagatedRes.numArchVGPR
+ << " agpr=" << kernel->info->propagatedRes.numAccVGPR
+ << " sgpr=" << kernel->info->propagatedRes.numSGPR
+ << " scratch=" << kernel->info->propagatedRes.scratchSize
+ << " dynstack="
+ << kernel->info->propagatedRes.hasDynSizedStack << "\n");
+ }
+}
+
+static ResolvedLinkingResources
+resolveObjectLinkingResources(const AMDGPU::TargetID &target,
+ const RawLinkingResources &raw) {
+ const AMDGPU::AMDGPUFeatureBitset &features =
+ AMDGPU::getFeatureBitset(target.getGPUKind());
+ bool hasAccVGPRs = features.test(AMDGPU::FEAT_GFX90A_INSTS);
+
+ ResolvedLinkingResources res;
+ res.totalVGPR = hasAccVGPRs && raw.numAccVGPR
+ ? alignTo(raw.numArchVGPR, 4) + raw.numAccVGPR
+ : std::max(raw.numArchVGPR, raw.numAccVGPR);
+ res.totalSGPR = raw.numSGPR + AMDGPU::getNumExtraSGPRs(target, raw.usesVCC,
+ raw.usesFlatScratch);
+ res.numAccVGPR = raw.numAccVGPR;
+ res.scratchSize = raw.scratchSize;
+ res.ldsSize = raw.ldsSize;
+ if (hasAccVGPRs)
+ res.accumOffset = divideCeil(std::max(raw.numArchVGPR, 1u), 4u) - 1;
+ if (features.test(AMDGPU::FEAT_GFX1250_INSTS))
+ res.encodedNamedBarrierCount = divideCeil(raw.numNamedBarrier, 4);
+ res.sgprBlocks = features.test(AMDGPU::FEAT_GFX10_INSTS)
+ ? 0
+ : AMDGPU::getNumSGPRBlocks(res.totalSGPR);
+ res.scratchEnable = raw.scratchSize > 0 || raw.hasDynSizedStack;
+ res.usesDynamicStack = raw.hasDynSizedStack;
+ return res;
+}
+
+// Build raw resource information from the linker's internal CGNode data.
+static RawLinkingResources buildRawResources(const CGNode &kernel) {
+ RawLinkingResources info = kernel.info->propagatedRes;
+ info.ldsSize = kernel.info->ldsSize;
+ info.numNamedBarrier = kernel.info->numNamedBarrier;
+ return info;
+}
+
+static void patchKernelDescriptor(MutableArrayRef<uint8_t> kd,
+ const AMDGPU::TargetID &target,
+ const ResolvedLinkingResources &resources,
+ bool patchLDS, bool patchResources) {
+ assert(kd.size() >= sizeof(amdhsa::kernel_descriptor_t) &&
+ "kernel descriptor buffer too small");
+
+ const AMDGPU::AMDGPUFeatureBitset &features =
+ AMDGPU::getFeatureBitset(target.getGPUKind());
+ uint8_t *buf = kd.data();
+
+ if (patchLDS)
+ write32le(buf + amdhsa::GROUP_SEGMENT_FIXED_SIZE_OFFSET, resources.ldsSize);
+
+ if (!patchResources)
+ return;
+
+ write32le(buf + amdhsa::PRIVATE_SEGMENT_FIXED_SIZE_OFFSET,
+ resources.scratchSize);
+
+ uint16_t kcp = read16le(buf + amdhsa::KERNEL_CODE_PROPERTIES_OFFSET);
+ AMDHSA_BITS_SET(kcp, KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK,
+ resources.usesDynamicStack ? 1 : 0);
+ write16le(buf + amdhsa::KERNEL_CODE_PROPERTIES_OFFSET, kcp);
+
+ bool enableWavefrontSize32 =
+ AMDHSA_BITS_GET(kcp, KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32);
+ unsigned waveSize = enableWavefrontSize32 ? 32 : 64;
+ unsigned vgprBlocks =
+ AMDGPU::getEncodedNumVGPRBlocks(target, resources.totalVGPR, waveSize);
+
+ uint32_t rsrc1 = read32le(buf + amdhsa::COMPUTE_PGM_RSRC1_OFFSET);
+ AMDHSA_BITS_SET(rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT,
+ vgprBlocks);
+ AMDHSA_BITS_SET(rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT,
+ resources.sgprBlocks);
+ write32le(buf + amdhsa::COMPUTE_PGM_RSRC1_OFFSET, rsrc1);
+
+ uint32_t rsrc2 = read32le(buf + amdhsa::COMPUTE_PGM_RSRC2_OFFSET);
+ AMDHSA_BITS_SET(rsrc2, COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT,
+ resources.scratchEnable ? 1 : 0);
+ write32le(buf + amdhsa::COMPUTE_PGM_RSRC2_OFFSET, rsrc2);
+
+ if (resources.accumOffset != 0 || resources.encodedNamedBarrierCount != 0) {
+ uint32_t rsrc3 = read32le(buf + amdhsa::COMPUTE_PGM_RSRC3_OFFSET);
+ if (resources.accumOffset != 0) {
+ assert(features.test(AMDGPU::FEAT_GFX90A_INSTS) &&
+ "nonzero accumulator offset requires accumulator VGPR support");
+ AMDHSA_BITS_SET(rsrc3, COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET,
+ resources.accumOffset);
+ }
+ if (resources.encodedNamedBarrierCount != 0) {
+ assert(features.test(AMDGPU::FEAT_GFX1250_INSTS) &&
+ "nonzero named-barrier count requires GFX12.5 support");
+ AMDHSA_BITS_SET(rsrc3, COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT,
+ resources.encodedNamedBarrierCount);
+ }
+ write32le(buf + amdhsa::COMPUTE_PGM_RSRC3_OFFSET, rsrc3);
+ }
+}
+
+// Patch kernel descriptors in-place with resolved LDS, named-barrier, and
+// propagated resource fields.
+static void patchKernelDescriptors(Ctx &ctx, AMDGPUCallGraph &cg,
+ const AMDGPU::TargetID &targetInfo,
+ bool hasLDS) {
+ // Track sections that have been copied to writable memory so we don't
+ // allocate redundant copies when multiple KDs share the same section.
+ DenseSet<InputSection *> copiedSections;
+
+ for (CGNode *kernel : cg.kernels()) {
+ StringRef name = kernel->sym->getName();
+ std::string kdName = (name + ".kd").str();
+ Symbol *kdSym = ctx.symtab->find(kdName);
+ if (!kdSym)
+ continue;
+ auto *kdDef = dyn_cast<Defined>(kdSym);
+ if (!kdDef || !kdDef->section)
+ continue;
+ auto *isec = dyn_cast<InputSection>(kdDef->section);
+ if (!isec)
+ continue;
+
+ uint64_t off = kdDef->value;
+ if (off + sizeof(amdhsa::kernel_descriptor_t) > isec->size)
+ continue;
+
+ // The section content may be in read-only mmap'd memory. Make a writable
+ // copy the first time we need to patch a KD in this section.
+ if (copiedSections.insert(isec).second) {
+ auto *newBuf = ctx.bAlloc.Allocate<uint8_t>(isec->size);
+ memcpy(newBuf, isec->content_, isec->size);
+ isec->content_ = newBuf;
+ }
+
+ auto *buf = const_cast<uint8_t *>(isec->content_) + off;
+ auto kdBuf =
+ MutableArrayRef<uint8_t>(buf, sizeof(amdhsa::kernel_descriptor_t));
+
+ ResolvedLinkingResources info =
+ resolveObjectLinkingResources(targetInfo, buildRawResources(*kernel));
+
+ patchKernelDescriptor(kdBuf, targetInfo, info, hasLDS, true);
+
+ LLVM_DEBUG(dbgs() << " patched " << name << ".kd: lds=" << info.ldsSize
+ << " scratch=" << info.scratchSize
+ << " dynstack=" << info.usesDynamicStack << "\n");
+ }
+}
+
+//===----------------------------------------------------------------------===//
+// HSA metadata patching
+//===----------------------------------------------------------------------===//
+
+static void patchHSAMetadataKernel(msgpack::MapDocNode &kernMap,
+ msgpack::Document &doc,
+ const ResolvedLinkingResources &resources,
+ bool patchLDS, bool patchResources) {
+ if (patchLDS)
+ kernMap[".group_segment_fixed_size"] = doc.getNode(resources.ldsSize);
+
+ if (!patchResources)
+ return;
+
+ kernMap[".sgpr_count"] = doc.getNode(resources.totalSGPR);
+ kernMap[".vgpr_count"] = doc.getNode(resources.totalVGPR);
+ kernMap[".agpr_count"] = doc.getNode(resources.numAccVGPR);
+ kernMap[".private_segment_fixed_size"] = doc.getNode(resources.scratchSize);
+ kernMap[".uses_dynamic_stack"] = doc.getNode(resources.usesDynamicStack);
+}
+
+// Rewrite AMDHSA metadata notes so the YAML/msgpack-visible kernel resource
+// fields match the patched kernel descriptors.
+template <class ELFT>
+static void patchHSAMetadata(Ctx &ctx, AMDGPUCallGraph &cg,
+ const AMDGPU::TargetID &targetInfo, bool hasLDS) {
+ DenseMap<StringRef, CGNode *> nameToKernel;
+ for (CGNode *kernel : cg.kernels())
+ nameToKernel[kernel->sym->getName()] = kernel;
+
+ for (ELFFileBase *file : ctx.objectFiles) {
+ auto *obj = cast<ObjFile<ELFT>>(file);
+ for (InputSectionBase *sec : obj->getSections()) {
+ if (!sec || sec == &InputSection::discarded)
+ continue;
+ auto *isec = dyn_cast<InputSection>(sec);
+ if (!isec || isec->type != SHT_NOTE)
+ continue;
+ if (isec->name != ".note")
+ continue;
+
+ ArrayRef<uint8_t> data = isec->contentMaybeDecompress();
+ if (data.size() < 12)
+ continue;
+
+ uint32_t nameSize = read32le(data.data());
+ uint32_t descSize = read32le(data.data() + 4);
+ uint32_t noteType = read32le(data.data() + 8);
+
+ // NT_AMDGPU_METADATA = 32
+ if (noteType != 32)
+ continue;
+
+ uint32_t nameOff = 12;
+ uint32_t namePadded = alignTo(nameSize, 4);
+ uint32_t descOff = nameOff + namePadded;
+
+ if (descOff + descSize > data.size())
+ continue;
+
+ ArrayRef<uint8_t> msgpackData = data.slice(descOff, descSize);
+ msgpack::Document doc;
+ if (!doc.readFromBlob(
+ StringRef(reinterpret_cast<const char *>(msgpackData.data()),
+ msgpackData.size()),
+ false))
+ continue;
+
+ msgpack::MapDocNode root = doc.getRoot().getMap();
+ msgpack::DocNode kernelsNode = root["amdhsa.kernels"];
+ if (kernelsNode.isEmpty())
+ continue;
+
+ bool modified = false;
+ msgpack::ArrayDocNode kernelsArray = kernelsNode.getArray();
+ for (size_t i = 0, e = kernelsArray.size(); i < e; ++i) {
+ msgpack::MapDocNode kernMap = kernelsArray[i].getMap();
+ msgpack::DocNode nameNode = kernMap[".name"];
+ if (nameNode.isEmpty())
+ continue;
+
+ StringRef kernName = nameNode.getString();
+ auto it = nameToKernel.find(kernName);
+ if (it == nameToKernel.end())
+ continue;
+ CGNode *kernel = it->second;
+
+ ResolvedLinkingResources info = resolveObjectLinkingResources(
+ targetInfo, buildRawResources(*kernel));
+
+ patchHSAMetadataKernel(kernMap, doc, info, hasLDS, true);
+ modified = true;
+ }
+
+ if (!modified)
+ continue;
+
+ std::string newMsgpack;
+ doc.writeToBlob(newMsgpack);
+
+ uint32_t newDescSize = newMsgpack.size();
+ uint32_t newDescPadded = alignTo(newDescSize, 4);
+ uint32_t newSize = nameOff + namePadded + newDescPadded;
+
+ auto *buf = ctx.bAlloc.Allocate<uint8_t>(newSize);
+ memset(buf, 0, newSize);
+ write32le(buf, nameSize);
+ write32le(buf + 4, newDescSize);
+ write32le(buf + 8, noteType);
+ memcpy(buf + nameOff, data.data() + nameOff, namePadded);
+ memcpy(buf + nameOff + namePadded, newMsgpack.data(), newMsgpack.size());
+
+ isec->content_ = buf;
+ isec->size = newSize;
+ }
+ }
+}
+
+//===----------------------------------------------------------------------===//
+// Main entry point
+//===----------------------------------------------------------------------===//
+
+static AMDGPU::TargetIDSetting decodeTargetIDSetting(uint32_t setting,
+ uint32_t any, uint32_t off,
+ uint32_t on) {
+ if (setting == any)
+ return AMDGPU::TargetIDSetting::Any;
+ if (setting == off)
+ return AMDGPU::TargetIDSetting::Off;
+ if (setting == on)
+ return AMDGPU::TargetIDSetting::On;
+ return AMDGPU::TargetIDSetting::Unsupported;
+}
+
+// Resolve AMDGPU object-linking metadata for one ELF class. This is run after
+// regular symbol resolution so all referenced symbols and kernel descriptors
+// are available for graph construction and patching.
+template <class ELFT> static void resolveAMDGPUObjectLinkingImpl(Ctx &ctx) {
+ llvm::TimeTraceScope timeScope("Resolve AMDGPU Object Linking");
+
+ LLVM_DEBUG(dbgs() << "AMDGPU: collecting LDS symbols\n");
+ SmallVector<LDSSymbolInfo, 16> ldsSymbols;
+ SmallVector<NamedBarrierInfo, 4> barriers;
+ collectLDSSymbols<ELFT>(ctx, ldsSymbols, barriers);
+ LLVM_DEBUG(dbgs() << "AMDGPU: found " << ldsSymbols.size() << " LDS symbols, "
+ << barriers.size() << " named barriers\n");
+
+ // We only support HSA for now.
+ if (ctx.arg.osabi != ELF::ELFOSABI_AMDGPU_HSA) {
+ if (!ldsSymbols.empty() || !barriers.empty())
+ Err(ctx) << "object linking for AMDGPU LDS requires the HSA OSABI";
+ return;
+ }
+
+ // Build sym->index maps once (used by section parsers).
+ DenseMap<Symbol *, size_t> ldsSymToIndex;
+ for (size_t i = 0, e = ldsSymbols.size(); i < e; ++i)
+ ldsSymToIndex[ldsSymbols[i].sym] = i;
+ DenseMap<Symbol *, size_t> barSymToIndex;
+ for (size_t i = 0, e = barriers.size(); i < e; ++i)
+ barSymToIndex[barriers[i].sym] = i;
+
+ AMDGPUCallGraph cg;
+
+ LLVM_DEBUG(dbgs() << "AMDGPU: parsing sections\n");
+ for (ELFFileBase *file : ctx.objectFiles) {
+ auto *obj = cast<ObjFile<ELFT>>(file);
+ parseInfoSection(obj, cg, ldsSymToIndex, barSymToIndex);
+ }
+
+ if (ldsSymbols.empty() && barriers.empty() && cg.empty()) {
+ LLVM_DEBUG(dbgs() << "AMDGPU: nothing to resolve\n");
+ return;
+ }
+
+ LLVM_DEBUG(dbgs() << "AMDGPU: resolving function aliases\n");
+ cg.resolveAliases();
+
+ LLVM_DEBUG(dbgs() << "AMDGPU: building indirect call edges\n");
+ cg.buildIndirectEdges();
+
+ LLVM_DEBUG(dbgs() << "AMDGPU: identifying kernels\n");
+ markKernelsWithDescriptors(ctx, cg);
+
+ LLVM_DEBUG(dbgs() << "AMDGPU: " << ldsSymbols.size() << " regular LDS, "
+ << barriers.size() << " named barriers, "
+ << cg.kernels().size() << " kernels\n");
+ if (cg.kernels().empty()) {
+ if (!ldsSymbols.empty() || !barriers.empty()) {
+ Err(ctx) << "cannot resolve AMDGPU LDS symbols without a kernel "
+ "descriptor and .amdgpu.info metadata";
+ }
+ return;
+ }
+
+ uint32_t eflags = ctx.arg.eflags;
+ StringRef cpu =
+ ELF::getAMDGPUArchNameFromELFMach(eflags & ELF::EF_AMDGPU_MACH);
+ AMDGPU::GPUKind kind = AMDGPU::parseArchAMDGCN(cpu);
+ if (kind == AMDGPU::GK_NONE)
+ kind = AMDGPU::parseArchAMDGCN("generic-hsa");
+
+ AMDGPU::TargetID targetInfo(
+ kind, Triple("amdgcn-amd-amdhsa"),
+ decodeTargetIDSetting(eflags & ELF::EF_AMDGPU_FEATURE_XNACK_V4,
+ ELF::EF_AMDGPU_FEATURE_XNACK_ANY_V4,
+ ELF::EF_AMDGPU_FEATURE_XNACK_OFF_V4,
+ ELF::EF_AMDGPU_FEATURE_XNACK_ON_V4),
+ decodeTargetIDSetting(eflags & ELF::EF_AMDGPU_FEATURE_SRAMECC_V4,
+ ELF::EF_AMDGPU_FEATURE_SRAMECC_ANY_V4,
+ ELF::EF_AMDGPU_FEATURE_SRAMECC_OFF_V4,
+ ELF::EF_AMDGPU_FEATURE_SRAMECC_ON_V4));
+
+ LLVM_DEBUG(dbgs() << "AMDGPU: validating local ABI metadata\n");
+ if (!validateLocalABI(ctx, cg, targetInfo))
+ return;
+
+ // Alias resolution and kernel discovery must be complete before this point:
+ // the shared SCC graph is consumed by call ABI validation, reachability
+ // propagation, and resource propagation.
+ LLVM_DEBUG(dbgs() << "AMDGPU: building condensed call graph\n");
+ if (!cg.buildCondensedGraph(ctx))
+ return;
+
+ LLVM_DEBUG(dbgs() << "AMDGPU: computing kernel reachability\n");
+ cg.computeKernelReachability();
+
+ LLVM_DEBUG(dbgs() << "AMDGPU: resolving LDS\n");
+ resolveLDS(ctx, ldsSymbols, cg);
+
+ LLVM_DEBUG(dbgs() << "AMDGPU: resolving named barriers\n");
+ resolveNamedBarriers(ctx, barriers, cg);
+
+ LLVM_DEBUG(dbgs() << "AMDGPU: propagating resource usage\n");
+ cg.propagateResourceUsage();
+ if (!validateKernelLDSOccupancy(ctx, cg, targetInfo))
+ return;
+
+ bool hasLDS = !ldsSymbols.empty();
+
+ LLVM_DEBUG(dbgs() << "AMDGPU: patching kernel descriptors\n");
+ patchKernelDescriptors(ctx, cg, targetInfo, hasLDS);
+
+ LLVM_DEBUG(dbgs() << "AMDGPU: patching HSA metadata\n");
+ patchHSAMetadata<ELFT>(ctx, cg, targetInfo, hasLDS);
+}
+
+void elf::resolveAMDGPUObjectLinking(Ctx &ctx) {
+ switch (ctx.arg.ekind) {
+ case ELF32LEKind:
+ resolveAMDGPUObjectLinkingImpl<ELF32LE>(ctx);
+ return;
+ case ELF64LEKind:
+ resolveAMDGPUObjectLinkingImpl<ELF64LE>(ctx);
+ return;
+ case ELF32BEKind:
+ case ELF64BEKind:
+ Err(ctx) << "AMDGPU only supports little-endian ELF";
+ return;
+ default:
+ llvm_unreachable("invalid ELF kind");
+ }
+}
diff --git a/lld/ELF/AMDGPUObjectLinking.h b/lld/ELF/AMDGPUObjectLinking.h
new file mode 100644
index 0000000000000..21cd4c446b7dd
--- /dev/null
+++ b/lld/ELF/AMDGPUObjectLinking.h
@@ -0,0 +1,25 @@
+//===- AMDGPUObjectLinking.h - AMDGPU link-time resolution ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implements link-time resolution for AMDGPU object linking. This includes
+// LDS (Local Data Share) offset assignment across translation units and
+// resource usage propagation through the cross-TU call graph.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLD_ELF_AMDGPUOBJECTLINKING_H
+#define LLD_ELF_AMDGPUOBJECTLINKING_H
+
+namespace lld::elf {
+struct Ctx;
+
+void resolveAMDGPUObjectLinking(Ctx &ctx);
+
+} // namespace lld::elf
+
+#endif // LLD_ELF_AMDGPUOBJECTLINKING_H
diff --git a/lld/ELF/Arch/AMDGPU.cpp b/lld/ELF/Arch/AMDGPU.cpp
index 52fc779855a36..547200d858edb 100644
--- a/lld/ELF/Arch/AMDGPU.cpp
+++ b/lld/ELF/Arch/AMDGPU.cpp
@@ -6,6 +6,7 @@
//
//===----------------------------------------------------------------------===//
+#include "AMDGPUObjectLinking.h"
#include "InputFiles.h"
#include "Symbols.h"
#include "Target.h"
@@ -30,6 +31,7 @@ class AMDGPU final : public TargetInfo {
public:
AMDGPU(Ctx &);
uint32_t calcEFlags() const override;
+ void finalizeSymbols() override;
void relocate(uint8_t *loc, const Relocation &rel,
uint64_t val) const override;
RelExpr getRelExpr(RelType type, const Symbol &s,
@@ -45,6 +47,8 @@ AMDGPU::AMDGPU(Ctx &ctx) : TargetInfo(ctx) {
symbolicRel = R_AMDGPU_ABS64;
}
+void AMDGPU::finalizeSymbols() { resolveAMDGPUObjectLinking(ctx); }
+
static uint32_t getEFlags(InputFile *file) {
return cast<ObjFile<ELF64LE>>(file)->getObj().getHeader().e_flags;
}
@@ -151,6 +155,7 @@ uint32_t AMDGPU::calcEFlags() const {
void AMDGPU::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
switch (rel.type) {
case R_AMDGPU_ABS32:
+ case R_AMDGPU_ABS32_LO:
case R_AMDGPU_GOTPCREL:
case R_AMDGPU_GOTPCREL32_LO:
case R_AMDGPU_REL32:
@@ -180,6 +185,7 @@ RelExpr AMDGPU::getRelExpr(RelType type, const Symbol &s,
const uint8_t *loc) const {
switch (type) {
case R_AMDGPU_ABS32:
+ case R_AMDGPU_ABS32_LO:
case R_AMDGPU_ABS64:
return R_ABS;
case R_AMDGPU_REL32:
diff --git a/lld/ELF/CMakeLists.txt b/lld/ELF/CMakeLists.txt
index e22897c2789d8..f258682b3ec71 100644
--- a/lld/ELF/CMakeLists.txt
+++ b/lld/ELF/CMakeLists.txt
@@ -20,6 +20,7 @@ endif()
add_lld_library(lldELF
AArch64ErrataFix.cpp
+ AMDGPUObjectLinking.cpp
Arch/AArch64.cpp
Arch/AMDGPU.cpp
Arch/ARM.cpp
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 0393410b35d4d..e0b00ce2dd53c 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -3448,6 +3448,16 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
// Apply symbol renames for --wrap and combine foo at v1 and foo@@v1.
redirectSymbols(ctx, wrapped);
+ // Read target properties before creating the Target instance. Some targets
+ // use these properties to select their TargetInfo implementation.
+ readSecurityNotes(ctx);
+ setTarget(ctx);
+ ctx.arg.eflags = ctx.target->calcEFlags();
+
+ ctx.target->finalizeSymbols();
+ if (errCount(ctx))
+ return;
+
// Replace common symbols with regular symbols.
replaceCommonSymbols(ctx);
@@ -3494,16 +3504,6 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
if (!ctx.arg.dependencyFile.empty())
writeDependencyFile(ctx);
- // Read .note.gnu.property sections from input object files which
- // contain a hint to tweak linker's and loader's behaviors.
- readSecurityNotes(ctx);
-
- // The Target instance handles target-specific stuff, such as applying
- // relocations or writing a PLT section. It also contains target-dependent
- // values such as a default image base address.
- setTarget(ctx);
-
- ctx.arg.eflags = ctx.target->calcEFlags();
// maxPageSize (sometimes called abi page size) is the maximum page size that
// the output can be run on. For example if the OS can use 4k or 64k page
// sizes then maxPageSize must be 64k for the output to be useable on both.
diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp
index 23649d6200719..b6cbf9abff19c 100644
--- a/lld/ELF/InputFiles.cpp
+++ b/lld/ELF/InputFiles.cpp
@@ -1220,7 +1220,7 @@ void ObjFile<ELFT>::initializeSymbols(const object::ELFFile<ELFT> &obj) {
Symbol *sym = symbols[i];
sym->isUsedInRegularObj = true;
- if (LLVM_UNLIKELY(eSym.st_shndx == SHN_COMMON)) {
+ if (LLVM_UNLIKELY(eSym.isCommon(this->emachine))) {
if (value == 0 || value >= UINT32_MAX)
Err(ctx) << this << ": common symbol '" << sym->getName()
<< "' has invalid alignment: " << value;
@@ -1327,7 +1327,7 @@ template <class ELFT> void ObjFile<ELFT>::postParse() {
if (!sym.isDefined() || secIdx == SHN_UNDEF)
continue;
if (LLVM_UNLIKELY(secIdx >= SHN_LORESERVE)) {
- if (secIdx == SHN_COMMON)
+ if (eSym.isCommon(this->emachine))
continue;
if (secIdx == SHN_XINDEX)
secIdx = check(getExtendedSymbolTableIndex<ELFT>(eSym, i, shndxTable));
@@ -1411,7 +1411,7 @@ static bool isNonCommonDef(Ctx &ctx, ELFKind ekind, MemoryBufferRef mb,
Expected<StringRef> name = sym.getName(stringtable);
if (name && name.get() == symName)
return sym.isDefined() && sym.getBinding() == STB_GLOBAL &&
- !sym.isCommon();
+ !sym.isCommon(obj->emachine);
}
return false;
}
diff --git a/lld/ELF/Target.h b/lld/ELF/Target.h
index 174e389db155d..4c33d34248da9 100644
--- a/lld/ELF/Target.h
+++ b/lld/ELF/Target.h
@@ -32,6 +32,9 @@ class TargetInfo {
public:
TargetInfo(Ctx &ctx) : ctx(ctx) {}
virtual uint32_t calcEFlags() const { return 0; }
+ // Finalize target-specific symbols after LTO and symbol redirection, but
+ // before common symbols are converted to regular definitions.
+ virtual void finalizeSymbols() {}
// Create target-specific synthetic sections, defined in Arch/ files.
virtual void initTargetSpecificSections() {}
virtual RelExpr getRelExpr(RelType type, const Symbol &s,
diff --git a/lld/test/ELF/amdgpu-big-endian.test b/lld/test/ELF/amdgpu-big-endian.test
new file mode 100644
index 0000000000000..bf49d8a5f1a67
--- /dev/null
+++ b/lld/test/ELF/amdgpu-big-endian.test
@@ -0,0 +1,19 @@
+# RUN: yaml2obj %s -o %t.o
+# RUN: not ld.lld %t.o -o /dev/null 2>&1 | FileCheck %s
+
+# CHECK: error: AMDGPU only supports little-endian ELF
+
+--- !ELF
+FileHeader:
+ Class: ELFCLASS64
+ Data: ELFDATA2MSB
+ Type: ET_REL
+ Machine: EM_AMDGPU
+Sections:
+ - Name: .text
+ Type: SHT_PROGBITS
+ Flags: [ SHF_ALLOC, SHF_EXECINSTR ]
+ Size: 4
+Symbols:
+ - Name: _start
+ Section: .text
diff --git a/lld/test/ELF/amdgpu-lds-link-time-align.s b/lld/test/ELF/amdgpu-lds-link-time-align.s
new file mode 100644
index 0000000000000..c616f433812cb
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-align.s
@@ -0,0 +1,110 @@
+# REQUIRES: amdgpu
+
+## Test that lld sorts LDS symbols by alignment (descending), then size
+## (descending), and correctly inserts padding between symbols.
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa -mcpu=gfx900 %t/a.s -o %t/a.o
+# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa -mcpu=gfx900 %t/b.s -o %t/b.o
+# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa -mcpu=gfx900 %t/c.s -o %t/c.o
+# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa -mcpu=gfx900 %t/kernel.s -o %t/kernel.o
+# RUN: ld.lld %t/a.o %t/b.o %t/c.o %t/kernel.o -o %t/out
+# RUN: llvm-readobj --syms %t/out | FileCheck %s
+
+## Three LDS symbols with varied alignments and sizes:
+## lds_small: align=4, size=12 (from a.o)
+## lds_big: align=16, size=64 (from b.o)
+## lds_medium: align=8, size=32 (from c.o)
+##
+## After sorting by alignment (desc), then size (desc):
+## lds_big (align=16, size=64) -> offset 0
+## lds_medium (align=8, size=32) -> offset 64
+## lds_small (align=4, size=12) -> offset 96
+
+## Symbols appear in input file order (a.o first). Check each with its offset.
+# CHECK: Name: lds_small
+# CHECK-NEXT: Value: 0x60
+# CHECK-NEXT: Size: 12
+# CHECK: Section: Absolute
+
+# CHECK: Name: lds_big
+# CHECK-NEXT: Value: 0x0
+# CHECK-NEXT: Size: 64
+# CHECK: Section: Absolute
+
+# CHECK: Name: lds_medium
+# CHECK-NEXT: Value: 0x40
+# CHECK-NEXT: Size: 32
+# CHECK: Section: Absolute
+
+#--- a.s
+ .text
+ .globl use_lds_small
+ .p2align 8
+ .type use_lds_small, at function
+use_lds_small:
+ s_mov_b32 s0, lds_small at abs32@lo
+ s_endpgm
+.Lfunc_end_a:
+ .size use_lds_small, .Lfunc_end_a-use_lds_small
+
+ .globl lds_small
+ .amdgpu_lds lds_small, 12, 4
+
+#--- b.s
+ .text
+ .globl use_lds_big
+ .p2align 8
+ .type use_lds_big, at function
+use_lds_big:
+ s_mov_b32 s0, lds_big at abs32@lo
+ s_endpgm
+.Lfunc_end_b:
+ .size use_lds_big, .Lfunc_end_b-use_lds_big
+
+ .globl lds_big
+ .amdgpu_lds lds_big, 64, 16
+
+#--- c.s
+ .text
+ .globl use_lds_medium
+ .p2align 8
+ .type use_lds_medium, at function
+use_lds_medium:
+ s_mov_b32 s0, lds_medium at abs32@lo
+ s_endpgm
+.Lfunc_end_c:
+ .size use_lds_medium, .Lfunc_end_c-use_lds_medium
+
+ .globl lds_medium
+ .amdgpu_lds lds_medium, 32, 8
+
+#--- kernel.s
+ .text
+ .globl test_kernel
+ .p2align 8
+ .type test_kernel, at function
+test_kernel:
+ s_endpgm
+.Lfunc_end_kernel:
+ .size test_kernel, .Lfunc_end_kernel-test_kernel
+
+ .amdgpu_info test_kernel
+ .amdgpu_flags 3
+ .amdgpu_num_vgpr 32
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 33
+ .amdgpu_private_segment_size 0
+ .amdgpu_use lds_small
+ .amdgpu_use lds_big
+ .amdgpu_use lds_medium
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .rodata,"a", at progbits
+ .p2align 6
+ .globl test_kernel.kd
+ .type test_kernel.kd, at object
+ .size test_kernel.kd, 64
+test_kernel.kd:
+ .zero 64
diff --git a/lld/test/ELF/amdgpu-lds-link-time-dynlds-alias.s b/lld/test/ELF/amdgpu-lds-link-time-dynlds-alias.s
new file mode 100644
index 0000000000000..6c74784f99fa3
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-dynlds-alias.s
@@ -0,0 +1,187 @@
+# REQUIRES: amdgpu
+
+## Test that dynamic LDS variables are placed at the current frontier using
+## their own alignment. The kernel directly uses dyn_a (align=4) and calls a
+## device function in another TU that uses dyn_b (align=16).
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/a.s -o %t/a.o
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/b.s -o %t/b.o
+# RUN: ld.lld %t/a.o %t/b.o -o %t/out
+
+## Dynamic LDS symbols do not advance the kernel frontier.
+# RUN: llvm-readelf -s %t/out | FileCheck %s --check-prefix=SYM
+
+## group_segment_fixed_size includes the alignment padding for the dynamic base.
+# RUN: llvm-readobj --notes %t/out | FileCheck %s --check-prefix=META
+
+## LDS layout:
+## small_lds: align=4, size=12 -> offset 0
+## dyn_a (align=4): offset 12
+## dyn_b (align=16): offset 16
+##
+## group_segment_fixed_size = 16
+
+# SYM-DAG: 0000000000000000 {{.*}} small_lds
+# SYM-DAG: 000000000000000c {{.*}} dyn_a
+# SYM-DAG: 0000000000000010 {{.*}} dyn_b
+
+# META: .group_segment_fixed_size: 16
+# META: .name: my_kernel
+
+#--- a.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl my_kernel ; -- Begin function my_kernel
+ .p2align 8
+ .type my_kernel, at function
+my_kernel:
+ s_endpgm
+.Lfunc_end0:
+ .size my_kernel, .Lfunc_end0-my_kernel
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl my_kernel.kd
+ .type my_kernel.kd, at object
+ .size my_kernel.kd, 64
+ .protected my_kernel
+my_kernel.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad my_kernel at rel64-my_kernel.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11469063
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .Lmy_kernel.num_vgpr, 32
+ .set .Lmy_kernel.num_agpr, 0
+ .set .Lmy_kernel.numbered_sgpr, 33
+ .set .Lmy_kernel.num_named_barrier, 0
+ .set .Lmy_kernel.private_seg_size, 0
+ .set .Lmy_kernel.uses_vcc, 1
+ .set .Lmy_kernel.uses_flat_scratch, 1
+ .set .Lmy_kernel.has_dyn_sized_stack, 0
+ .set .Lmy_kernel.has_recursion, 0
+ .set .Lmy_kernel.has_indirect_call, 0
+ .amdgpu_info my_kernel
+ .amdgpu_flags 3
+ .amdgpu_num_vgpr 32
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 33
+ .amdgpu_private_segment_size 0
+ .amdgpu_use small_lds
+ .amdgpu_use dyn_a
+ .amdgpu_call device_func
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 0
+ .set amdgpu.max_num_named_barrier, 0
+ .globl small_lds
+ .amdgpu_lds small_lds, 12, 16
+ .globl dyn_a
+ .amdgpu_lds dyn_a, 0, 4
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: my_kernel
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: my_kernel.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- b.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl device_func ; -- Begin function device_func
+ .p2align 6
+ .type device_func, at function
+device_func:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end0:
+ .size device_func, .Lfunc_end0-device_func
+ .set .Ldevice_func.num_vgpr, 2
+ .set .Ldevice_func.num_agpr, 0
+ .set .Ldevice_func.numbered_sgpr, 32
+ .set .Ldevice_func.num_named_barrier, 0
+ .set .Ldevice_func.private_seg_size, 0
+ .set .Ldevice_func.uses_vcc, 0
+ .set .Ldevice_func.uses_flat_scratch, 0
+ .set .Ldevice_func.has_dyn_sized_stack, 0
+ .set .Ldevice_func.has_recursion, 0
+ .set .Ldevice_func.has_indirect_call, 0
+ .amdgpu_info device_func
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 2
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 0
+ .amdgpu_use dyn_b
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 2
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 32
+ .set amdgpu.max_num_named_barrier, 0
+ .globl dyn_b
+ .amdgpu_lds dyn_b, 0, 16
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels: []
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-lds-link-time-dynlds-align.s b/lld/test/ELF/amdgpu-lds-link-time-dynlds-align.s
new file mode 100644
index 0000000000000..b1c3bd5db8a32
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-dynlds-align.s
@@ -0,0 +1,182 @@
+# REQUIRES: amdgpu
+
+## Test that size-0 LDS symbols (dynamic LDS) with high alignment are
+## correctly padded past the static LDS region.
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/a.s -o %t/a.o
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/b.s -o %t/b.o
+# RUN: ld.lld %t/a.o %t/b.o -o %t/out
+
+## Verify symbol offsets.
+# RUN: llvm-readelf -s %t/out | FileCheck %s --check-prefix=SYM
+
+## Verify group_segment_fixed_size in HSA metadata.
+# RUN: llvm-readobj --notes %t/out | FileCheck %s --check-prefix=META
+
+## LDS layout:
+## small_lds: align=4, size=12 -> offset 0
+## dyn_lds: align=128, size=0 -> offset 128
+## (12 bytes of static, then padded to 128 for alignment)
+##
+## group_segment_fixed_size = 128 (the dynlds offset with alignment padding).
+
+# SYM-DAG: 0000000000000000 {{.*}} small_lds
+# SYM-DAG: 0000000000000080 {{.*}} dyn_lds
+
+# META: .group_segment_fixed_size: 128
+# META: .name: my_kernel
+
+#--- a.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl my_kernel ; -- Begin function my_kernel
+ .p2align 8
+ .type my_kernel, at function
+my_kernel:
+ s_endpgm
+.Lfunc_end0:
+ .size my_kernel, .Lfunc_end0-my_kernel
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl my_kernel.kd
+ .type my_kernel.kd, at object
+ .size my_kernel.kd, 64
+ .protected my_kernel
+my_kernel.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad my_kernel at rel64-my_kernel.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11469063
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .Lmy_kernel.num_vgpr, 32
+ .set .Lmy_kernel.num_agpr, 0
+ .set .Lmy_kernel.numbered_sgpr, 33
+ .set .Lmy_kernel.num_named_barrier, 0
+ .set .Lmy_kernel.private_seg_size, 0
+ .set .Lmy_kernel.uses_vcc, 1
+ .set .Lmy_kernel.uses_flat_scratch, 1
+ .set .Lmy_kernel.has_dyn_sized_stack, 0
+ .set .Lmy_kernel.has_recursion, 0
+ .set .Lmy_kernel.has_indirect_call, 0
+ .amdgpu_info my_kernel
+ .amdgpu_flags 3
+ .amdgpu_num_vgpr 32
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 33
+ .amdgpu_private_segment_size 0
+ .amdgpu_use small_lds
+ .amdgpu_use dyn_lds
+ .amdgpu_call helper
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 0
+ .set amdgpu.max_num_named_barrier, 0
+ .globl small_lds
+ .amdgpu_lds small_lds, 12, 16
+ .globl dyn_lds
+ .amdgpu_lds dyn_lds, 0, 128
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: my_kernel
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: my_kernel.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- b.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl helper ; -- Begin function helper
+ .p2align 6
+ .type helper, at function
+helper:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end0:
+ .size helper, .Lfunc_end0-helper
+ .set .Lhelper.num_vgpr, 0
+ .set .Lhelper.num_agpr, 0
+ .set .Lhelper.numbered_sgpr, 32
+ .set .Lhelper.num_named_barrier, 0
+ .set .Lhelper.private_seg_size, 0
+ .set .Lhelper.uses_vcc, 0
+ .set .Lhelper.uses_flat_scratch, 0
+ .set .Lhelper.has_dyn_sized_stack, 0
+ .set .Lhelper.has_recursion, 0
+ .set .Lhelper.has_indirect_call, 0
+ .amdgpu_info helper
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 0
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 32
+ .set amdgpu.max_num_named_barrier, 0
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels: []
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-lds-link-time-dynlds.s b/lld/test/ELF/amdgpu-lds-link-time-dynlds.s
new file mode 100644
index 0000000000000..f9fdd3c4d4663
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-dynlds.s
@@ -0,0 +1,184 @@
+# REQUIRES: amdgpu
+
+## Test that size-0 LDS symbols (dynamic LDS) are placed after all static LDS
+## symbols and that the kernel descriptor's group_segment_fixed_size reflects
+## the dynamic base offset.
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/a.s -o %t/a.o
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/b.s -o %t/b.o
+# RUN: ld.lld %t/a.o %t/b.o -o %t/out
+
+## Verify symbol offsets.
+# RUN: llvm-readelf -s %t/out | FileCheck %s --check-prefix=SYM
+
+## Verify group_segment_fixed_size in HSA metadata is patched.
+# RUN: llvm-readobj --notes %t/out | FileCheck %s --check-prefix=META
+
+## LDS layout:
+## static_lds: align=16, size=256 -> offset 0
+## dyn_lds: align=4, size=0 -> offset 256
+## Total fixed LDS = 256 = 0x100 (dynamic base)
+##
+## group_segment_fixed_size = 256 because the size-0 dyn_lds symbol
+## sits at offset 256 but contributes 0 bytes.
+
+# SYM-DAG: 0000000000000000 {{.*}} static_lds
+# SYM-DAG: 0000000000000100 {{.*}} dyn_lds
+
+# META: .group_segment_fixed_size: 256
+# META: .name: my_kernel
+
+#--- a.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl my_kernel ; -- Begin function my_kernel
+ .p2align 8
+ .type my_kernel, at function
+my_kernel:
+ s_endpgm
+.Lfunc_end0:
+ .size my_kernel, .Lfunc_end0-my_kernel
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl my_kernel.kd
+ .type my_kernel.kd, at object
+ .size my_kernel.kd, 64
+ .protected my_kernel
+my_kernel.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad my_kernel at rel64-my_kernel.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11469063
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .Lmy_kernel.num_vgpr, 32
+ .set .Lmy_kernel.num_agpr, 0
+ .set .Lmy_kernel.numbered_sgpr, 33
+ .set .Lmy_kernel.num_named_barrier, 0
+ .set .Lmy_kernel.private_seg_size, 0
+ .set .Lmy_kernel.uses_vcc, 1
+ .set .Lmy_kernel.uses_flat_scratch, 1
+ .set .Lmy_kernel.has_dyn_sized_stack, 0
+ .set .Lmy_kernel.has_recursion, 0
+ .set .Lmy_kernel.has_indirect_call, 0
+ .amdgpu_info my_kernel
+ .amdgpu_flags 3
+ .amdgpu_num_vgpr 32
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 33
+ .amdgpu_private_segment_size 0
+ .amdgpu_use static_lds
+ .amdgpu_use dyn_lds
+ .amdgpu_call helper
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 0
+ .set amdgpu.max_num_named_barrier, 0
+ .globl static_lds
+ .amdgpu_lds static_lds, 256, 16
+ .globl dyn_lds
+ .amdgpu_lds dyn_lds, 0, 4
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: my_kernel
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: my_kernel.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- b.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl helper ; -- Begin function helper
+ .p2align 6
+ .type helper, at function
+helper:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end0:
+ .size helper, .Lfunc_end0-helper
+ .set .Lhelper.num_vgpr, 0
+ .set .Lhelper.num_agpr, 0
+ .set .Lhelper.numbered_sgpr, 32
+ .set .Lhelper.num_named_barrier, 0
+ .set .Lhelper.private_seg_size, 0
+ .set .Lhelper.uses_vcc, 0
+ .set .Lhelper.uses_flat_scratch, 0
+ .set .Lhelper.has_dyn_sized_stack, 0
+ .set .Lhelper.has_recursion, 0
+ .set .Lhelper.has_indirect_call, 0
+ .amdgpu_info helper
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 0
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 32
+ .set amdgpu.max_num_named_barrier, 0
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels: []
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-lds-link-time-grouped.s b/lld/test/ELF/amdgpu-lds-link-time-grouped.s
new file mode 100644
index 0000000000000..4e15f72e9df5c
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-grouped.s
@@ -0,0 +1,425 @@
+# REQUIRES: amdgpu
+
+## Test grouped LDS allocation with independent kernel groups and tier ordering.
+##
+## TU1 has:
+## - func: uses @lds_callee (callee-scope, internal, 64 bytes)
+## - K1: uses @lds_global (global-scope, external, 32 bytes),
+## uses @lds_k1 (kernel-scope, internal, 16 bytes), calls func
+## - K2: uses @lds_global, calls func (no kernel-scope LDS)
+##
+## TU2 has:
+## - K3: uses @lds_k3 (kernel-scope, internal, 128 bytes)
+##
+## Grouping:
+## K1, K2 share lds_callee and lds_global -> Group 1
+## K3 is independent -> Group 2
+##
+## Group 1 tier classification:
+## Shared tier: __amdgpu_lds.func (callee-scope, user=func not a kernel)
+## lds_global (global-scope, users=K1,K2)
+## Kernel tier: __amdgpu_lds.K1 (kernel-scope, user=K1 a kernel)
+##
+## Group 1 layout (per-kernel frontier order; both shared vars have use_count=2,
+## tie-broken by size -- larger first):
+## Shared: __amdgpu_lds.func (align 16, size 64) @ 0x00
+## lds_global (align 16, size 32) @ 0x40
+## Kernel: __amdgpu_lds.K1 (align 16, size 16) @ 0x60
+##
+## Group 2 layout (starts from offset 0):
+## __amdgpu_lds.K3 (align 16, size 128) @ 0x00
+##
+## Per-kernel sizes:
+## K1: reaches func's struct(0..64), lds_global(64..96), K1's struct(96..112) -> 0x70
+## K2: reaches func's struct(0..64), lds_global(64..96) -> 0x60
+## K3: reaches K3's struct(0..128) -> 0x80
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/tu1.s -o %t/tu1.o
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/tu2.s -o %t/tu2.o
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/tu3.s -o %t/tu3.o
+# RUN: ld.lld %t/tu1.o %t/tu2.o %t/tu3.o -o %t/out
+# RUN: llvm-readelf -s %t/out | FileCheck %s --check-prefix=SYM
+# RUN: llvm-readobj --notes %t/out | FileCheck %s --check-prefix=META
+
+## LDS symbol offsets (Group 1 shared tier first, kernel tier last)
+# SYM-DAG: 0000000000000000 {{.*}} __amdgpu_lds.func
+# SYM-DAG: 0000000000000040 {{.*}} lds_global
+# SYM-DAG: 0000000000000060 {{.*}} __amdgpu_lds.K1
+
+## Group 2 allocates from offset 0 independently
+# SYM-DAG: 0000000000000000 {{.*}} __amdgpu_lds.K3
+
+## Per-kernel LDS sizes patched into the kernel descriptor and HSA metadata
+# META-DAG: .group_segment_fixed_size: 112
+# META-DAG: .group_segment_fixed_size: 96
+# META-DAG: .group_segment_fixed_size: 128
+
+#--- tu1.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl func ; -- Begin function func
+ .p2align 6
+ .type func, at function
+func:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end0:
+ .size func, .Lfunc_end0-func
+ .set .Lfunc.num_vgpr, 41
+ .set .Lfunc.num_agpr, 0
+ .set .Lfunc.numbered_sgpr, 34
+ .set .Lfunc.num_named_barrier, 0
+ .set .Lfunc.private_seg_size, 16
+ .set .Lfunc.uses_vcc, 1
+ .set .Lfunc.uses_flat_scratch, 0
+ .set .Lfunc.has_dyn_sized_stack, 0
+ .set .Lfunc.has_recursion, 0
+ .set .Lfunc.has_indirect_call, 0
+ .text
+ .globl K1 ; -- Begin function K1
+ .p2align 8
+ .type K1, at function
+K1:
+ s_endpgm
+.Lfunc_end1:
+ .size K1, .Lfunc_end1-K1
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl K1.kd
+ .type K1.kd, at object
+ .size K1.kd, 64
+ .protected K1
+K1.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad K1 at rel64-K1.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11469063
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .LK1.num_vgpr, 32
+ .set .LK1.num_agpr, 0
+ .set .LK1.numbered_sgpr, 33
+ .set .LK1.num_named_barrier, 0
+ .set .LK1.private_seg_size, 0
+ .set .LK1.uses_vcc, 1
+ .set .LK1.uses_flat_scratch, 1
+ .set .LK1.has_dyn_sized_stack, 0
+ .set .LK1.has_recursion, 1
+ .set .LK1.has_indirect_call, 0
+ .text
+ .globl K2 ; -- Begin function K2
+ .p2align 8
+ .type K2, at function
+K2:
+ s_endpgm
+.Lfunc_end2:
+ .size K2, .Lfunc_end2-K2
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl K2.kd
+ .type K2.kd, at object
+ .size K2.kd, 64
+ .protected K2
+K2.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad K2 at rel64-K2.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11469063
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .LK2.num_vgpr, 32
+ .set .LK2.num_agpr, 0
+ .set .LK2.numbered_sgpr, 33
+ .set .LK2.num_named_barrier, 0
+ .set .LK2.private_seg_size, 0
+ .set .LK2.uses_vcc, 1
+ .set .LK2.uses_flat_scratch, 1
+ .set .LK2.has_dyn_sized_stack, 0
+ .set .LK2.has_recursion, 1
+ .set .LK2.has_indirect_call, 0
+ .amdgpu_info func
+ .amdgpu_flags 1
+ .amdgpu_num_vgpr 41
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 34
+ .amdgpu_private_segment_size 16
+ .amdgpu_use __amdgpu_lds.func
+ .amdgpu_call extern_func
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info K1
+ .amdgpu_flags 3
+ .amdgpu_num_vgpr 32
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 33
+ .amdgpu_private_segment_size 0
+ .amdgpu_use __amdgpu_lds.K1
+ .amdgpu_use lds_global
+ .amdgpu_call func
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info K2
+ .amdgpu_flags 3
+ .amdgpu_num_vgpr 32
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 33
+ .amdgpu_private_segment_size 0
+ .amdgpu_use lds_global
+ .amdgpu_call func
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 41
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 34
+ .set amdgpu.max_num_named_barrier, 0
+ .globl lds_global
+ .amdgpu_lds lds_global, 32, 16
+ .globl __amdgpu_lds.func
+ .amdgpu_lds __amdgpu_lds.func, 64, 16
+ .globl __amdgpu_lds.K1
+ .amdgpu_lds __amdgpu_lds.K1, 16, 16
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: K1
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: K1.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: K2
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: K2.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- tu2.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl K3 ; -- Begin function K3
+ .p2align 8
+ .type K3, at function
+K3:
+ s_endpgm
+.Lfunc_end0:
+ .size K3, .Lfunc_end0-K3
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl K3.kd
+ .type K3.kd, at object
+ .size K3.kd, 64
+ .protected K3
+K3.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad K3 at rel64-K3.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11468864
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .LK3.num_vgpr, 2
+ .set .LK3.num_agpr, 0
+ .set .LK3.numbered_sgpr, 10
+ .set .LK3.num_named_barrier, 0
+ .set .LK3.private_seg_size, 0
+ .set .LK3.uses_vcc, 0
+ .set .LK3.uses_flat_scratch, 0
+ .set .LK3.has_dyn_sized_stack, 0
+ .set .LK3.has_recursion, 0
+ .set .LK3.has_indirect_call, 0
+ .amdgpu_info K3
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 2
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 10
+ .amdgpu_private_segment_size 0
+ .amdgpu_use __amdgpu_lds.K3
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 0
+ .set amdgpu.max_num_named_barrier, 0
+ .globl __amdgpu_lds.K3
+ .amdgpu_lds __amdgpu_lds.K3, 128, 16
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: K3
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: K3.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- tu3.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl extern_func ; -- Begin function extern_func
+ .p2align 6
+ .type extern_func, at function
+extern_func:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end0:
+ .size extern_func, .Lfunc_end0-extern_func
+ .set .Lextern_func.num_vgpr, 0
+ .set .Lextern_func.num_agpr, 0
+ .set .Lextern_func.numbered_sgpr, 32
+ .set .Lextern_func.num_named_barrier, 0
+ .set .Lextern_func.private_seg_size, 0
+ .set .Lextern_func.uses_vcc, 0
+ .set .Lextern_func.uses_flat_scratch, 0
+ .set .Lextern_func.has_dyn_sized_stack, 0
+ .set .Lextern_func.has_recursion, 0
+ .set .Lextern_func.has_indirect_call, 0
+ .amdgpu_info extern_func
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 0
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 32
+ .set amdgpu.max_num_named_barrier, 0
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels: []
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-lds-link-time-indirect-basic.s b/lld/test/ELF/amdgpu-lds-link-time-indirect-basic.s
new file mode 100644
index 0000000000000..30d2b5e0e57fe
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-indirect-basic.s
@@ -0,0 +1,186 @@
+# REQUIRES: amdgpu
+
+## Test basic indirect call LDS resolution: kernel calls a function that makes
+## an indirect call to an LDS-using function. The linker should discover the
+## LDS variable through the indirect call edge (prototype matching) and assign
+## it an offset. The kernel's LDS size should include the indirectly-reachable
+## LDS.
+##
+## Call graph: my_kernel -> indirect_caller --(indirect)--> target_func -> lds_var
+##
+## target_func: void(i32) -> prototype encoding "vi"
+## indirect_caller: makes indirect call with encoding "vi"
+## target_func: address-taken (passed as ptr to indirect_caller)
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/tu1.s -o %t/tu1.o
+# RUN: ld.lld %t/tu1.o -o %t/out
+# RUN: llvm-readelf -s %t/out | FileCheck %s --check-prefix=SYM
+# RUN: llvm-readobj --notes %t/out | FileCheck %s --check-prefix=META
+
+## LDS variable should be resolved with an offset.
+# SYM: 0000000000000000 {{.*}} lds_var
+
+## Kernel descriptor should have non-zero LDS size (128 bytes for [32 x i32]).
+# META: .group_segment_fixed_size: 128
+
+#--- tu1.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl target_func ; -- Begin function target_func
+ .p2align 6
+ .type target_func, at function
+target_func:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end0:
+ .size target_func, .Lfunc_end0-target_func
+ .set .Ltarget_func.num_vgpr, 2
+ .set .Ltarget_func.num_agpr, 0
+ .set .Ltarget_func.numbered_sgpr, 32
+ .set .Ltarget_func.num_named_barrier, 0
+ .set .Ltarget_func.private_seg_size, 0
+ .set .Ltarget_func.uses_vcc, 0
+ .set .Ltarget_func.uses_flat_scratch, 0
+ .set .Ltarget_func.has_dyn_sized_stack, 0
+ .set .Ltarget_func.has_recursion, 0
+ .set .Ltarget_func.has_indirect_call, 0
+ .text
+ .globl indirect_caller ; -- Begin function indirect_caller
+ .p2align 6
+ .type indirect_caller, at function
+indirect_caller:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end1:
+ .size indirect_caller, .Lfunc_end1-indirect_caller
+ .set .Lindirect_caller.num_vgpr, 41
+ .set .Lindirect_caller.num_agpr, 0
+ .set .Lindirect_caller.numbered_sgpr, 66
+ .set .Lindirect_caller.num_named_barrier, 0
+ .set .Lindirect_caller.private_seg_size, 16
+ .set .Lindirect_caller.uses_vcc, 1
+ .set .Lindirect_caller.uses_flat_scratch, 1
+ .set .Lindirect_caller.has_dyn_sized_stack, 1
+ .set .Lindirect_caller.has_recursion, 1
+ .set .Lindirect_caller.has_indirect_call, 1
+ .text
+ .globl my_kernel ; -- Begin function my_kernel
+ .p2align 8
+ .type my_kernel, at function
+my_kernel:
+ s_endpgm
+.Lfunc_end2:
+ .size my_kernel, .Lfunc_end2-my_kernel
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl my_kernel.kd
+ .type my_kernel.kd, at object
+ .size my_kernel.kd, 64
+ .protected my_kernel
+my_kernel.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad my_kernel at rel64-my_kernel.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11469063
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .Lmy_kernel.num_vgpr, 32
+ .set .Lmy_kernel.num_agpr, 0
+ .set .Lmy_kernel.numbered_sgpr, 33
+ .set .Lmy_kernel.num_named_barrier, 0
+ .set .Lmy_kernel.private_seg_size, 0
+ .set .Lmy_kernel.uses_vcc, 1
+ .set .Lmy_kernel.uses_flat_scratch, 1
+ .set .Lmy_kernel.has_dyn_sized_stack, 0
+ .set .Lmy_kernel.has_recursion, 1
+ .set .Lmy_kernel.has_indirect_call, 0
+ .amdgpu_info target_func
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 2
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 0
+ .amdgpu_use lds_var
+ .amdgpu_typeid "vi"
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info indirect_caller
+ .amdgpu_flags 7
+ .amdgpu_num_vgpr 41
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 66
+ .amdgpu_private_segment_size 16
+ .amdgpu_indirect_call "vi"
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info my_kernel
+ .amdgpu_flags 3
+ .amdgpu_num_vgpr 32
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 33
+ .amdgpu_private_segment_size 0
+ .amdgpu_call indirect_caller
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 41
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 66
+ .set amdgpu.max_num_named_barrier, 0
+ .globl lds_var
+ .amdgpu_lds lds_var, 128, 16
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: my_kernel
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: my_kernel.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-lds-link-time-indirect-cross-tu.s b/lld/test/ELF/amdgpu-lds-link-time-indirect-cross-tu.s
new file mode 100644
index 0000000000000..8acb6f07a76fd
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-indirect-cross-tu.s
@@ -0,0 +1,203 @@
+# REQUIRES: amdgpu
+
+## Test cross-TU address-taken: the function is defined in TU1 but its address
+## is taken in TU2. The linker should match the indirect call in TU2 with the
+## function from TU1 via prototype matching.
+##
+## TU1: defines target_func (uses lds_var)
+## TU2: defines kern which passes target_func as a pointer to caller
+## defines caller which makes an indirect call
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/tu1.s -o %t/tu1.o
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/tu2.s -o %t/tu2.o
+# RUN: ld.lld %t/tu1.o %t/tu2.o -o %t/out
+# RUN: llvm-readobj --notes %t/out | FileCheck %s --check-prefix=META
+
+## Kernel LDS size should include lds_var (128 bytes).
+# META: .group_segment_fixed_size: 128
+
+#--- tu1.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl target_func ; -- Begin function target_func
+ .p2align 6
+ .type target_func, at function
+target_func:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end0:
+ .size target_func, .Lfunc_end0-target_func
+ .set .Ltarget_func.num_vgpr, 2
+ .set .Ltarget_func.num_agpr, 0
+ .set .Ltarget_func.numbered_sgpr, 32
+ .set .Ltarget_func.num_named_barrier, 0
+ .set .Ltarget_func.private_seg_size, 0
+ .set .Ltarget_func.uses_vcc, 0
+ .set .Ltarget_func.uses_flat_scratch, 0
+ .set .Ltarget_func.has_dyn_sized_stack, 0
+ .set .Ltarget_func.has_recursion, 0
+ .set .Ltarget_func.has_indirect_call, 0
+ .amdgpu_info target_func
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 2
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 0
+ .amdgpu_use lds_var
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 2
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 32
+ .set amdgpu.max_num_named_barrier, 0
+ .globl lds_var
+ .amdgpu_lds lds_var, 128, 16
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels: []
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- tu2.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl caller ; -- Begin function caller
+ .p2align 6
+ .type caller, at function
+caller:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end0:
+ .size caller, .Lfunc_end0-caller
+ .set .Lcaller.num_vgpr, 41
+ .set .Lcaller.num_agpr, 0
+ .set .Lcaller.numbered_sgpr, 66
+ .set .Lcaller.num_named_barrier, 0
+ .set .Lcaller.private_seg_size, 16
+ .set .Lcaller.uses_vcc, 1
+ .set .Lcaller.uses_flat_scratch, 1
+ .set .Lcaller.has_dyn_sized_stack, 1
+ .set .Lcaller.has_recursion, 1
+ .set .Lcaller.has_indirect_call, 1
+ .text
+ .globl kern ; -- Begin function kern
+ .p2align 8
+ .type kern, at function
+kern:
+ s_endpgm
+.Lfunc_end1:
+ .size kern, .Lfunc_end1-kern
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl kern.kd
+ .type kern.kd, at object
+ .size kern.kd, 64
+ .protected kern
+kern.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad kern at rel64-kern.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11469063
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .Lkern.num_vgpr, 32
+ .set .Lkern.num_agpr, 0
+ .set .Lkern.numbered_sgpr, 33
+ .set .Lkern.num_named_barrier, 0
+ .set .Lkern.private_seg_size, 0
+ .set .Lkern.uses_vcc, 1
+ .set .Lkern.uses_flat_scratch, 1
+ .set .Lkern.has_dyn_sized_stack, 0
+ .set .Lkern.has_recursion, 1
+ .set .Lkern.has_indirect_call, 0
+ .amdgpu_info caller
+ .amdgpu_flags 7
+ .amdgpu_num_vgpr 41
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 66
+ .amdgpu_private_segment_size 16
+ .amdgpu_indirect_call "vi"
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info kern
+ .amdgpu_flags 3
+ .amdgpu_num_vgpr 32
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 33
+ .amdgpu_private_segment_size 0
+ .amdgpu_call caller
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info target_func
+ .amdgpu_typeid "vi"
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 41
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 66
+ .set amdgpu.max_num_named_barrier, 0
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: kern
+ .private_segment_fixed_size: 0
+ .sgpr_count: 39
+ .sgpr_spill_count: 0
+ .symbol: kern.kd
+ .uses_dynamic_stack: true
+ .vgpr_count: 32
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-lds-link-time-indirect-mixed.s b/lld/test/ELF/amdgpu-lds-link-time-indirect-mixed.s
new file mode 100644
index 0000000000000..cb010601677ff
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-indirect-mixed.s
@@ -0,0 +1,180 @@
+# REQUIRES: amdgpu
+
+## Test a function called both directly and indirectly. The LDS should be
+## reachable through both paths. The kernel's LDS size should include the
+## function's LDS regardless of which path discovers it.
+##
+## Call graph:
+## kern -> target_func (direct call)
+## kern -> caller --(indirect)--> target_func
+## target_func -> lds_var
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/tu1.s -o %t/tu1.o
+# RUN: ld.lld %t/tu1.o -o %t/out
+# RUN: llvm-readobj --notes %t/out | FileCheck %s --check-prefix=META
+
+## Kernel LDS size should include lds_var (128 bytes).
+# META: .group_segment_fixed_size: 128
+
+#--- tu1.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl target_func ; -- Begin function target_func
+ .p2align 6
+ .type target_func, at function
+target_func:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end0:
+ .size target_func, .Lfunc_end0-target_func
+ .set .Ltarget_func.num_vgpr, 2
+ .set .Ltarget_func.num_agpr, 0
+ .set .Ltarget_func.numbered_sgpr, 32
+ .set .Ltarget_func.num_named_barrier, 0
+ .set .Ltarget_func.private_seg_size, 0
+ .set .Ltarget_func.uses_vcc, 0
+ .set .Ltarget_func.uses_flat_scratch, 0
+ .set .Ltarget_func.has_dyn_sized_stack, 0
+ .set .Ltarget_func.has_recursion, 0
+ .set .Ltarget_func.has_indirect_call, 0
+ .text
+ .globl caller ; -- Begin function caller
+ .p2align 6
+ .type caller, at function
+caller:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end1:
+ .size caller, .Lfunc_end1-caller
+ .set .Lcaller.num_vgpr, 41
+ .set .Lcaller.num_agpr, 0
+ .set .Lcaller.numbered_sgpr, 66
+ .set .Lcaller.num_named_barrier, 0
+ .set .Lcaller.private_seg_size, 16
+ .set .Lcaller.uses_vcc, 1
+ .set .Lcaller.uses_flat_scratch, 1
+ .set .Lcaller.has_dyn_sized_stack, 1
+ .set .Lcaller.has_recursion, 1
+ .set .Lcaller.has_indirect_call, 1
+ .text
+ .globl kern ; -- Begin function kern
+ .p2align 8
+ .type kern, at function
+kern:
+ s_endpgm
+.Lfunc_end2:
+ .size kern, .Lfunc_end2-kern
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl kern.kd
+ .type kern.kd, at object
+ .size kern.kd, 64
+ .protected kern
+kern.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad kern at rel64-kern.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11469063
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .Lkern.num_vgpr, 32
+ .set .Lkern.num_agpr, 0
+ .set .Lkern.numbered_sgpr, 33
+ .set .Lkern.num_named_barrier, 0
+ .set .Lkern.private_seg_size, 0
+ .set .Lkern.uses_vcc, 1
+ .set .Lkern.uses_flat_scratch, 1
+ .set .Lkern.has_dyn_sized_stack, 0
+ .set .Lkern.has_recursion, 1
+ .set .Lkern.has_indirect_call, 0
+ .amdgpu_info target_func
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 2
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 0
+ .amdgpu_use lds_var
+ .amdgpu_typeid "vi"
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info caller
+ .amdgpu_flags 7
+ .amdgpu_num_vgpr 41
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 66
+ .amdgpu_private_segment_size 16
+ .amdgpu_indirect_call "vi"
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info kern
+ .amdgpu_flags 3
+ .amdgpu_num_vgpr 32
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 33
+ .amdgpu_private_segment_size 0
+ .amdgpu_call target_func
+ .amdgpu_call caller
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 41
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 66
+ .set amdgpu.max_num_named_barrier, 0
+ .globl lds_var
+ .amdgpu_lds lds_var, 128, 16
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: kern
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: kern.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-lds-link-time-indirect-multi.s b/lld/test/ELF/amdgpu-lds-link-time-indirect-multi.s
new file mode 100644
index 0000000000000..7e68327083b0c
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-indirect-multi.s
@@ -0,0 +1,220 @@
+# REQUIRES: amdgpu
+
+## Test indirect call with multiple potential callees. Two address-taken
+## functions have the same prototype. Both should be considered potential
+## callees of the indirect call site, and LDS from both should be reachable.
+##
+## Call graph:
+## my_kernel -> caller --(indirect)--> {target_a, target_b}
+## target_a -> lds_a (64 bytes)
+## target_b -> lds_b (32 bytes)
+##
+## Both targets: void(i32) -> encoding "vi"
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/tu1.s -o %t/tu1.o
+# RUN: ld.lld %t/tu1.o -o %t/out
+# RUN: llvm-readelf -s %t/out | FileCheck %s --check-prefix=SYM
+# RUN: llvm-readobj --notes %t/out | FileCheck %s --check-prefix=META
+
+## Both LDS variables should be resolved.
+# SYM-DAG: {{[0-9a-f]+}} {{.*}} lds_a
+# SYM-DAG: {{[0-9a-f]+}} {{.*}} lds_b
+
+## Kernel's LDS size should include both lds_a (256 bytes) and lds_b (128 bytes).
+## lds_a: align=4, size=256 -> offset 0, end 256
+## lds_b: align=4, size=128 -> offset 256, end 384
+# META: .group_segment_fixed_size: 384
+
+#--- tu1.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl target_a ; -- Begin function target_a
+ .p2align 6
+ .type target_a, at function
+target_a:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end0:
+ .size target_a, .Lfunc_end0-target_a
+ .set .Ltarget_a.num_vgpr, 2
+ .set .Ltarget_a.num_agpr, 0
+ .set .Ltarget_a.numbered_sgpr, 32
+ .set .Ltarget_a.num_named_barrier, 0
+ .set .Ltarget_a.private_seg_size, 0
+ .set .Ltarget_a.uses_vcc, 0
+ .set .Ltarget_a.uses_flat_scratch, 0
+ .set .Ltarget_a.has_dyn_sized_stack, 0
+ .set .Ltarget_a.has_recursion, 0
+ .set .Ltarget_a.has_indirect_call, 0
+ .text
+ .globl target_b ; -- Begin function target_b
+ .p2align 6
+ .type target_b, at function
+target_b:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end1:
+ .size target_b, .Lfunc_end1-target_b
+ .set .Ltarget_b.num_vgpr, 2
+ .set .Ltarget_b.num_agpr, 0
+ .set .Ltarget_b.numbered_sgpr, 32
+ .set .Ltarget_b.num_named_barrier, 0
+ .set .Ltarget_b.private_seg_size, 0
+ .set .Ltarget_b.uses_vcc, 0
+ .set .Ltarget_b.uses_flat_scratch, 0
+ .set .Ltarget_b.has_dyn_sized_stack, 0
+ .set .Ltarget_b.has_recursion, 0
+ .set .Ltarget_b.has_indirect_call, 0
+ .text
+ .globl caller ; -- Begin function caller
+ .p2align 6
+ .type caller, at function
+caller:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end2:
+ .size caller, .Lfunc_end2-caller
+ .set .Lcaller.num_vgpr, 41
+ .set .Lcaller.num_agpr, 0
+ .set .Lcaller.numbered_sgpr, 66
+ .set .Lcaller.num_named_barrier, 0
+ .set .Lcaller.private_seg_size, 16
+ .set .Lcaller.uses_vcc, 1
+ .set .Lcaller.uses_flat_scratch, 1
+ .set .Lcaller.has_dyn_sized_stack, 1
+ .set .Lcaller.has_recursion, 1
+ .set .Lcaller.has_indirect_call, 1
+ .text
+ .globl my_kernel ; -- Begin function my_kernel
+ .p2align 8
+ .type my_kernel, at function
+my_kernel:
+ s_endpgm
+.Lfunc_end3:
+ .size my_kernel, .Lfunc_end3-my_kernel
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl my_kernel.kd
+ .type my_kernel.kd, at object
+ .size my_kernel.kd, 64
+ .protected my_kernel
+my_kernel.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad my_kernel at rel64-my_kernel.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11469514
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .Lmy_kernel.num_vgpr, 42
+ .set .Lmy_kernel.num_agpr, 0
+ .set .Lmy_kernel.numbered_sgpr, 88
+ .set .Lmy_kernel.num_named_barrier, 0
+ .set .Lmy_kernel.private_seg_size, 0
+ .set .Lmy_kernel.uses_vcc, 1
+ .set .Lmy_kernel.uses_flat_scratch, 1
+ .set .Lmy_kernel.has_dyn_sized_stack, 0
+ .set .Lmy_kernel.has_recursion, 1
+ .set .Lmy_kernel.has_indirect_call, 0
+ .amdgpu_info target_a
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 2
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 0
+ .amdgpu_use lds_a
+ .amdgpu_typeid "vi"
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info target_b
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 2
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 0
+ .amdgpu_use lds_b
+ .amdgpu_typeid "vi"
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info caller
+ .amdgpu_flags 7
+ .amdgpu_num_vgpr 41
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 66
+ .amdgpu_private_segment_size 16
+ .amdgpu_indirect_call "vi"
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info my_kernel
+ .amdgpu_flags 3
+ .amdgpu_num_vgpr 42
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 88
+ .amdgpu_private_segment_size 0
+ .amdgpu_call caller
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 41
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 66
+ .set amdgpu.max_num_named_barrier, 0
+ .globl lds_a
+ .amdgpu_lds lds_a, 256, 16
+ .globl lds_b
+ .amdgpu_lds lds_b, 128, 16
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: my_kernel
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: my_kernel.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-lds-link-time-indirect-nested.s b/lld/test/ELF/amdgpu-lds-link-time-indirect-nested.s
new file mode 100644
index 0000000000000..d555b4e6d9290
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-indirect-nested.s
@@ -0,0 +1,252 @@
+# REQUIRES: amdgpu
+
+## Test nested indirect calls (C++ virtual method pattern): an address-taken
+## function itself makes an indirect call to another function. Both functions
+## use LDS. The linker must discover all LDS through the indirect call chain
+## and assign sequential non-overlapping offsets.
+##
+## Call graph:
+## kern -> caller --(indirect)--> outer_target --(indirect)--> inner_target
+## outer_target -> lds_outer (128 bytes)
+## inner_target -> lds_inner (64 bytes)
+##
+## Both indirect calls: void(i32) -> encoding "vi"
+## Both targets are address-taken with encoding "vi"
+##
+## All LDS must be assigned unique offsets (no slot reuse).
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/tu1.s -o %t/tu1.o
+# RUN: ld.lld %t/tu1.o -o %t/out
+# RUN: llvm-readelf -s %t/out | FileCheck %s --check-prefix=SYM
+# RUN: llvm-readobj --notes %t/out | FileCheck %s --check-prefix=META
+
+## Both LDS variables resolved with non-overlapping offsets.
+# SYM-DAG: {{[0-9a-f]+}} {{.*}} lds_outer
+# SYM-DAG: {{[0-9a-f]+}} {{.*}} lds_inner
+
+## Kernel LDS size includes both: lds_outer (128) + lds_inner (64) = 192.
+# META: .group_segment_fixed_size: 192
+
+#--- tu1.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl inner_target ; -- Begin function inner_target
+ .p2align 6
+ .type inner_target, at function
+inner_target:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end0:
+ .size inner_target, .Lfunc_end0-inner_target
+ .set .Linner_target.num_vgpr, 2
+ .set .Linner_target.num_agpr, 0
+ .set .Linner_target.numbered_sgpr, 32
+ .set .Linner_target.num_named_barrier, 0
+ .set .Linner_target.private_seg_size, 0
+ .set .Linner_target.uses_vcc, 0
+ .set .Linner_target.uses_flat_scratch, 0
+ .set .Linner_target.has_dyn_sized_stack, 0
+ .set .Linner_target.has_recursion, 0
+ .set .Linner_target.has_indirect_call, 0
+ .text
+ .globl dispatch ; -- Begin function dispatch
+ .p2align 6
+ .type dispatch, at function
+dispatch:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end1:
+ .size dispatch, .Lfunc_end1-dispatch
+ .set .Ldispatch.num_vgpr, 41
+ .set .Ldispatch.num_agpr, 0
+ .set .Ldispatch.numbered_sgpr, 66
+ .set .Ldispatch.num_named_barrier, 0
+ .set .Ldispatch.private_seg_size, 16
+ .set .Ldispatch.uses_vcc, 1
+ .set .Ldispatch.uses_flat_scratch, 1
+ .set .Ldispatch.has_dyn_sized_stack, 1
+ .set .Ldispatch.has_recursion, 1
+ .set .Ldispatch.has_indirect_call, 1
+ .text
+ .globl outer_target ; -- Begin function outer_target
+ .p2align 6
+ .type outer_target, at function
+outer_target:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end2:
+ .size outer_target, .Lfunc_end2-outer_target
+ .set .Louter_target.num_vgpr, 42
+ .set .Louter_target.num_agpr, 0
+ .set .Louter_target.numbered_sgpr, 66
+ .set .Louter_target.num_named_barrier, 0
+ .set .Louter_target.private_seg_size, 16
+ .set .Louter_target.uses_vcc, 1
+ .set .Louter_target.uses_flat_scratch, 0
+ .set .Louter_target.has_dyn_sized_stack, 0
+ .set .Louter_target.has_recursion, 1
+ .set .Louter_target.has_indirect_call, 0
+ .text
+ .globl caller ; -- Begin function caller
+ .p2align 6
+ .type caller, at function
+caller:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end3:
+ .size caller, .Lfunc_end3-caller
+ .set .Lcaller.num_vgpr, 41
+ .set .Lcaller.num_agpr, 0
+ .set .Lcaller.numbered_sgpr, 66
+ .set .Lcaller.num_named_barrier, 0
+ .set .Lcaller.private_seg_size, 16
+ .set .Lcaller.uses_vcc, 1
+ .set .Lcaller.uses_flat_scratch, 1
+ .set .Lcaller.has_dyn_sized_stack, 1
+ .set .Lcaller.has_recursion, 1
+ .set .Lcaller.has_indirect_call, 1
+ .text
+ .globl kern ; -- Begin function kern
+ .p2align 8
+ .type kern, at function
+kern:
+ s_endpgm
+.Lfunc_end4:
+ .size kern, .Lfunc_end4-kern
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl kern.kd
+ .type kern.kd, at object
+ .size kern.kd, 64
+ .protected kern
+kern.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad kern at rel64-kern.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11469063
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .Lkern.num_vgpr, 32
+ .set .Lkern.num_agpr, 0
+ .set .Lkern.numbered_sgpr, 33
+ .set .Lkern.num_named_barrier, 0
+ .set .Lkern.private_seg_size, 0
+ .set .Lkern.uses_vcc, 1
+ .set .Lkern.uses_flat_scratch, 1
+ .set .Lkern.has_dyn_sized_stack, 0
+ .set .Lkern.has_recursion, 1
+ .set .Lkern.has_indirect_call, 0
+ .amdgpu_info inner_target
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 2
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 0
+ .amdgpu_use lds_inner
+ .amdgpu_typeid "vi"
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info dispatch
+ .amdgpu_flags 7
+ .amdgpu_num_vgpr 41
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 66
+ .amdgpu_private_segment_size 16
+ .amdgpu_indirect_call "vi"
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info outer_target
+ .amdgpu_flags 1
+ .amdgpu_num_vgpr 42
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 66
+ .amdgpu_private_segment_size 16
+ .amdgpu_use lds_outer
+ .amdgpu_call dispatch
+ .amdgpu_typeid "vi"
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info caller
+ .amdgpu_flags 7
+ .amdgpu_num_vgpr 41
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 66
+ .amdgpu_private_segment_size 16
+ .amdgpu_indirect_call "vi"
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info kern
+ .amdgpu_flags 3
+ .amdgpu_num_vgpr 32
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 33
+ .amdgpu_private_segment_size 0
+ .amdgpu_call caller
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 42
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 66
+ .set amdgpu.max_num_named_barrier, 0
+ .globl lds_outer
+ .amdgpu_lds lds_outer, 128, 16
+ .globl lds_inner
+ .amdgpu_lds lds_inner, 64, 16
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: kern
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: kern.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-lds-link-time-indirect-prototype.s b/lld/test/ELF/amdgpu-lds-link-time-indirect-prototype.s
new file mode 100644
index 0000000000000..23087a387dd97
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-indirect-prototype.s
@@ -0,0 +1,245 @@
+# REQUIRES: amdgpu
+
+## Test LDS discovery through an indirect call, with prototype-based filtering.
+## Only address-taken functions matching the indirect call's prototype should be
+## considered as potential callees.
+##
+## target_match: void(i32) -> encoding "vi" -- matches the indirect call
+## target_nomatch: i32(i32, i32) -> encoding "iii" -- does NOT match
+##
+## Only lds_match should be reachable from the kernel through the indirect edge.
+## lds_nomatch should NOT be reachable (its function has a different prototype).
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/tu1.s -o %t/tu1.o
+# RUN: ld.lld %t/tu1.o -o %t/out
+# RUN: llvm-readelf -s %t/out | FileCheck %s --check-prefix=SYM
+# RUN: llvm-readobj --notes %t/out | FileCheck %s --check-prefix=META
+
+## The matching target's LDS is discovered through the indirect call edge.
+# SYM: 0000000000000000 {{.*}} lds_match
+
+## Kernel's LDS size should only include lds_match (128 bytes), not lds_nomatch.
+# META: .group_segment_fixed_size: 128
+
+#--- tu1.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl target_match ; -- Begin function target_match
+ .p2align 6
+ .type target_match, at function
+target_match:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end0:
+ .size target_match, .Lfunc_end0-target_match
+ .set .Ltarget_match.num_vgpr, 2
+ .set .Ltarget_match.num_agpr, 0
+ .set .Ltarget_match.numbered_sgpr, 32
+ .set .Ltarget_match.num_named_barrier, 0
+ .set .Ltarget_match.private_seg_size, 0
+ .set .Ltarget_match.uses_vcc, 0
+ .set .Ltarget_match.uses_flat_scratch, 0
+ .set .Ltarget_match.has_dyn_sized_stack, 0
+ .set .Ltarget_match.has_recursion, 0
+ .set .Ltarget_match.has_indirect_call, 0
+ .text
+ .globl target_nomatch ; -- Begin function target_nomatch
+ .p2align 6
+ .type target_nomatch, at function
+target_nomatch:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end1:
+ .size target_nomatch, .Lfunc_end1-target_nomatch
+ .set .Ltarget_nomatch.num_vgpr, 3
+ .set .Ltarget_nomatch.num_agpr, 0
+ .set .Ltarget_nomatch.numbered_sgpr, 32
+ .set .Ltarget_nomatch.num_named_barrier, 0
+ .set .Ltarget_nomatch.private_seg_size, 0
+ .set .Ltarget_nomatch.uses_vcc, 0
+ .set .Ltarget_nomatch.uses_flat_scratch, 0
+ .set .Ltarget_nomatch.has_dyn_sized_stack, 0
+ .set .Ltarget_nomatch.has_recursion, 0
+ .set .Ltarget_nomatch.has_indirect_call, 0
+ .text
+ .globl caller ; -- Begin function caller
+ .p2align 6
+ .type caller, at function
+caller:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end2:
+ .size caller, .Lfunc_end2-caller
+ .set .Lcaller.num_vgpr, 41
+ .set .Lcaller.num_agpr, 0
+ .set .Lcaller.numbered_sgpr, 66
+ .set .Lcaller.num_named_barrier, 0
+ .set .Lcaller.private_seg_size, 16
+ .set .Lcaller.uses_vcc, 1
+ .set .Lcaller.uses_flat_scratch, 1
+ .set .Lcaller.has_dyn_sized_stack, 1
+ .set .Lcaller.has_recursion, 1
+ .set .Lcaller.has_indirect_call, 1
+ .text
+ .globl addr_taker ; -- Begin function addr_taker
+ .p2align 6
+ .type addr_taker, at function
+addr_taker:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end3:
+ .size addr_taker, .Lfunc_end3-addr_taker
+ .set .Laddr_taker.num_vgpr, 2
+ .set .Laddr_taker.num_agpr, 0
+ .set .Laddr_taker.numbered_sgpr, 33
+ .set .Laddr_taker.num_named_barrier, 0
+ .set .Laddr_taker.private_seg_size, 16
+ .set .Laddr_taker.uses_vcc, 0
+ .set .Laddr_taker.uses_flat_scratch, 0
+ .set .Laddr_taker.has_dyn_sized_stack, 0
+ .set .Laddr_taker.has_recursion, 0
+ .set .Laddr_taker.has_indirect_call, 0
+ .text
+ .globl my_kernel ; -- Begin function my_kernel
+ .p2align 8
+ .type my_kernel, at function
+my_kernel:
+ s_endpgm
+.Lfunc_end4:
+ .size my_kernel, .Lfunc_end4-my_kernel
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl my_kernel.kd
+ .type my_kernel.kd, at object
+ .size my_kernel.kd, 64
+ .protected my_kernel
+my_kernel.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad my_kernel at rel64-my_kernel.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11469514
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .Lmy_kernel.num_vgpr, 42
+ .set .Lmy_kernel.num_agpr, 0
+ .set .Lmy_kernel.numbered_sgpr, 85
+ .set .Lmy_kernel.num_named_barrier, 0
+ .set .Lmy_kernel.private_seg_size, 0
+ .set .Lmy_kernel.uses_vcc, 1
+ .set .Lmy_kernel.uses_flat_scratch, 1
+ .set .Lmy_kernel.has_dyn_sized_stack, 0
+ .set .Lmy_kernel.has_recursion, 1
+ .set .Lmy_kernel.has_indirect_call, 0
+ .amdgpu_info target_match
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 2
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 0
+ .amdgpu_use lds_match
+ .amdgpu_typeid "vi"
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info target_nomatch
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 3
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 0
+ .amdgpu_use lds_nomatch
+ .amdgpu_typeid "iii"
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info caller
+ .amdgpu_flags 7
+ .amdgpu_num_vgpr 41
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 66
+ .amdgpu_private_segment_size 16
+ .amdgpu_indirect_call "vi"
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info addr_taker
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 2
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 33
+ .amdgpu_private_segment_size 16
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info my_kernel
+ .amdgpu_flags 3
+ .amdgpu_num_vgpr 42
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 85
+ .amdgpu_private_segment_size 0
+ .amdgpu_call caller
+ .amdgpu_call addr_taker
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 41
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 66
+ .set amdgpu.max_num_named_barrier, 0
+ .globl lds_match
+ .amdgpu_lds lds_match, 128, 16
+ .globl lds_nomatch
+ .amdgpu_lds lds_nomatch, 256, 16
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: my_kernel
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: my_kernel.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-lds-link-time-indirect-resource.s b/lld/test/ELF/amdgpu-lds-link-time-indirect-resource.s
new file mode 100644
index 0000000000000..77b1efe8c4d26
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-indirect-resource.s
@@ -0,0 +1,176 @@
+# REQUIRES: amdgpu
+
+## Test that resource usage (VGPR/SGPR/scratch) propagates correctly through
+## indirect call edges. The kernel should pick up the resource requirements
+## of the indirectly-called function.
+##
+## Call graph: kern -> caller --(indirect)--> heavy_func
+## heavy_func uses significant VGPRs (via inline asm) and scratch.
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/tu1.s -o %t/tu1.o
+# RUN: ld.lld %t/tu1.o -o %t/out
+# RUN: llvm-readobj --notes %t/out | FileCheck %s --check-prefix=META
+
+## The kernel should have non-trivial resource usage propagated from heavy_func.
+## Check that private_segment_fixed_size is non-zero (scratch from heavy_func).
+# META: .private_segment_fixed_size:
+# META-NOT: .private_segment_fixed_size: 0
+
+#--- tu1.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl heavy_func ; -- Begin function heavy_func
+ .p2align 6
+ .type heavy_func, at function
+heavy_func:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end0:
+ .size heavy_func, .Lfunc_end0-heavy_func
+ .set .Lheavy_func.num_vgpr, 2
+ .set .Lheavy_func.num_agpr, 0
+ .set .Lheavy_func.numbered_sgpr, 33
+ .set .Lheavy_func.num_named_barrier, 0
+ .set .Lheavy_func.private_seg_size, 260
+ .set .Lheavy_func.uses_vcc, 0
+ .set .Lheavy_func.uses_flat_scratch, 0
+ .set .Lheavy_func.has_dyn_sized_stack, 0
+ .set .Lheavy_func.has_recursion, 0
+ .set .Lheavy_func.has_indirect_call, 0
+ .text
+ .globl caller ; -- Begin function caller
+ .p2align 6
+ .type caller, at function
+caller:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end1:
+ .size caller, .Lfunc_end1-caller
+ .set .Lcaller.num_vgpr, 41
+ .set .Lcaller.num_agpr, 0
+ .set .Lcaller.numbered_sgpr, 66
+ .set .Lcaller.num_named_barrier, 0
+ .set .Lcaller.private_seg_size, 16
+ .set .Lcaller.uses_vcc, 1
+ .set .Lcaller.uses_flat_scratch, 1
+ .set .Lcaller.has_dyn_sized_stack, 1
+ .set .Lcaller.has_recursion, 1
+ .set .Lcaller.has_indirect_call, 1
+ .text
+ .globl kern ; -- Begin function kern
+ .p2align 8
+ .type kern, at function
+kern:
+ s_endpgm
+.Lfunc_end2:
+ .size kern, .Lfunc_end2-kern
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl kern.kd
+ .type kern.kd, at object
+ .size kern.kd, 64
+ .protected kern
+kern.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad kern at rel64-kern.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11469063
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .Lkern.num_vgpr, 32
+ .set .Lkern.num_agpr, 0
+ .set .Lkern.numbered_sgpr, 33
+ .set .Lkern.num_named_barrier, 0
+ .set .Lkern.private_seg_size, 0
+ .set .Lkern.uses_vcc, 1
+ .set .Lkern.uses_flat_scratch, 1
+ .set .Lkern.has_dyn_sized_stack, 0
+ .set .Lkern.has_recursion, 1
+ .set .Lkern.has_indirect_call, 0
+ .amdgpu_info heavy_func
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 2
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 33
+ .amdgpu_private_segment_size 260
+ .amdgpu_typeid "vi"
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info caller
+ .amdgpu_flags 7
+ .amdgpu_num_vgpr 41
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 66
+ .amdgpu_private_segment_size 16
+ .amdgpu_indirect_call "vi"
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info kern
+ .amdgpu_flags 3
+ .amdgpu_num_vgpr 32
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 33
+ .amdgpu_private_segment_size 0
+ .amdgpu_call caller
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 41
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 66
+ .set amdgpu.max_num_named_barrier, 0
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: kern
+ .private_segment_fixed_size: 0
+ .sgpr_count: 39
+ .sgpr_spill_count: 0
+ .symbol: kern.kd
+ .uses_dynamic_stack: true
+ .vgpr_count: 32
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-lds-link-time-instructions.s b/lld/test/ELF/amdgpu-lds-link-time-instructions.s
new file mode 100644
index 0000000000000..5cef21753cb0c
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-instructions.s
@@ -0,0 +1,198 @@
+# REQUIRES: amdgpu
+
+## End-to-end verification for link-time LDS symbol resolution. This test is
+## the focused home for the full object-to-linked-binary flow:
+## 1) Pre-link: .amdgpu_lds directives produce SHN_AMDGPU_LDS symbols.
+## 2) Pre-link: object files contain R_AMDGPU_ABS32_LO relocations and
+## placeholder 0 literals in the load-address instructions.
+## 3) Post-link: LDS symbols become absolute symbols with resolved offsets.
+## 4) Post-link: the linker patches instructions with those offsets, and the
+## ds_read/ds_write instructions remain intact.
+##
+## LDS layout after linking (alignment desc, size desc):
+## lds_arr (align=16, size=64) -> offset 0x00
+## lds_buf (align=4, size=32) -> offset 0x40
+##
+## TU a.s: kernel that writes lds_arr[idx] = 42 and reads lds_buf[idx].
+## TU b.s: kernel that writes lds_buf[idx] = 99.
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa -mcpu=gfx900 %t/a.s -o %t/a.o
+# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa -mcpu=gfx900 %t/b.s -o %t/b.o
+
+## Pre-link: LDS symbols and relocations in the object files.
+# RUN: llvm-readobj --syms %t/a.o | FileCheck %s --check-prefix=OBJ-A
+# RUN: llvm-readobj --syms %t/b.o | FileCheck %s --check-prefix=OBJ-B
+# RUN: llvm-objdump -d -r %t/a.o | FileCheck %s --check-prefix=PRE-A
+# RUN: llvm-objdump -d -r %t/b.o | FileCheck %s --check-prefix=PRE-B
+
+## Link.
+# RUN: ld.lld %t/a.o %t/b.o -o %t/out
+
+## Post-link: absolute LDS symbols and resolved instructions in the linked binary.
+# RUN: llvm-readobj --syms %t/out | FileCheck %s --check-prefix=LINKED
+# RUN: llvm-objdump -d %t/out | FileCheck %s --check-prefix=POST
+
+## === Pre-link LDS symbols ===
+
+# OBJ-A: Symbol {
+# OBJ-A: Name: lds_arr
+# OBJ-A-NEXT: Value: 0x10
+# OBJ-A-NEXT: Size: 64
+# OBJ-A-NEXT: Binding: Global
+# OBJ-A-NEXT: Type: Object
+# OBJ-A: Section: Processor Specific (0xFF00)
+# OBJ-A-NEXT: }
+
+# OBJ-B: Symbol {
+# OBJ-B: Name: lds_buf
+# OBJ-B-NEXT: Value: 0x4
+# OBJ-B-NEXT: Size: 32
+# OBJ-B-NEXT: Binding: Global
+# OBJ-B-NEXT: Type: Object
+# OBJ-B: Section: Processor Specific (0xFF00)
+# OBJ-B-NEXT: }
+
+## === Pre-link TU a: two LDS references (lds_arr and lds_buf) ===
+
+# PRE-A-LABEL: <use_lds_a>:
+## Load base of lds_arr — placeholder 0 with relocation.
+# PRE-A: s_mov_b32 s0, lit(0x0)
+# PRE-A-NEXT: {{.*}} R_AMDGPU_ABS32_LO lds_arr
+## Move base to VGPR and store to LDS.
+# PRE-A: v_mov_b32_e32 v1, s0
+# PRE-A: ds_write_b8 v1, v2
+
+## Load base of lds_buf — placeholder 0 with relocation.
+# PRE-A: s_mov_b32 s1, lit(0x0)
+# PRE-A-NEXT: {{.*}} R_AMDGPU_ABS32_LO lds_buf
+## Move base to VGPR and read from LDS.
+# PRE-A: v_mov_b32_e32 v0, s1
+# PRE-A: ds_read_u8 v0, v0
+
+## === Pre-link TU b: one LDS reference (lds_buf) ===
+
+# PRE-B-LABEL: <use_lds_b>:
+## Load base of lds_buf — placeholder 0 with relocation.
+# PRE-B: s_mov_b32 s0, lit(0x0)
+# PRE-B-NEXT: {{.*}} R_AMDGPU_ABS32_LO lds_buf
+## Move base to VGPR and store to LDS.
+# PRE-B: v_mov_b32_e32 v1, s0
+# PRE-B: ds_write_b8 v1, v2
+
+## === Post-link: relocated instructions resolved ===
+
+# LINKED: Symbol {
+# LINKED: Name: lds_arr
+# LINKED-NEXT: Value: 0x0
+# LINKED-NEXT: Size: 64
+# LINKED-NEXT: Binding: Global
+# LINKED-NEXT: Type: None
+# LINKED: Section: Absolute
+# LINKED-NEXT: }
+
+# LINKED: Symbol {
+# LINKED: Name: lds_buf
+# LINKED-NEXT: Value: 0x40
+# LINKED-NEXT: Size: 32
+# LINKED-NEXT: Binding: Global
+# LINKED-NEXT: Type: None
+# LINKED: Section: Absolute
+# LINKED-NEXT: }
+
+## lds_arr at offset 0x00 — s_mov_b32 resolved to 0.
+# POST-LABEL: <use_lds_a>:
+# POST: s_mov_b32 s0, lit(0x0)
+# POST: v_mov_b32_e32 v1, s0
+# POST: ds_write_b8 v1, v2
+## lds_buf at offset 0x40 — s_mov_b32 patched to 0x40 (literal encoding).
+# POST: s_mov_b32 s1, lit(0x40)
+# POST: v_mov_b32_e32 v0, s1
+# POST: ds_read_u8 v0, v0
+
+## In use_lds_b, lds_buf also resolved to 0x40 (same cross-TU symbol).
+# POST-LABEL: <use_lds_b>:
+# POST: s_mov_b32 s0, lit(0x40)
+# POST: v_mov_b32_e32 v1, s0
+# POST: ds_write_b8 v1, v2
+
+#--- a.s
+ .text
+ .globl use_lds_a
+ .p2align 8
+ .type use_lds_a, at function
+use_lds_a:
+ ; Load base address of lds_arr (relocation target).
+ s_mov_b32 s0, lds_arr at abs32@lo
+ ; Move base to VGPR.
+ v_mov_b32_e32 v1, s0
+ ; Store i8 42 to lds_arr[base].
+ v_mov_b32_e32 v2, 42
+ ds_write_b8 v1, v2
+
+ ; Load base address of lds_buf (relocation target).
+ s_mov_b32 s1, lds_buf at abs32@lo
+ ; Move base to VGPR.
+ v_mov_b32_e32 v0, s1
+ ; Load i8 from lds_buf[base].
+ ds_read_u8 v0, v0
+ s_endpgm
+.Luse_lds_a_end:
+ .size use_lds_a, .Luse_lds_a_end-use_lds_a
+
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl use_lds_a.kd
+ .type use_lds_a.kd, at object
+ .size use_lds_a.kd, 64
+use_lds_a.kd:
+ .zero 64
+
+ .text
+ .amdgpu_info use_lds_a
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_sgpr 4
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .globl lds_arr
+ .amdgpu_lds lds_arr, 64, 16
+
+#--- b.s
+ .text
+ .globl use_lds_b
+ .p2align 8
+ .type use_lds_b, at function
+use_lds_b:
+ ; Load base address of lds_buf (relocation target).
+ s_mov_b32 s0, lds_buf at abs32@lo
+ ; Move base to VGPR.
+ v_mov_b32_e32 v1, s0
+ ; Store i8 99 to lds_buf[base].
+ v_mov_b32_e32 v2, 99
+ ds_write_b8 v1, v2
+ s_endpgm
+.Luse_lds_b_end:
+ .size use_lds_b, .Luse_lds_b_end-use_lds_b
+
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl use_lds_b.kd
+ .type use_lds_b.kd, at object
+ .size use_lds_b.kd, 64
+use_lds_b.kd:
+ .zero 64
+
+ .text
+ .amdgpu_info use_lds_b
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_sgpr 4
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .globl lds_buf
+ .amdgpu_lds lds_buf, 32, 4
diff --git a/lld/test/ELF/amdgpu-lds-link-time-ir-instructions.s b/lld/test/ELF/amdgpu-lds-link-time-ir-instructions.s
new file mode 100644
index 0000000000000..5848699296809
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-ir-instructions.s
@@ -0,0 +1,266 @@
+# REQUIRES: amdgpu
+
+## End-to-end IR-to-linked-binary instruction verification for link-time LDS.
+## Two TUs share an LDS symbol (__lds_shared) across translation units and
+## TU2 also has a private LDS symbol (__lds_private). This tests the full
+## pipeline from compiler output through linking, checking:
+## 1) Pre-link: relocations in object files (R_AMDGPU_ABS32_LO)
+## 2) Pre-link: placeholder 0 in address-computation instructions
+## 3) Post-link: resolved offsets patched into instructions
+## 4) LDS load/store instructions remain intact through linking
+##
+## LDS layout (alignment desc, size desc):
+## __lds_shared (align=16, size=256) -> offset 0x000
+## __lds_private (align=4, size=128) -> offset 0x100
+##
+## TU1: kernel_a stores 42 to __lds_shared[idx].
+## TU2: kernel_b reads __lds_shared[idx], increments it, writes it back,
+## and stores 99 to __lds_private[idx].
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/tu1.s -o %t/tu1.o
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/tu2.s -o %t/tu2.o
+
+## Pre-link: verify relocations target the LDS symbols.
+# RUN: llvm-objdump -d -r %t/tu1.o | FileCheck %s --check-prefix=PRE1
+# RUN: llvm-objdump -d -r %t/tu2.o | FileCheck %s --check-prefix=PRE2
+
+## Link.
+# RUN: ld.lld %t/tu1.o %t/tu2.o -o %t/out
+
+## Post-link: verify resolved instructions.
+# RUN: llvm-objdump -d %t/out | FileCheck %s --check-prefix=POST
+
+## === Pre-link TU1: kernel_a ===
+## The compiler emits s_lshl2_add_u32 to compute byte offset (idx << 2) + base.
+## The LDS base is a placeholder 0 with a relocation.
+
+# PRE1-LABEL: <kernel_a>:
+# PRE1: s_lshl2_add_u32 s0, s0, lit(0x0)
+# PRE1-NEXT: {{.*}} R_AMDGPU_ABS32_LO __lds_shared
+# PRE1: ds_write_b32 v{{[0-9]+}}, v{{[0-9]+}}
+
+## === Pre-link TU2: kernel_b ===
+## Two LDS accesses: __lds_shared and __lds_private, both with relocations.
+
+# PRE2-LABEL: <kernel_b>:
+## First access: __lds_shared (s_add_i32 with relocation).
+# PRE2: s_add_i32 s{{[0-9]+}}, s0, lit(0x0)
+# PRE2-NEXT: {{.*}} R_AMDGPU_ABS32_LO __lds_shared
+# PRE2: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}}
+## Second access: __lds_private (s_add_i32 with relocation).
+# PRE2: s_add_i32 s0, s0, lit(0x0)
+# PRE2-NEXT: {{.*}} R_AMDGPU_ABS32_LO __lds_private
+# PRE2: ds_write_b32 v{{[0-9]+}}, v{{[0-9]+}}
+## Increment and write-back to __lds_shared.
+# PRE2: v_add_u32_e32 v{{[0-9]+}}, 1, v{{[0-9]+}}
+# PRE2: ds_write_b32 v{{[0-9]+}}, v{{[0-9]+}}
+
+## === Post-link: resolved offsets ===
+
+## kernel_a: __lds_shared resolved to offset 0.
+# POST-LABEL: <kernel_a>:
+# POST: s_lshl2_add_u32 s0, s0, lit(0x0)
+# POST: ds_write_b32 v{{[0-9]+}}, v{{[0-9]+}}
+
+## kernel_b: __lds_shared at offset 0, __lds_private at offset 0x100.
+# POST-LABEL: <kernel_b>:
+## __lds_shared base = 0 (unchanged from placeholder).
+# POST: s_add_i32 s{{[0-9]+}}, s0, lit(0x0)
+# POST: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}}
+## __lds_private base = 0x100 (resolved from placeholder 0).
+# POST: s_add_i32 s0, s0, 0x100
+# POST: ds_write_b32 v{{[0-9]+}}, v{{[0-9]+}}
+## Increment + write-back.
+# POST: v_add_u32_e32 v{{[0-9]+}}, 1, v{{[0-9]+}}
+# POST: ds_write_b32 v{{[0-9]+}}, v{{[0-9]+}}
+
+#--- tu1.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl kernel_a ; -- Begin function kernel_a
+ .p2align 8
+ .type kernel_a, at function
+kernel_a: ; @kernel_a
+ s_load_dword s0, s[8:9], 0x0
+ v_mov_b32_e32 v0, 42
+ s_waitcnt lgkmcnt(0)
+ s_lshl2_add_u32 s0, s0, __lds_shared at abs32@lo
+ v_mov_b32_e32 v1, s0
+ ds_write_b32 v1, v0
+ s_endpgm
+.Lfunc_end0:
+ .size kernel_a, .Lfunc_end0-kernel_a
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl kernel_a.kd
+ .type kernel_a.kd, at object
+ .size kernel_a.kd, 64
+ .protected kernel_a
+kernel_a.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad kernel_a at rel64-kernel_a.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11468864
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .amdgpu_info kernel_a
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 2
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 10
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .globl __lds_shared
+ .amdgpu_lds __lds_shared, 256, 16
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: kernel_a
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: kernel_a.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+#--- tu2.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl kernel_b ; -- Begin function kernel_b
+ .p2align 8
+ .type kernel_b, at function
+kernel_b: ; @kernel_b
+ s_load_dword s0, s[8:9], 0x0
+ v_mov_b32_e32 v2, 0x63
+ s_waitcnt lgkmcnt(0)
+ s_lshl_b32 s0, s0, 2
+ s_add_i32 s1, s0, __lds_shared at abs32@lo
+ v_mov_b32_e32 v0, s1
+ ds_read_b32 v1, v0
+ s_add_i32 s0, s0, __lds_private at abs32@lo
+ v_mov_b32_e32 v3, s0
+ ds_write_b32 v3, v2
+ s_waitcnt lgkmcnt(1)
+ v_add_u32_e32 v1, 1, v1
+ ds_write_b32 v0, v1
+ s_endpgm
+.Lfunc_end0:
+ .size kernel_b, .Lfunc_end0-kernel_b
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl kernel_b.kd
+ .type kernel_b.kd, at object
+ .size kernel_b.kd, 64
+ .protected kernel_b
+kernel_b.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad kernel_b at rel64-kernel_b.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11468864
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .amdgpu_info kernel_b
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 10
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .globl __lds_private
+ .amdgpu_lds __lds_private, 128, 4
+ .globl __lds_shared
+ .amdgpu_lds __lds_shared, 256, 16
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: kernel_b
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: kernel_b.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-lds-link-time-kd.s b/lld/test/ELF/amdgpu-lds-link-time-kd.s
new file mode 100644
index 0000000000000..cc9eb11c0c54c
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-kd.s
@@ -0,0 +1,175 @@
+# REQUIRES: amdgpu
+
+## Test that lld patches the kernel descriptor's group_segment_fixed_size field
+## with the resolved per-kernel LDS size.
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/a.s -o %t/a.o
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/b.s -o %t/b.o
+# RUN: ld.lld %t/a.o %t/b.o -o %t/out
+
+## Verify LDS symbol offset is assigned.
+# RUN: llvm-readelf -s %t/out | FileCheck %s --check-prefix=SYM
+
+## Verify group_segment_fixed_size in HSA metadata is patched to 256 (0x100).
+# RUN: llvm-readobj --notes %t/out | FileCheck %s --check-prefix=META
+
+## LDS layout:
+## lds_data: align=16, size=256 -> offset 0
+## Total LDS = 256 = 0x100
+
+# SYM: 0000000000000000 {{.*}} lds_data
+
+# META: .group_segment_fixed_size: 256
+# META: .name: my_kernel
+
+#--- a.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl my_kernel ; -- Begin function my_kernel
+ .p2align 8
+ .type my_kernel, at function
+my_kernel:
+ s_endpgm
+.Lfunc_end0:
+ .size my_kernel, .Lfunc_end0-my_kernel
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl my_kernel.kd
+ .type my_kernel.kd, at object
+ .size my_kernel.kd, 64
+ .protected my_kernel
+my_kernel.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad my_kernel at rel64-my_kernel.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11469063
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .Lmy_kernel.num_vgpr, 32
+ .set .Lmy_kernel.num_agpr, 0
+ .set .Lmy_kernel.numbered_sgpr, 33
+ .set .Lmy_kernel.num_named_barrier, 0
+ .set .Lmy_kernel.private_seg_size, 0
+ .set .Lmy_kernel.uses_vcc, 1
+ .set .Lmy_kernel.uses_flat_scratch, 1
+ .set .Lmy_kernel.has_dyn_sized_stack, 0
+ .set .Lmy_kernel.has_recursion, 0
+ .set .Lmy_kernel.has_indirect_call, 0
+ .amdgpu_info my_kernel
+ .amdgpu_flags 3
+ .amdgpu_num_vgpr 32
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 33
+ .amdgpu_private_segment_size 0
+ .amdgpu_use lds_data
+ .amdgpu_call helper
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 0
+ .set amdgpu.max_num_named_barrier, 0
+ .globl lds_data
+ .amdgpu_lds lds_data, 256, 16
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: my_kernel
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: my_kernel.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- b.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl helper ; -- Begin function helper
+ .p2align 6
+ .type helper, at function
+helper:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end0:
+ .size helper, .Lfunc_end0-helper
+ .set .Lhelper.num_vgpr, 0
+ .set .Lhelper.num_agpr, 0
+ .set .Lhelper.numbered_sgpr, 32
+ .set .Lhelper.num_named_barrier, 0
+ .set .Lhelper.private_seg_size, 0
+ .set .Lhelper.uses_vcc, 0
+ .set .Lhelper.uses_flat_scratch, 0
+ .set .Lhelper.has_dyn_sized_stack, 0
+ .set .Lhelper.has_recursion, 0
+ .set .Lhelper.has_indirect_call, 0
+ .amdgpu_info helper
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 0
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 32
+ .set amdgpu.max_num_named_barrier, 0
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels: []
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-lds-link-time-layout.s b/lld/test/ELF/amdgpu-lds-link-time-layout.s
new file mode 100644
index 0000000000000..b5d7338b23b79
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-layout.s
@@ -0,0 +1,170 @@
+# REQUIRES: amdgpu
+
+## Comprehensive test for the LDS layout algorithm. The linker sorts
+## SHN_AMDGPU_LDS symbols by alignment (desc), size (desc), then name (asc)
+## for deterministic output, and inserts padding for alignment.
+##
+## Symbols (deliberately supplied in a scrambled order across TUs):
+## lds_a16_s64 align=16 size=64 — tier 1 (highest alignment)
+## lds_a16_s32 align=16 size=32 — tier 1, smaller (size tiebreaker)
+## lds_a8_s20 align=8 size=20 — tier 2
+## lds_a4_s12_x align=4 size=12 — tier 3 (name tiebreaker with y)
+## lds_a4_s12_y align=4 size=12 — tier 3, same align+size, name > x
+## lds_a1_s3 align=1 size=3 — tier 4 (lowest alignment)
+##
+## Expected sorted order (alignment desc, size desc, name asc):
+## lds_a16_s64 align=16 size=64 -> offset 0x00 (0)
+## lds_a16_s32 align=16 size=32 -> offset 0x40 (64)
+## lds_a8_s20 align=8 size=20 -> offset 0x60 (96, 64+32=96 already 8-aligned)
+## lds_a4_s12_x align=4 size=12 -> offset 0x74 (116, 96+20=116 already 4-aligned)
+## lds_a4_s12_y align=4 size=12 -> offset 0x80 (128, 116+12=128 already 4-aligned)
+## lds_a1_s3 align=1 size=3 -> offset 0x8C (140, 128+12=140 1-aligned trivially)
+##
+## Total: 143 bytes.
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa -mcpu=gfx900 %t/tu1.s -o %t/tu1.o
+# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa -mcpu=gfx900 %t/tu2.s -o %t/tu2.o
+# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa -mcpu=gfx900 %t/tu3.s -o %t/tu3.o
+# RUN: ld.lld %t/tu1.o %t/tu2.o %t/tu3.o -o %t/out
+# RUN: llvm-readobj --syms %t/out | FileCheck %s
+
+## Symbols appear in file-definition order after linking (tu1, tu2, tu3).
+## Verify each symbol's assigned offset matches the layout algorithm:
+## sort by alignment desc, size desc, name asc.
+
+## From tu1: lds_a4_s12_y (align=4, size=12) -> offset 0x80
+# CHECK: Name: lds_a4_s12_y
+# CHECK-NEXT: Value: 0x80
+# CHECK-NEXT: Size: 12
+
+## From tu1: lds_a16_s64 (align=16, size=64) -> offset 0x0
+# CHECK: Name: lds_a16_s64
+# CHECK-NEXT: Value: 0x0
+# CHECK-NEXT: Size: 64
+
+## From tu2: lds_a1_s3 (align=1, size=3) -> offset 0x8C
+# CHECK: Name: lds_a1_s3
+# CHECK-NEXT: Value: 0x8C
+# CHECK-NEXT: Size: 3
+
+## From tu2: lds_a8_s20 (align=8, size=20) -> offset 0x60
+# CHECK: Name: lds_a8_s20
+# CHECK-NEXT: Value: 0x60
+# CHECK-NEXT: Size: 20
+
+## From tu3: lds_a16_s32 (align=16, size=32) -> offset 0x40
+# CHECK: Name: lds_a16_s32
+# CHECK-NEXT: Value: 0x40
+# CHECK-NEXT: Size: 32
+
+## From tu3: lds_a4_s12_x (align=4, size=12) -> offset 0x74
+# CHECK: Name: lds_a4_s12_x
+# CHECK-NEXT: Value: 0x74
+# CHECK-NEXT: Size: 12
+
+#--- tu1.s
+## Deliberately interleave symbols from different tiers.
+ .text
+ .globl f1
+ .p2align 8
+ .type f1, at function
+f1:
+ s_mov_b32 s0, lds_a4_s12_y at abs32@lo
+ s_mov_b32 s1, lds_a16_s64 at abs32@lo
+ s_endpgm
+.Lf1_end:
+ .size f1, .Lf1_end-f1
+
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl f1.kd
+ .type f1.kd, at object
+ .size f1.kd, 64
+f1.kd:
+ .zero 64
+
+ .text
+ .amdgpu_info f1
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_sgpr 4
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .globl lds_a4_s12_y
+ .amdgpu_lds lds_a4_s12_y, 12, 4
+
+ .globl lds_a16_s64
+ .amdgpu_lds lds_a16_s64, 64, 16
+
+#--- tu2.s
+ .text
+ .globl f2
+ .p2align 8
+ .type f2, at function
+f2:
+ s_mov_b32 s0, lds_a1_s3 at abs32@lo
+ s_mov_b32 s1, lds_a8_s20 at abs32@lo
+ s_endpgm
+.Lf2_end:
+ .size f2, .Lf2_end-f2
+
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl f2.kd
+ .type f2.kd, at object
+ .size f2.kd, 64
+f2.kd:
+ .zero 64
+
+ .text
+ .amdgpu_info f2
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_sgpr 4
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .globl lds_a1_s3
+ .amdgpu_lds lds_a1_s3, 3, 1
+
+ .globl lds_a8_s20
+ .amdgpu_lds lds_a8_s20, 20, 8
+
+#--- tu3.s
+ .text
+ .globl f3
+ .p2align 8
+ .type f3, at function
+f3:
+ s_mov_b32 s0, lds_a16_s32 at abs32@lo
+ s_mov_b32 s1, lds_a4_s12_x at abs32@lo
+ s_endpgm
+.Lf3_end:
+ .size f3, .Lf3_end-f3
+
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl f3.kd
+ .type f3.kd, at object
+ .size f3.kd, 64
+f3.kd:
+ .zero 64
+
+ .text
+ .amdgpu_info f3
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_sgpr 4
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .globl lds_a16_s32
+ .amdgpu_lds lds_a16_s32, 32, 16
+
+ .globl lds_a4_s12_x
+ .amdgpu_lds lds_a4_s12_x, 12, 4
diff --git a/lld/test/ELF/amdgpu-lds-link-time-named-barrier.s b/lld/test/ELF/amdgpu-lds-link-time-named-barrier.s
new file mode 100644
index 0000000000000..7ee5c1063144f
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-named-barrier.s
@@ -0,0 +1,260 @@
+# REQUIRES: amdgpu
+
+## Test that lld patches compute_pgm_rsrc3.NAMED_BAR_CNT with the cross-TU
+## propagated named-barrier count for GFX1250.
+##
+## TU A: kern_a uses 1 named barrier, calls external helper.
+## TU B: helper uses 5 named barriers, kern_b also uses 5.
+## After linking, kern_a gets max(1, 5) = 5 barriers -> NamedBarCnt = ceil(5/4) = 2.
+## NAMED_BAR_CNT occupies bits [14:16] of compute_pgm_rsrc3.
+## Expected: 2 << 14 = 0x8000 for both kern_a and kern_b.
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx1250 -filetype=obj %t/a.s -o %t/a.o
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx1250 -filetype=obj %t/b.s -o %t/b.o
+# RUN: ld.lld %t/a.o %t/b.o -o %t/out
+
+## .rodata contains two 64-byte kernel descriptors. Check compute_pgm_rsrc3 at
+## offset 44 from each KD start. NAMED_BAR_CNT=2 is encoded as 0x00008000,
+## printed below as little-endian bytes 00800000.
+# RUN: llvm-objdump -s -j .rodata %t/out | FileCheck %s --check-prefix=KD
+
+# KD: {{[0-9a-f]+}} 00000000 00000000 00000000 00800000
+# KD: {{[0-9a-f]+}} 00000000 00000000 00000000 00800000
+
+#--- a.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx1250"
+ .amdhsa_code_object_version 6
+ .text
+ .globl kern_a ; -- Begin function kern_a
+ .p2align 8
+ .type kern_a, at function
+kern_a:
+ s_endpgm
+.Lfunc_end0:
+ .size kern_a, .Lfunc_end0-kern_a
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl kern_a.kd
+ .type kern_a.kd, at object
+ .size kern_a.kd, 64
+ .protected kern_a
+kern_a.kd:
+ .long 0
+ .long 0
+ .long 256
+ .long 0
+ .quad kern_a at rel64-kern_a.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 3222208513
+ .long 5008
+ .short 1054
+ .short 0
+ .long 0
+ .text
+ .set .Lkern_a.num_vgpr, 32
+ .set .Lkern_a.num_agpr, 0
+ .set .Lkern_a.numbered_sgpr, 33
+ .set .Lkern_a.num_named_barrier, 0
+ .set .Lkern_a.private_seg_size, 0
+ .set .Lkern_a.uses_vcc, 1
+ .set .Lkern_a.uses_flat_scratch, 0
+ .set .Lkern_a.has_dyn_sized_stack, 0
+ .set .Lkern_a.has_recursion, 0
+ .set .Lkern_a.has_indirect_call, 0
+ .text
+ .amdgpu_info kern_a
+ .amdgpu_flags 17
+ .amdgpu_num_vgpr 32
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 33
+ .amdgpu_private_segment_size 0
+ .amdgpu_use __amdgpu_named_barrier.bar_a.4afea3fa75d4f11a11e1e1e3237d94b2
+ .amdgpu_call helper
+ .amdgpu_occupancy 8
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 0
+ .set amdgpu.max_num_named_barrier, 0
+ .text
+ .globl __amdgpu_named_barrier.bar_a.4afea3fa75d4f11a11e1e1e3237d94b2
+ .amdgpu_lds __amdgpu_named_barrier.bar_a.4afea3fa75d4f11a11e1e1e3237d94b2, 16, 4
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 256
+ .max_flat_workgroup_size: 1024
+ .name: kern_a
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: kern_a.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 32
+amdhsa.target: amdgcn-amd-amdhsa--gfx1250
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- b.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx1250"
+ .amdhsa_code_object_version 6
+ .text
+ .globl helper ; -- Begin function helper
+ .p2align 7
+ .type helper, at function
+helper:
+ s_setpc_b64 s[30:31]
+.Lfunc_end0:
+ .size helper, .Lfunc_end0-helper
+ .set .Lhelper.num_vgpr, 0
+ .set .Lhelper.num_agpr, 0
+ .set .Lhelper.numbered_sgpr, 32
+ .set .Lhelper.num_named_barrier, 0
+ .set .Lhelper.private_seg_size, 0
+ .set .Lhelper.uses_vcc, 0
+ .set .Lhelper.uses_flat_scratch, 0
+ .set .Lhelper.has_dyn_sized_stack, 0
+ .set .Lhelper.has_recursion, 0
+ .set .Lhelper.has_indirect_call, 0
+ .text
+ .globl kern_b ; -- Begin function kern_b
+ .p2align 8
+ .type kern_b, at function
+kern_b:
+ s_endpgm
+.Lfunc_end1:
+ .size kern_b, .Lfunc_end1-kern_b
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl kern_b.kd
+ .type kern_b.kd, at object
+ .size kern_b.kd, 64
+ .protected kern_b
+kern_b.kd:
+ .long 0
+ .long 0
+ .long 256
+ .long 0
+ .quad kern_b at rel64-kern_b.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 3222208512
+ .long 5008
+ .short 1054
+ .short 0
+ .long 0
+ .text
+ .set .Lkern_b.num_vgpr, 0
+ .set .Lkern_b.num_agpr, 0
+ .set .Lkern_b.numbered_sgpr, 1
+ .set .Lkern_b.num_named_barrier, 0
+ .set .Lkern_b.private_seg_size, 0
+ .set .Lkern_b.uses_vcc, 0
+ .set .Lkern_b.uses_flat_scratch, 0
+ .set .Lkern_b.has_dyn_sized_stack, 0
+ .set .Lkern_b.has_recursion, 0
+ .set .Lkern_b.has_indirect_call, 0
+ .text
+ .amdgpu_info helper
+ .amdgpu_flags 16
+ .amdgpu_num_vgpr 0
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 0
+ .amdgpu_use __amdgpu_named_barrier.bar_b.36a90f902d5ba5557b314d56b5259233
+ .amdgpu_occupancy 8
+ .end_amdgpu_info
+
+ .amdgpu_info kern_b
+ .amdgpu_flags 16
+ .amdgpu_num_vgpr 0
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use __amdgpu_named_barrier.bar_b.36a90f902d5ba5557b314d56b5259233
+ .amdgpu_occupancy 8
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 32
+ .set amdgpu.max_num_named_barrier, 0
+ .text
+ .globl __amdgpu_named_barrier.bar_b.36a90f902d5ba5557b314d56b5259233
+ .amdgpu_lds __amdgpu_named_barrier.bar_b.36a90f902d5ba5557b314d56b5259233, 80, 4
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 256
+ .max_flat_workgroup_size: 1024
+ .name: kern_b
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: kern_b.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 32
+amdhsa.target: amdgcn-amd-amdhsa--gfx1250
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-lds-link-time-ordering-complex.s b/lld/test/ELF/amdgpu-lds-link-time-ordering-complex.s
new file mode 100644
index 0000000000000..ea0cac388bb6b
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-ordering-complex.s
@@ -0,0 +1,402 @@
+# REQUIRES: amdgpu
+
+## Test shared-tier ordering with 4 kernels and 4 shared-tier variables with
+## a complex overlapping usage pattern.
+##
+## Variables (all external linkage -> global-scope -> shared tier):
+## V1: 4 bytes align 4, used by K1, K2, K3, K4 (4 users)
+## V2: 8 bytes align 4, used by K1, K2, K3 (3 users)
+## V3: 16 bytes align 4->16*, used by K1, K2 (2 users)
+## V4: 32 bytes align 4->16*, used by K1, K3 (2 users)
+## (* superAlignLDSGlobals bumps alignment to 16 for vars >= 16 bytes)
+##
+## Per-kernel frontier order:
+## V1 and V2 are placed first by use count. V4 and V3 have equal use counts,
+## so V4 is placed before V3 by the size/alignment tie-breaker.
+##
+## Layout: V1(4,a4)@0x00, V2(8,a8)@0x08, V4(32,a16)@0x10, V3(16,a16)@0x30
+## (superAlignLDSGlobals also bumps V2 to align 8)
+##
+## Per-kernel sizes:
+## K1: all -> max(4, 16, 32, 64) = 64 = 0x40
+## K2: V1,V2,V3 -> max(4, 16, 64) = 64 = 0x40
+## K3: V1,V2,V4 -> max(4, 16, 48) = 48 = 0x30
+## K4: V1 -> 4 = 0x04
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/tu1.s -o %t/tu1.o
+# RUN: ld.lld %t/tu1.o -o %t/out
+# RUN: llvm-readelf -s %t/out | FileCheck %s --check-prefix=SYM
+# RUN: llvm-readobj --notes %t/out | FileCheck %s --check-prefix=META
+
+## V1 (4 users) at lowest offset
+# SYM-DAG: 0000000000000000 {{.*}} V1
+## V2 (was 3 users, placed early after dynamic updates; aligned to 8)
+# SYM-DAG: 0000000000000008 {{.*}} V2
+## V4 (2 users, largest) before V3
+# SYM-DAG: 0000000000000010 {{.*}} V4
+# SYM-DAG: 0000000000000030 {{.*}} V3
+
+## K1 uses all: size = 64
+# META-DAG: .group_segment_fixed_size: 64
+## K2 uses V1,V2,V3: size = 64
+# META-DAG: .group_segment_fixed_size: 64
+## K3 uses V1,V2,V4: size = 48
+# META-DAG: .group_segment_fixed_size: 48
+## K4 uses only V1: minimal size = 4
+# META-DAG: .group_segment_fixed_size: 4
+
+#--- tu1.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl K1 ; -- Begin function K1
+ .p2align 8
+ .type K1, at function
+K1:
+ s_endpgm
+.Lfunc_end0:
+ .size K1, .Lfunc_end0-K1
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl K1.kd
+ .type K1.kd, at object
+ .size K1.kd, 64
+ .protected K1
+K1.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad K1 at rel64-K1.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11468864
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .LK1.num_vgpr, 3
+ .set .LK1.num_agpr, 0
+ .set .LK1.numbered_sgpr, 10
+ .set .LK1.num_named_barrier, 0
+ .set .LK1.private_seg_size, 0
+ .set .LK1.uses_vcc, 0
+ .set .LK1.uses_flat_scratch, 0
+ .set .LK1.has_dyn_sized_stack, 0
+ .set .LK1.has_recursion, 0
+ .set .LK1.has_indirect_call, 0
+ .text
+ .globl K2 ; -- Begin function K2
+ .p2align 8
+ .type K2, at function
+K2:
+ s_endpgm
+.Lfunc_end1:
+ .size K2, .Lfunc_end1-K2
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl K2.kd
+ .type K2.kd, at object
+ .size K2.kd, 64
+ .protected K2
+K2.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad K2 at rel64-K2.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11468864
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .LK2.num_vgpr, 3
+ .set .LK2.num_agpr, 0
+ .set .LK2.numbered_sgpr, 10
+ .set .LK2.num_named_barrier, 0
+ .set .LK2.private_seg_size, 0
+ .set .LK2.uses_vcc, 0
+ .set .LK2.uses_flat_scratch, 0
+ .set .LK2.has_dyn_sized_stack, 0
+ .set .LK2.has_recursion, 0
+ .set .LK2.has_indirect_call, 0
+ .text
+ .globl K3 ; -- Begin function K3
+ .p2align 8
+ .type K3, at function
+K3:
+ s_endpgm
+.Lfunc_end2:
+ .size K3, .Lfunc_end2-K3
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl K3.kd
+ .type K3.kd, at object
+ .size K3.kd, 64
+ .protected K3
+K3.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad K3 at rel64-K3.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11468864
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .LK3.num_vgpr, 3
+ .set .LK3.num_agpr, 0
+ .set .LK3.numbered_sgpr, 10
+ .set .LK3.num_named_barrier, 0
+ .set .LK3.private_seg_size, 0
+ .set .LK3.uses_vcc, 0
+ .set .LK3.uses_flat_scratch, 0
+ .set .LK3.has_dyn_sized_stack, 0
+ .set .LK3.has_recursion, 0
+ .set .LK3.has_indirect_call, 0
+ .text
+ .globl K4 ; -- Begin function K4
+ .p2align 8
+ .type K4, at function
+K4:
+ s_endpgm
+.Lfunc_end3:
+ .size K4, .Lfunc_end3-K4
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl K4.kd
+ .type K4.kd, at object
+ .size K4.kd, 64
+ .protected K4
+K4.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad K4 at rel64-K4.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11468800
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .LK4.num_vgpr, 2
+ .set .LK4.num_agpr, 0
+ .set .LK4.numbered_sgpr, 0
+ .set .LK4.num_named_barrier, 0
+ .set .LK4.private_seg_size, 0
+ .set .LK4.uses_vcc, 0
+ .set .LK4.uses_flat_scratch, 0
+ .set .LK4.has_dyn_sized_stack, 0
+ .set .LK4.has_recursion, 0
+ .set .LK4.has_indirect_call, 0
+ .amdgpu_info K1
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 3
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 10
+ .amdgpu_private_segment_size 0
+ .amdgpu_use V3
+ .amdgpu_use V4
+ .amdgpu_use V1
+ .amdgpu_use V2
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info K2
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 3
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 10
+ .amdgpu_private_segment_size 0
+ .amdgpu_use V3
+ .amdgpu_use V1
+ .amdgpu_use V2
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info K3
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 3
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 10
+ .amdgpu_private_segment_size 0
+ .amdgpu_use V4
+ .amdgpu_use V1
+ .amdgpu_use V2
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info K4
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 2
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 0
+ .amdgpu_private_segment_size 0
+ .amdgpu_use V1
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 0
+ .set amdgpu.max_num_named_barrier, 0
+ .globl V1
+ .amdgpu_lds V1, 4, 4
+ .globl V2
+ .amdgpu_lds V2, 8, 8
+ .globl V3
+ .amdgpu_lds V3, 16, 16
+ .globl V4
+ .amdgpu_lds V4, 32, 16
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: K1
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: K1.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: K2
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: K2.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: K3
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: K3.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: K4
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: K4.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-lds-link-time-ordering-frequency.s b/lld/test/ELF/amdgpu-lds-link-time-ordering-frequency.s
new file mode 100644
index 0000000000000..d96e3c56bd4a8
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-ordering-frequency.s
@@ -0,0 +1,311 @@
+# REQUIRES: amdgpu
+
+## Test that usage frequency drives shared-tier placement. The variable used
+## by the most kernels should be placed at the lowest offset regardless of
+## its size.
+##
+## Variables (all external linkage -> global-scope -> shared tier):
+## S1: 16 bytes align 4->16*, used by K1, K2 (2 users)
+## S2: 8 bytes align 4, used by K1, K2, K3 (3 users, smallest!)
+## S3: 32 bytes align 4->16*, used by K1, K2 (2 users)
+## (* superAlignLDSGlobals bumps alignment to 16 for vars >= 16 bytes)
+##
+## Per-kernel frontier order:
+## S2 has the most users and is placed first. S3 and S1 have equal use
+## counts, so S3 is placed before S1 by the size/alignment tie-breaker.
+##
+## Layout: S2(8,a4)@0x00, S3(32,a16)@0x10, S1(16,a16)@0x30
+##
+## Per-kernel sizes:
+## K1: reaches all -> 64 = 0x40
+## K2: same -> 64 = 0x40
+## K3: reaches S2(0..8) -> 8 = 0x08 (zero waste!)
+##
+## A naive size-desc sort [S3,S1,S2] would give K3 size = 64 (waste = 56).
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/tu1.s -o %t/tu1.o
+# RUN: ld.lld %t/tu1.o -o %t/out
+# RUN: llvm-readelf -s %t/out | FileCheck %s --check-prefix=SYM
+# RUN: llvm-readobj --notes %t/out | FileCheck %s --check-prefix=META
+
+## S2 (most shared, 3 users) at lowest offset despite being smallest
+# SYM-DAG: 0000000000000000 {{.*}} S2
+# SYM-DAG: 0000000000000010 {{.*}} S3
+# SYM-DAG: 0000000000000030 {{.*}} S1
+
+## K3 uses only S2: minimal size
+# META-DAG: .group_segment_fixed_size: 8
+## K1 and K2 use all three
+# META-DAG: .group_segment_fixed_size: 64
+# META-DAG: .group_segment_fixed_size: 64
+
+#--- tu1.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl K1 ; -- Begin function K1
+ .p2align 8
+ .type K1, at function
+K1:
+ s_endpgm
+.Lfunc_end0:
+ .size K1, .Lfunc_end0-K1
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl K1.kd
+ .type K1.kd, at object
+ .size K1.kd, 64
+ .protected K1
+K1.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad K1 at rel64-K1.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11468864
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .LK1.num_vgpr, 4
+ .set .LK1.num_agpr, 0
+ .set .LK1.numbered_sgpr, 10
+ .set .LK1.num_named_barrier, 0
+ .set .LK1.private_seg_size, 0
+ .set .LK1.uses_vcc, 0
+ .set .LK1.uses_flat_scratch, 0
+ .set .LK1.has_dyn_sized_stack, 0
+ .set .LK1.has_recursion, 0
+ .set .LK1.has_indirect_call, 0
+ .text
+ .globl K2 ; -- Begin function K2
+ .p2align 8
+ .type K2, at function
+K2:
+ s_endpgm
+.Lfunc_end1:
+ .size K2, .Lfunc_end1-K2
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl K2.kd
+ .type K2.kd, at object
+ .size K2.kd, 64
+ .protected K2
+K2.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad K2 at rel64-K2.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11468864
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .LK2.num_vgpr, 4
+ .set .LK2.num_agpr, 0
+ .set .LK2.numbered_sgpr, 10
+ .set .LK2.num_named_barrier, 0
+ .set .LK2.private_seg_size, 0
+ .set .LK2.uses_vcc, 0
+ .set .LK2.uses_flat_scratch, 0
+ .set .LK2.has_dyn_sized_stack, 0
+ .set .LK2.has_recursion, 0
+ .set .LK2.has_indirect_call, 0
+ .text
+ .globl K3 ; -- Begin function K3
+ .p2align 8
+ .type K3, at function
+K3:
+ s_endpgm
+.Lfunc_end2:
+ .size K3, .Lfunc_end2-K3
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl K3.kd
+ .type K3.kd, at object
+ .size K3.kd, 64
+ .protected K3
+K3.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad K3 at rel64-K3.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11468864
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .LK3.num_vgpr, 2
+ .set .LK3.num_agpr, 0
+ .set .LK3.numbered_sgpr, 10
+ .set .LK3.num_named_barrier, 0
+ .set .LK3.private_seg_size, 0
+ .set .LK3.uses_vcc, 0
+ .set .LK3.uses_flat_scratch, 0
+ .set .LK3.has_dyn_sized_stack, 0
+ .set .LK3.has_recursion, 0
+ .set .LK3.has_indirect_call, 0
+ .amdgpu_info K1
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 10
+ .amdgpu_private_segment_size 0
+ .amdgpu_use S3
+ .amdgpu_use S2
+ .amdgpu_use S1
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info K2
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 10
+ .amdgpu_private_segment_size 0
+ .amdgpu_use S3
+ .amdgpu_use S2
+ .amdgpu_use S1
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info K3
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 2
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 10
+ .amdgpu_private_segment_size 0
+ .amdgpu_use S2
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 0
+ .set amdgpu.max_num_named_barrier, 0
+ .globl S1
+ .amdgpu_lds S1, 16, 16
+ .globl S2
+ .amdgpu_lds S2, 8, 8
+ .globl S3
+ .amdgpu_lds S3, 32, 16
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: K1
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: K1.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: K2
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: K2.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: K3
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: K3.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-lds-link-time-ordering-frontier.s b/lld/test/ELF/amdgpu-lds-link-time-ordering-frontier.s
new file mode 100644
index 0000000000000..38309bae811d1
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-ordering-frontier.s
@@ -0,0 +1,112 @@
+# REQUIRES: amdgpu
+
+## Test per-kernel LDS frontiers. P is allocated first because it has the most
+## users. L can still start at offset 0 because none of P's users use L. M then
+## lands at the max frontier of K1 and K2.
+##
+## K1: P, M
+## K2: M, L
+## K3: L
+## K4: P
+## K5: P
+
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %s -o %t.o
+# RUN: ld.lld %t.o -o %t
+# RUN: llvm-readelf -s %t | FileCheck %s --check-prefix=SYM
+
+# SYM-DAG: 0000000000000000 {{.*}} P
+# SYM-DAG: 0000000000000000 {{.*}} L
+# SYM-DAG: 0000000000000010 {{.*}} M
+
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+
+ .macro kernel name
+ .text
+ .globl \name
+ .p2align 8
+ .type \name, at function
+\name:
+ s_endpgm
+.L\name\()_end:
+ .size \name, .L\name\()_end-\name
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl \name\().kd
+ .type \name\().kd, at object
+ .size \name\().kd, 64
+ .protected \name
+\name\().kd:
+ .long 0
+ .long 0
+ .long 0
+ .long 0
+ .quad \name at rel64-\name\().kd
+ .zero 20
+ .long 0
+ .long 0
+ .long 0
+ .short 0
+ .short 0
+ .long 0
+ .endm
+
+ kernel K1
+ kernel K2
+ kernel K3
+ kernel K4
+ kernel K5
+
+ .amdgpu_info K1
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use P
+ .amdgpu_use M
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info K2
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use M
+ .amdgpu_use L
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info K3
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use L
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info K4
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use P
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info K5
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use P
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .globl P
+ .amdgpu_lds P, 8, 8
+ .globl L
+ .amdgpu_lds L, 16, 16
+ .globl M
+ .amdgpu_lds M, 4, 4
diff --git a/lld/test/ELF/amdgpu-lds-link-time-ordering-multigroup.s b/lld/test/ELF/amdgpu-lds-link-time-ordering-multigroup.s
new file mode 100644
index 0000000000000..0ad2b86f1409e
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-ordering-multigroup.s
@@ -0,0 +1,505 @@
+# REQUIRES: amdgpu
+
+## Test that multiple independent groups each have their shared-tier ordering
+## optimized independently, and each group allocates from offset 0.
+##
+## Group 1 (K1, K2 share A and B):
+## A: 8 bytes align 4, used by K1, K2, K3 (3 users)
+## B: 16 bytes align 4->16*, used by K1, K2 (2 users)
+## C: 32 bytes align 4->16*, used by K1, K2 (2 users)
+##
+## Group 2 (K4, K5 share D):
+## D: 4 bytes align 4, used by K4, K5 (2 users)
+## E: 8 bytes align 4, used by K4, K5 (2 users)
+##
+## Groups are independent (no shared LDS between them).
+##
+## Group 1 frontier order: [A, C, B] (A at 0x00, C at 0x10, B at 0x30)
+## K3 uses only A -> size = 8
+##
+## Group 2 frontier order: both D and E have use_count=2.
+## Tie-break by size: E(8)>D(4), so E is placed first.
+## Result: [E, D] (E at 0x00, D at 0x08)
+## (* superAlignLDSGlobals bumps alignment to 16 for vars >= 16 bytes,
+## and E to align 8 since it is 8 bytes)
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/tu1.s -o %t/tu1.o
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/tu2.s -o %t/tu2.o
+# RUN: ld.lld %t/tu1.o %t/tu2.o -o %t/out
+# RUN: llvm-readelf -s %t/out | FileCheck %s --check-prefix=SYM
+# RUN: llvm-readobj --notes %t/out | FileCheck %s --check-prefix=META
+
+## Group 1: A most shared, at offset 0
+# SYM-DAG: 0000000000000000 {{.*}} A
+# SYM-DAG: 0000000000000010 {{.*}} C
+# SYM-DAG: 0000000000000030 {{.*}} B
+
+## Group 2 is independent and starts from offset 0
+# SYM-DAG: 0000000000000000 {{.*}} E
+# SYM-DAG: 0000000000000008 {{.*}} D
+
+## K3 uses only A -> 8 bytes
+# META-DAG: .group_segment_fixed_size: 8
+## K1 and K2 use all of group 1 -> 64 bytes
+# META-DAG: .group_segment_fixed_size: 64
+# META-DAG: .group_segment_fixed_size: 64
+## K4 and K5 use all of group 2 -> 12 bytes (E at 0..8, D at 8..12)
+# META-DAG: .group_segment_fixed_size: 12
+# META-DAG: .group_segment_fixed_size: 12
+
+#--- tu1.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl K1 ; -- Begin function K1
+ .p2align 8
+ .type K1, at function
+K1:
+ s_endpgm
+.Lfunc_end0:
+ .size K1, .Lfunc_end0-K1
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl K1.kd
+ .type K1.kd, at object
+ .size K1.kd, 64
+ .protected K1
+K1.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad K1 at rel64-K1.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11468864
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .LK1.num_vgpr, 4
+ .set .LK1.num_agpr, 0
+ .set .LK1.numbered_sgpr, 10
+ .set .LK1.num_named_barrier, 0
+ .set .LK1.private_seg_size, 0
+ .set .LK1.uses_vcc, 0
+ .set .LK1.uses_flat_scratch, 0
+ .set .LK1.has_dyn_sized_stack, 0
+ .set .LK1.has_recursion, 0
+ .set .LK1.has_indirect_call, 0
+ .text
+ .globl K2 ; -- Begin function K2
+ .p2align 8
+ .type K2, at function
+K2:
+ s_endpgm
+.Lfunc_end1:
+ .size K2, .Lfunc_end1-K2
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl K2.kd
+ .type K2.kd, at object
+ .size K2.kd, 64
+ .protected K2
+K2.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad K2 at rel64-K2.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11468864
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .LK2.num_vgpr, 4
+ .set .LK2.num_agpr, 0
+ .set .LK2.numbered_sgpr, 10
+ .set .LK2.num_named_barrier, 0
+ .set .LK2.private_seg_size, 0
+ .set .LK2.uses_vcc, 0
+ .set .LK2.uses_flat_scratch, 0
+ .set .LK2.has_dyn_sized_stack, 0
+ .set .LK2.has_recursion, 0
+ .set .LK2.has_indirect_call, 0
+ .text
+ .globl K3 ; -- Begin function K3
+ .p2align 8
+ .type K3, at function
+K3:
+ s_endpgm
+.Lfunc_end2:
+ .size K3, .Lfunc_end2-K3
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl K3.kd
+ .type K3.kd, at object
+ .size K3.kd, 64
+ .protected K3
+K3.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad K3 at rel64-K3.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11468864
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .LK3.num_vgpr, 2
+ .set .LK3.num_agpr, 0
+ .set .LK3.numbered_sgpr, 10
+ .set .LK3.num_named_barrier, 0
+ .set .LK3.private_seg_size, 0
+ .set .LK3.uses_vcc, 0
+ .set .LK3.uses_flat_scratch, 0
+ .set .LK3.has_dyn_sized_stack, 0
+ .set .LK3.has_recursion, 0
+ .set .LK3.has_indirect_call, 0
+ .amdgpu_info K1
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 10
+ .amdgpu_private_segment_size 0
+ .amdgpu_use B
+ .amdgpu_use C
+ .amdgpu_use A
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info K2
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 10
+ .amdgpu_private_segment_size 0
+ .amdgpu_use B
+ .amdgpu_use C
+ .amdgpu_use A
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info K3
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 2
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 10
+ .amdgpu_private_segment_size 0
+ .amdgpu_use A
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 0
+ .set amdgpu.max_num_named_barrier, 0
+ .globl A
+ .amdgpu_lds A, 8, 8
+ .globl B
+ .amdgpu_lds B, 16, 16
+ .globl C
+ .amdgpu_lds C, 32, 16
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: K1
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: K1.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: K2
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: K2.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: K3
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: K3.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- tu2.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl K4 ; -- Begin function K4
+ .p2align 8
+ .type K4, at function
+K4:
+ s_endpgm
+.Lfunc_end0:
+ .size K4, .Lfunc_end0-K4
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl K4.kd
+ .type K4.kd, at object
+ .size K4.kd, 64
+ .protected K4
+K4.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad K4 at rel64-K4.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11468864
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .LK4.num_vgpr, 2
+ .set .LK4.num_agpr, 0
+ .set .LK4.numbered_sgpr, 10
+ .set .LK4.num_named_barrier, 0
+ .set .LK4.private_seg_size, 0
+ .set .LK4.uses_vcc, 0
+ .set .LK4.uses_flat_scratch, 0
+ .set .LK4.has_dyn_sized_stack, 0
+ .set .LK4.has_recursion, 0
+ .set .LK4.has_indirect_call, 0
+ .text
+ .globl K5 ; -- Begin function K5
+ .p2align 8
+ .type K5, at function
+K5:
+ s_endpgm
+.Lfunc_end1:
+ .size K5, .Lfunc_end1-K5
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl K5.kd
+ .type K5.kd, at object
+ .size K5.kd, 64
+ .protected K5
+K5.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad K5 at rel64-K5.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11468864
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .LK5.num_vgpr, 2
+ .set .LK5.num_agpr, 0
+ .set .LK5.numbered_sgpr, 10
+ .set .LK5.num_named_barrier, 0
+ .set .LK5.private_seg_size, 0
+ .set .LK5.uses_vcc, 0
+ .set .LK5.uses_flat_scratch, 0
+ .set .LK5.has_dyn_sized_stack, 0
+ .set .LK5.has_recursion, 0
+ .set .LK5.has_indirect_call, 0
+ .amdgpu_info K4
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 2
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 10
+ .amdgpu_private_segment_size 0
+ .amdgpu_use E
+ .amdgpu_use D
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info K5
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 2
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 10
+ .amdgpu_private_segment_size 0
+ .amdgpu_use E
+ .amdgpu_use D
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 0
+ .set amdgpu.max_num_named_barrier, 0
+ .globl D
+ .amdgpu_lds D, 4, 4
+ .globl E
+ .amdgpu_lds E, 8, 8
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: K4
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: K4.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: K5
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: K5.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-lds-link-time-ordering-nested.s b/lld/test/ELF/amdgpu-lds-link-time-ordering-nested.s
new file mode 100644
index 0000000000000..01ef8749c608e
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-ordering-nested.s
@@ -0,0 +1,312 @@
+# REQUIRES: amdgpu
+
+## Test shared-tier ordering with nested usage subsets. The greedy algorithm
+## should produce zero waste by placing the most-shared variable at the lowest
+## offset.
+##
+## Variables (all external linkage -> global-scope -> shared tier):
+## A: 8 bytes align 4, used by K1, K2, K3 (3 users)
+## B: 16 bytes align 4->16*, used by K1, K2 (2 users)
+## C: 32 bytes align 4->16*, used by K1, K2 (2 users)
+## (* superAlignLDSGlobals bumps alignment to 16 for vars >= 16 bytes)
+##
+## Per-kernel frontier order:
+## A has the most users and is placed first. C and B have equal use counts,
+## so C is placed before B by the size/alignment tie-breaker.
+##
+## Layout: A(8,a4)@0x00, C(32,a16)@0x10, B(16,a16)@0x30
+##
+## Per-kernel sizes:
+## K1: reaches A(0..8), C(16..48), B(48..64) -> 64 = 0x40
+## K2: same -> 64 = 0x40
+## K3: reaches A(0..8) -> 8 = 0x08 (zero waste!)
+##
+## A naive size-desc sort [C,B,A] would give K3 size = 64 (waste = 56).
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/tu1.s -o %t/tu1.o
+# RUN: ld.lld %t/tu1.o -o %t/out
+# RUN: llvm-readelf -s %t/out | FileCheck %s --check-prefix=SYM
+# RUN: llvm-readobj --notes %t/out | FileCheck %s --check-prefix=META
+
+## A (most shared, 3 users) at lowest offset
+# SYM-DAG: 0000000000000000 {{.*}} A
+## C next, then B at K1/K2's frontier
+# SYM-DAG: 0000000000000010 {{.*}} C
+# SYM-DAG: 0000000000000030 {{.*}} B
+
+## K3 should have minimal LDS size (only uses A, 8 bytes)
+# META-DAG: .group_segment_fixed_size: 8
+## K1 and K2 use all three: 64 bytes
+# META-DAG: .group_segment_fixed_size: 64
+# META-DAG: .group_segment_fixed_size: 64
+
+#--- tu1.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl K1 ; -- Begin function K1
+ .p2align 8
+ .type K1, at function
+K1:
+ s_endpgm
+.Lfunc_end0:
+ .size K1, .Lfunc_end0-K1
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl K1.kd
+ .type K1.kd, at object
+ .size K1.kd, 64
+ .protected K1
+K1.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad K1 at rel64-K1.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11468864
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .LK1.num_vgpr, 4
+ .set .LK1.num_agpr, 0
+ .set .LK1.numbered_sgpr, 10
+ .set .LK1.num_named_barrier, 0
+ .set .LK1.private_seg_size, 0
+ .set .LK1.uses_vcc, 0
+ .set .LK1.uses_flat_scratch, 0
+ .set .LK1.has_dyn_sized_stack, 0
+ .set .LK1.has_recursion, 0
+ .set .LK1.has_indirect_call, 0
+ .text
+ .globl K2 ; -- Begin function K2
+ .p2align 8
+ .type K2, at function
+K2:
+ s_endpgm
+.Lfunc_end1:
+ .size K2, .Lfunc_end1-K2
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl K2.kd
+ .type K2.kd, at object
+ .size K2.kd, 64
+ .protected K2
+K2.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad K2 at rel64-K2.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11468864
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .LK2.num_vgpr, 4
+ .set .LK2.num_agpr, 0
+ .set .LK2.numbered_sgpr, 10
+ .set .LK2.num_named_barrier, 0
+ .set .LK2.private_seg_size, 0
+ .set .LK2.uses_vcc, 0
+ .set .LK2.uses_flat_scratch, 0
+ .set .LK2.has_dyn_sized_stack, 0
+ .set .LK2.has_recursion, 0
+ .set .LK2.has_indirect_call, 0
+ .text
+ .globl K3 ; -- Begin function K3
+ .p2align 8
+ .type K3, at function
+K3:
+ s_endpgm
+.Lfunc_end2:
+ .size K3, .Lfunc_end2-K3
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl K3.kd
+ .type K3.kd, at object
+ .size K3.kd, 64
+ .protected K3
+K3.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad K3 at rel64-K3.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11468864
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .LK3.num_vgpr, 2
+ .set .LK3.num_agpr, 0
+ .set .LK3.numbered_sgpr, 10
+ .set .LK3.num_named_barrier, 0
+ .set .LK3.private_seg_size, 0
+ .set .LK3.uses_vcc, 0
+ .set .LK3.uses_flat_scratch, 0
+ .set .LK3.has_dyn_sized_stack, 0
+ .set .LK3.has_recursion, 0
+ .set .LK3.has_indirect_call, 0
+ .amdgpu_info K1
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 10
+ .amdgpu_private_segment_size 0
+ .amdgpu_use C
+ .amdgpu_use B
+ .amdgpu_use A
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info K2
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 10
+ .amdgpu_private_segment_size 0
+ .amdgpu_use C
+ .amdgpu_use B
+ .amdgpu_use A
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info K3
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 2
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 10
+ .amdgpu_private_segment_size 0
+ .amdgpu_use A
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 0
+ .set amdgpu.max_num_named_barrier, 0
+ .globl A
+ .amdgpu_lds A, 8, 8
+ .globl B
+ .amdgpu_lds B, 16, 16
+ .globl C
+ .amdgpu_lds C, 32, 16
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: K1
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: K1.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: K2
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: K2.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: K3
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: K3.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-lds-link-time-ordering-sizes.s b/lld/test/ELF/amdgpu-lds-link-time-ordering-sizes.s
new file mode 100644
index 0000000000000..b0d45f8529fdd
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-ordering-sizes.s
@@ -0,0 +1,209 @@
+# REQUIRES: amdgpu
+
+## Test size-based tie-breaking when shared-tier variables have equal use
+## counts. The per-kernel frontier allocator breaks ties by placing larger
+## variables first.
+##
+## Variables (all external linkage -> global-scope -> shared tier):
+## big: 64 bytes align 4->16*, used by K1, K2 (2 users)
+## small: 8 bytes align 4, used by K1, K2 (2 users)
+## (* superAlignLDSGlobals bumps alignment to 16 for vars >= 16 bytes)
+##
+## Both have use_count=2. The tie breaks by size, so big is placed first.
+##
+## Layout: big(64,a16)@0x00, small(8,a8)@0x40
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/tu1.s -o %t/tu1.o
+# RUN: ld.lld %t/tu1.o -o %t/out
+# RUN: llvm-readelf -s %t/out | FileCheck %s --check-prefix=SYM
+
+## big is first, then small at the shared kernel frontier
+# SYM-DAG: 0000000000000000 {{.*}} big
+# SYM-DAG: 0000000000000040 {{.*}} small
+
+#--- tu1.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl K1 ; -- Begin function K1
+ .p2align 8
+ .type K1, at function
+K1:
+ s_endpgm
+.Lfunc_end0:
+ .size K1, .Lfunc_end0-K1
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl K1.kd
+ .type K1.kd, at object
+ .size K1.kd, 64
+ .protected K1
+K1.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad K1 at rel64-K1.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11468864
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .LK1.num_vgpr, 4
+ .set .LK1.num_agpr, 0
+ .set .LK1.numbered_sgpr, 10
+ .set .LK1.num_named_barrier, 0
+ .set .LK1.private_seg_size, 0
+ .set .LK1.uses_vcc, 0
+ .set .LK1.uses_flat_scratch, 0
+ .set .LK1.has_dyn_sized_stack, 0
+ .set .LK1.has_recursion, 0
+ .set .LK1.has_indirect_call, 0
+ .text
+ .globl K2 ; -- Begin function K2
+ .p2align 8
+ .type K2, at function
+K2:
+ s_endpgm
+.Lfunc_end1:
+ .size K2, .Lfunc_end1-K2
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl K2.kd
+ .type K2.kd, at object
+ .size K2.kd, 64
+ .protected K2
+K2.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad K2 at rel64-K2.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11468864
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .LK2.num_vgpr, 4
+ .set .LK2.num_agpr, 0
+ .set .LK2.numbered_sgpr, 10
+ .set .LK2.num_named_barrier, 0
+ .set .LK2.private_seg_size, 0
+ .set .LK2.uses_vcc, 0
+ .set .LK2.uses_flat_scratch, 0
+ .set .LK2.has_dyn_sized_stack, 0
+ .set .LK2.has_recursion, 0
+ .set .LK2.has_indirect_call, 0
+ .amdgpu_info K1
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 10
+ .amdgpu_private_segment_size 0
+ .amdgpu_use small
+ .amdgpu_use big
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info K2
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 10
+ .amdgpu_private_segment_size 0
+ .amdgpu_use small
+ .amdgpu_use big
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 0
+ .set amdgpu.max_num_named_barrier, 0
+ .globl big
+ .amdgpu_lds big, 64, 16
+ .globl small
+ .amdgpu_lds small, 8, 8
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: K1
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: K1.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: K2
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: K2.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-lds-link-time-ordering-tiers.s b/lld/test/ELF/amdgpu-lds-link-time-ordering-tiers.s
new file mode 100644
index 0000000000000..965b1813a5f81
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-ordering-tiers.s
@@ -0,0 +1,313 @@
+# REQUIRES: amdgpu
+
+## Test that shared-tier ordering is optimized while kernel-tier variables
+## are correctly appended after the shared tier.
+##
+## Variables:
+## shared_a: 8 bytes align 4, external, used by K1, K2, K3 (shared, 3 users)
+## shared_b: 32 bytes align 4->16*, external, used by K1, K2 (shared, 2 users)
+## k1_priv: 16 bytes align 4->16*, internal, used by K1 (kernel tier)
+## (* superAlignLDSGlobals bumps alignment to 16 for vars >= 16 bytes)
+##
+## Greedy shared tier from tail:
+## Position 2: use_count shared_a=3, shared_b=2. Min=2 -> shared_b.
+## K1,K2 fixed. shared_a->1.
+## Position 1: shared_a left.
+## Result: [shared_a, shared_b]
+##
+## Layout:
+## Shared: shared_a(8,a4)@0x00, shared_b(32,a16)@0x10
+## Kernel: __amdgpu_lds.K1(16,a16)@0x30
+##
+## Per-kernel sizes:
+## K1: shared_a(0..8), shared_b(16..48), K1's struct(48..64) -> 64 = 0x40
+## K2: shared_a(0..8), shared_b(16..48) -> 48 = 0x30
+## K3: shared_a(0..8) -> 8 = 0x08 (zero waste!)
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/tu1.s -o %t/tu1.o
+# RUN: ld.lld %t/tu1.o -o %t/out
+# RUN: llvm-readelf -s %t/out | FileCheck %s --check-prefix=SYM
+# RUN: llvm-readobj --notes %t/out | FileCheck %s --check-prefix=META
+
+## Shared tier: shared_a first (most shared), shared_b after
+# SYM-DAG: 0000000000000000 {{.*}} shared_a
+# SYM-DAG: 0000000000000010 {{.*}} shared_b
+## Kernel tier appended after shared tier
+# SYM-DAG: 0000000000000030 {{.*}} __amdgpu_lds.K1
+
+## K3 (uses only shared_a) has minimal size
+# META-DAG: .group_segment_fixed_size: 8
+## K2 (uses shared_a + shared_b)
+# META-DAG: .group_segment_fixed_size: 48
+## K1 (uses shared_a + shared_b + k1_priv)
+# META-DAG: .group_segment_fixed_size: 64
+
+#--- tu1.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl K1 ; -- Begin function K1
+ .p2align 8
+ .type K1, at function
+K1:
+ s_endpgm
+.Lfunc_end0:
+ .size K1, .Lfunc_end0-K1
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl K1.kd
+ .type K1.kd, at object
+ .size K1.kd, 64
+ .protected K1
+K1.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad K1 at rel64-K1.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11468864
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .LK1.num_vgpr, 4
+ .set .LK1.num_agpr, 0
+ .set .LK1.numbered_sgpr, 10
+ .set .LK1.num_named_barrier, 0
+ .set .LK1.private_seg_size, 0
+ .set .LK1.uses_vcc, 0
+ .set .LK1.uses_flat_scratch, 0
+ .set .LK1.has_dyn_sized_stack, 0
+ .set .LK1.has_recursion, 0
+ .set .LK1.has_indirect_call, 0
+ .text
+ .globl K2 ; -- Begin function K2
+ .p2align 8
+ .type K2, at function
+K2:
+ s_endpgm
+.Lfunc_end1:
+ .size K2, .Lfunc_end1-K2
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl K2.kd
+ .type K2.kd, at object
+ .size K2.kd, 64
+ .protected K2
+K2.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad K2 at rel64-K2.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11468864
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .LK2.num_vgpr, 4
+ .set .LK2.num_agpr, 0
+ .set .LK2.numbered_sgpr, 10
+ .set .LK2.num_named_barrier, 0
+ .set .LK2.private_seg_size, 0
+ .set .LK2.uses_vcc, 0
+ .set .LK2.uses_flat_scratch, 0
+ .set .LK2.has_dyn_sized_stack, 0
+ .set .LK2.has_recursion, 0
+ .set .LK2.has_indirect_call, 0
+ .text
+ .globl K3 ; -- Begin function K3
+ .p2align 8
+ .type K3, at function
+K3:
+ s_endpgm
+.Lfunc_end2:
+ .size K3, .Lfunc_end2-K3
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl K3.kd
+ .type K3.kd, at object
+ .size K3.kd, 64
+ .protected K3
+K3.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad K3 at rel64-K3.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11468864
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .LK3.num_vgpr, 2
+ .set .LK3.num_agpr, 0
+ .set .LK3.numbered_sgpr, 10
+ .set .LK3.num_named_barrier, 0
+ .set .LK3.private_seg_size, 0
+ .set .LK3.uses_vcc, 0
+ .set .LK3.uses_flat_scratch, 0
+ .set .LK3.has_dyn_sized_stack, 0
+ .set .LK3.has_recursion, 0
+ .set .LK3.has_indirect_call, 0
+ .amdgpu_info K1
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 10
+ .amdgpu_private_segment_size 0
+ .amdgpu_use __amdgpu_lds.K1
+ .amdgpu_use shared_b
+ .amdgpu_use shared_a
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info K2
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 10
+ .amdgpu_private_segment_size 0
+ .amdgpu_use shared_b
+ .amdgpu_use shared_a
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info K3
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 2
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 10
+ .amdgpu_private_segment_size 0
+ .amdgpu_use shared_a
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 0
+ .set amdgpu.max_num_named_barrier, 0
+ .globl shared_a
+ .amdgpu_lds shared_a, 8, 8
+ .globl shared_b
+ .amdgpu_lds shared_b, 32, 16
+ .globl __amdgpu_lds.K1
+ .amdgpu_lds __amdgpu_lds.K1, 16, 16
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: K1
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: K1.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: K2
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: K2.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: K3
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: K3.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-lds-link-time-padding.s b/lld/test/ELF/amdgpu-lds-link-time-padding.s
new file mode 100644
index 0000000000000..e39f9833ab396
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-padding.s
@@ -0,0 +1,116 @@
+# REQUIRES: amdgpu
+
+## Test that the linker inserts padding between LDS symbols when a symbol's
+## size is not a multiple of the next symbol's alignment requirement.
+##
+## Symbols:
+## lds_big align=16 size=20 — leaves offset at 20, not 8-aligned
+## lds_med align=8 size=7 — needs padding to offset 24, leaves at 31
+## lds_small align=4 size=5 — needs padding to offset 32, leaves at 37
+## lds_tiny align=1 size=1 — no padding needed, offset 37
+##
+## Expected layout (alignment desc, size desc):
+## lds_big -> offset 0x00 (0)
+## lds_med -> offset 0x18 (24 = alignTo(20, 8), 4 bytes padding)
+## lds_small -> offset 0x20 (32 = alignTo(31, 4), 1 byte padding)
+## lds_tiny -> offset 0x25 (37 = alignTo(37, 1), no padding)
+##
+## Total: 38 bytes. Padding: 4 bytes at [20,24) + 1 byte at [31,32).
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa -mcpu=gfx900 %t/a.s -o %t/a.o
+# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa -mcpu=gfx900 %t/b.s -o %t/b.o
+# RUN: ld.lld %t/a.o %t/b.o -o %t/out
+# RUN: llvm-readobj --syms %t/out | FileCheck %s
+
+## Symbols appear in file-definition order (a.o then b.o).
+
+## From a.o: lds_tiny (align=1, size=1) -> offset 0x25 (no padding from lds_small)
+# CHECK: Name: lds_tiny
+# CHECK-NEXT: Value: 0x25
+# CHECK-NEXT: Size: 1
+
+## From a.o: lds_big (align=16, size=20) -> offset 0x0
+# CHECK: Name: lds_big
+# CHECK-NEXT: Value: 0x0
+# CHECK-NEXT: Size: 20
+
+## From b.o: lds_small (align=4, size=5) -> offset 0x20 (1 byte padding from 31)
+# CHECK: Name: lds_small
+# CHECK-NEXT: Value: 0x20
+# CHECK-NEXT: Size: 5
+
+## From b.o: lds_med (align=8, size=7) -> offset 0x18 (4 bytes padding from 20)
+# CHECK: Name: lds_med
+# CHECK-NEXT: Value: 0x18
+# CHECK-NEXT: Size: 7
+
+#--- a.s
+ .text
+ .globl f1
+ .p2align 8
+ .type f1, at function
+f1:
+ s_mov_b32 s0, lds_tiny at abs32@lo
+ s_mov_b32 s1, lds_big at abs32@lo
+ s_endpgm
+.Lf1_end:
+ .size f1, .Lf1_end-f1
+
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl f1.kd
+ .type f1.kd, at object
+ .size f1.kd, 64
+f1.kd:
+ .zero 64
+
+ .text
+ .amdgpu_info f1
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_sgpr 4
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .globl lds_tiny
+ .amdgpu_lds lds_tiny, 1, 1
+
+ .globl lds_big
+ .amdgpu_lds lds_big, 20, 16
+
+#--- b.s
+ .text
+ .globl f2
+ .p2align 8
+ .type f2, at function
+f2:
+ s_mov_b32 s0, lds_small at abs32@lo
+ s_mov_b32 s1, lds_med at abs32@lo
+ s_endpgm
+.Lf2_end:
+ .size f2, .Lf2_end-f2
+
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl f2.kd
+ .type f2.kd, at object
+ .size f2.kd, 64
+f2.kd:
+ .zero 64
+
+ .text
+ .amdgpu_info f2
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_sgpr 4
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .globl lds_small
+ .amdgpu_lds lds_small, 5, 4
+
+ .globl lds_med
+ .amdgpu_lds lds_med, 7, 8
diff --git a/lld/test/ELF/amdgpu-lds-link-time-per-kernel-sizes.s b/lld/test/ELF/amdgpu-lds-link-time-per-kernel-sizes.s
new file mode 100644
index 0000000000000..41ffc1628b40c
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-per-kernel-sizes.s
@@ -0,0 +1,240 @@
+# REQUIRES: amdgpu
+
+## Test that the linker computes LDS offsets and per-kernel LDS sizes for
+## disjoint kernel groups. This is the focused test for patching each kernel's
+## group_segment_fixed_size from the resolved LDS layout.
+##
+## TU1: kernel_a uses lds_a (256 bytes, align 16)
+## TU2: kernel_b uses lds_b (128 bytes, align 4)
+##
+## These are independent groups, so each allocates from offset 0:
+## Group 1: lds_a at offset 0 -> kernel_a size = 0x100
+## Group 2: lds_b at offset 0 -> kernel_b size = 0x080
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/tu1.s -o %t/tu1.o
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/tu2.s -o %t/tu2.o
+
+## Link.
+# RUN: ld.lld %t/tu1.o %t/tu2.o -o %t/out
+
+## Verify independent groups both allocate from offset 0.
+# RUN: llvm-readelf -s %t/out | FileCheck %s --check-prefix=SYM
+
+## Verify per-kernel LDS sizes via kernel descriptor patching and HSA metadata.
+# RUN: llvm-readobj --notes %t/out | FileCheck %s --check-prefix=META
+
+# SYM-DAG: 0000000000000000 {{.*}} lds_a
+# SYM-DAG: 0000000000000000 {{.*}} lds_b
+
+## kernel_a: lds_a at offset 0, size 256 -> group_segment_fixed_size = 256
+# META: .group_segment_fixed_size: 256
+# META: .name: kernel_a
+
+## kernel_b: lds_b at offset 0, size 128 -> group_segment_fixed_size = 128
+# META: .group_segment_fixed_size: 128
+# META: .name: kernel_b
+
+#--- tu1.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl kernel_a ; -- Begin function kernel_a
+ .p2align 8
+ .type kernel_a, at function
+kernel_a:
+ s_endpgm
+.Lfunc_end0:
+ .size kernel_a, .Lfunc_end0-kernel_a
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl kernel_a.kd
+ .type kernel_a.kd, at object
+ .size kernel_a.kd, 64
+ .protected kernel_a
+kernel_a.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad kernel_a at rel64-kernel_a.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11468864
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .Lkernel_a.num_vgpr, 2
+ .set .Lkernel_a.num_agpr, 0
+ .set .Lkernel_a.numbered_sgpr, 10
+ .set .Lkernel_a.num_named_barrier, 0
+ .set .Lkernel_a.private_seg_size, 0
+ .set .Lkernel_a.uses_vcc, 0
+ .set .Lkernel_a.uses_flat_scratch, 0
+ .set .Lkernel_a.has_dyn_sized_stack, 0
+ .set .Lkernel_a.has_recursion, 0
+ .set .Lkernel_a.has_indirect_call, 0
+ .amdgpu_info kernel_a
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 2
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 10
+ .amdgpu_private_segment_size 0
+ .amdgpu_use lds_a
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 0
+ .set amdgpu.max_num_named_barrier, 0
+ .globl lds_a
+ .amdgpu_lds lds_a, 256, 16
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: kernel_a
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: kernel_a.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- tu2.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl kernel_b ; -- Begin function kernel_b
+ .p2align 8
+ .type kernel_b, at function
+kernel_b:
+ s_endpgm
+.Lfunc_end0:
+ .size kernel_b, .Lfunc_end0-kernel_b
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl kernel_b.kd
+ .type kernel_b.kd, at object
+ .size kernel_b.kd, 64
+ .protected kernel_b
+kernel_b.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad kernel_b at rel64-kernel_b.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11468864
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .Lkernel_b.num_vgpr, 2
+ .set .Lkernel_b.num_agpr, 0
+ .set .Lkernel_b.numbered_sgpr, 10
+ .set .Lkernel_b.num_named_barrier, 0
+ .set .Lkernel_b.private_seg_size, 0
+ .set .Lkernel_b.uses_vcc, 0
+ .set .Lkernel_b.uses_flat_scratch, 0
+ .set .Lkernel_b.has_dyn_sized_stack, 0
+ .set .Lkernel_b.has_recursion, 0
+ .set .Lkernel_b.has_indirect_call, 0
+ .amdgpu_info kernel_b
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 2
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 10
+ .amdgpu_private_segment_size 0
+ .amdgpu_use lds_b
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 0
+ .set amdgpu.max_num_named_barrier, 0
+ .globl lds_b
+ .amdgpu_lds lds_b, 128, 16
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: kernel_b
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: kernel_b.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-lds-link-time-random-layout.s b/lld/test/ELF/amdgpu-lds-link-time-random-layout.s
new file mode 100644
index 0000000000000..bfc9cdf8ac46b
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-random-layout.s
@@ -0,0 +1,432 @@
+# REQUIRES: amdgpu
+
+## Generated deterministic random LDS/kernel layouts for the per-kernel
+## frontier allocator. The generator was run ahead of time; lit does not
+## execute any script.
+
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %s -o %t.o
+# RUN: ld.lld %t.o -o %t
+# RUN: llvm-readelf -s %t | FileCheck %s --check-prefix=SYM
+
+## Case 0: 2 kernels, 4 LDS symbols, allocation order c0_lds1, c0_lds3, c0_lds2, c0_lds0
+## c0_lds0: size=20, align=4, users={c0_k1}, offset=0x1c
+# SYM-DAG: 000000000000001c {{.*}} c0_lds0
+## c0_lds1: size=24, align=4, users={c0_k0, c0_k1}, offset=0x0
+# SYM-DAG: 0000000000000000 {{.*}} c0_lds1
+## c0_lds2: size=4, align=8, users={c0_k0}, offset=0x20
+# SYM-DAG: 0000000000000020 {{.*}} c0_lds2
+## c0_lds3: size=4, align=4, users={c0_k0, c0_k1}, offset=0x18
+# SYM-DAG: 0000000000000018 {{.*}} c0_lds3
+## Case 1: 6 kernels, 6 LDS symbols, allocation order c1_lds2, c1_lds5, c1_lds1, c1_lds4, c1_lds3, c1_lds0
+## c1_lds0: size=4, align=8, users={c1_k1, c1_k4}, offset=0x70
+# SYM-DAG: 0000000000000070 {{.*}} c1_lds0
+## c1_lds1: size=32, align=4, users={c1_k0, c1_k2, c1_k3, c1_k4}, offset=0x38
+# SYM-DAG: 0000000000000038 {{.*}} c1_lds1
+## c1_lds2: size=32, align=16, users={c1_k0, c1_k1, c1_k2, c1_k3, c1_k4, c1_k5}, offset=0x0
+# SYM-DAG: 0000000000000000 {{.*}} c1_lds2
+## c1_lds3: size=8, align=4, users={c1_k2, c1_k4, c1_k5}, offset=0x68
+# SYM-DAG: 0000000000000068 {{.*}} c1_lds3
+## c1_lds4: size=16, align=8, users={c1_k1, c1_k2, c1_k3}, offset=0x58
+# SYM-DAG: 0000000000000058 {{.*}} c1_lds4
+## c1_lds5: size=24, align=4, users={c1_k0, c1_k1, c1_k2, c1_k3, c1_k4, c1_k5}, offset=0x20
+# SYM-DAG: 0000000000000020 {{.*}} c1_lds5
+## Case 2: 2 kernels, 4 LDS symbols, allocation order c2_lds1, c2_lds2, c2_lds0, c2_lds3
+## c2_lds0: size=16, align=4, users={c2_k0, c2_k1}, offset=0x1c
+# SYM-DAG: 000000000000001c {{.*}} c2_lds0
+## c2_lds1: size=12, align=8, users={c2_k0, c2_k1}, offset=0x0
+# SYM-DAG: 0000000000000000 {{.*}} c2_lds1
+## c2_lds2: size=12, align=8, users={c2_k0, c2_k1}, offset=0x10
+# SYM-DAG: 0000000000000010 {{.*}} c2_lds2
+## c2_lds3: size=4, align=4, users={c2_k0, c2_k1}, offset=0x2c
+# SYM-DAG: 000000000000002c {{.*}} c2_lds3
+## Case 3: 2 kernels, 5 LDS symbols, allocation order c3_lds1, c3_lds3, c3_lds2, c3_lds4, c3_lds0
+## c3_lds0: size=12, align=4, users={c3_k0}, offset=0x90
+# SYM-DAG: 0000000000000090 {{.*}} c3_lds0
+## c3_lds1: size=32, align=16, users={c3_k0, c3_k1}, offset=0x0
+# SYM-DAG: 0000000000000000 {{.*}} c3_lds1
+## c3_lds2: size=32, align=4, users={c3_k0, c3_k1}, offset=0x50
+# SYM-DAG: 0000000000000050 {{.*}} c3_lds2
+## c3_lds3: size=48, align=8, users={c3_k0, c3_k1}, offset=0x20
+# SYM-DAG: 0000000000000020 {{.*}} c3_lds3
+## c3_lds4: size=32, align=4, users={c3_k0, c3_k1}, offset=0x70
+# SYM-DAG: 0000000000000070 {{.*}} c3_lds4
+## Case 4: 2 kernels, 3 LDS symbols, allocation order c4_lds2, c4_lds0, c4_lds1
+## c4_lds0: size=48, align=16, users={c4_k0}, offset=0x20
+# SYM-DAG: 0000000000000020 {{.*}} c4_lds0
+## c4_lds1: size=24, align=8, users={c4_k1}, offset=0x18
+# SYM-DAG: 0000000000000018 {{.*}} c4_lds1
+## c4_lds2: size=20, align=8, users={c4_k0, c4_k1}, offset=0x0
+# SYM-DAG: 0000000000000000 {{.*}} c4_lds2
+## Case 5: 6 kernels, 6 LDS symbols, allocation order c5_lds5, c5_lds3, c5_lds4, c5_lds0, c5_lds1, c5_lds2
+## c5_lds0: size=12, align=8, users={c5_k0, c5_k1, c5_k2, c5_k4}, offset=0x50
+# SYM-DAG: 0000000000000050 {{.*}} c5_lds0
+## c5_lds1: size=8, align=8, users={c5_k0, c5_k3, c5_k4}, offset=0x60
+# SYM-DAG: 0000000000000060 {{.*}} c5_lds1
+## c5_lds2: size=4, align=8, users={c5_k5}, offset=0x30
+# SYM-DAG: 0000000000000030 {{.*}} c5_lds2
+## c5_lds3: size=32, align=16, users={c5_k0, c5_k3, c5_k4, c5_k5}, offset=0x10
+# SYM-DAG: 0000000000000010 {{.*}} c5_lds3
+## c5_lds4: size=32, align=8, users={c5_k1, c5_k2, c5_k3, c5_k4}, offset=0x30
+# SYM-DAG: 0000000000000030 {{.*}} c5_lds4
+## c5_lds5: size=12, align=4, users={c5_k0, c5_k1, c5_k2, c5_k3, c5_k4, c5_k5}, offset=0x0
+# SYM-DAG: 0000000000000000 {{.*}} c5_lds5
+
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+
+ .macro kernel name
+ .text
+ .globl \name
+ .p2align 8
+ .type \name, at function
+\name:
+ s_endpgm
+.L\name\()_end:
+ .size \name, .L\name\()_end-\name
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl \name\().kd
+ .type \name\().kd, at object
+ .size \name\().kd, 64
+ .protected \name
+\name\().kd:
+ .long 0
+ .long 0
+ .long 0
+ .long 0
+ .quad \name at rel64-\name\().kd
+ .zero 20
+ .long 0
+ .long 0
+ .long 0
+ .short 0
+ .short 0
+ .long 0
+ .endm
+
+## Case 0
+ kernel c0_k0
+ kernel c0_k1
+
+ .amdgpu_info c0_k0
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use c0_lds1
+ .amdgpu_use c0_lds2
+ .amdgpu_use c0_lds3
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info c0_k1
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use c0_lds0
+ .amdgpu_use c0_lds1
+ .amdgpu_use c0_lds3
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .globl c0_lds0
+ .amdgpu_lds c0_lds0, 20, 4
+ .globl c0_lds1
+ .amdgpu_lds c0_lds1, 24, 4
+ .globl c0_lds2
+ .amdgpu_lds c0_lds2, 4, 8
+ .globl c0_lds3
+ .amdgpu_lds c0_lds3, 4, 4
+
+## Case 1
+ kernel c1_k0
+ kernel c1_k1
+ kernel c1_k2
+ kernel c1_k3
+ kernel c1_k4
+ kernel c1_k5
+
+ .amdgpu_info c1_k0
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use c1_lds1
+ .amdgpu_use c1_lds2
+ .amdgpu_use c1_lds5
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info c1_k1
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use c1_lds0
+ .amdgpu_use c1_lds2
+ .amdgpu_use c1_lds4
+ .amdgpu_use c1_lds5
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info c1_k2
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use c1_lds1
+ .amdgpu_use c1_lds2
+ .amdgpu_use c1_lds3
+ .amdgpu_use c1_lds4
+ .amdgpu_use c1_lds5
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info c1_k3
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use c1_lds1
+ .amdgpu_use c1_lds2
+ .amdgpu_use c1_lds4
+ .amdgpu_use c1_lds5
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info c1_k4
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use c1_lds0
+ .amdgpu_use c1_lds1
+ .amdgpu_use c1_lds2
+ .amdgpu_use c1_lds3
+ .amdgpu_use c1_lds5
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info c1_k5
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use c1_lds2
+ .amdgpu_use c1_lds3
+ .amdgpu_use c1_lds5
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .globl c1_lds0
+ .amdgpu_lds c1_lds0, 4, 8
+ .globl c1_lds1
+ .amdgpu_lds c1_lds1, 32, 4
+ .globl c1_lds2
+ .amdgpu_lds c1_lds2, 32, 16
+ .globl c1_lds3
+ .amdgpu_lds c1_lds3, 8, 4
+ .globl c1_lds4
+ .amdgpu_lds c1_lds4, 16, 8
+ .globl c1_lds5
+ .amdgpu_lds c1_lds5, 24, 4
+
+## Case 2
+ kernel c2_k0
+ kernel c2_k1
+
+ .amdgpu_info c2_k0
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use c2_lds0
+ .amdgpu_use c2_lds1
+ .amdgpu_use c2_lds2
+ .amdgpu_use c2_lds3
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info c2_k1
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use c2_lds0
+ .amdgpu_use c2_lds1
+ .amdgpu_use c2_lds2
+ .amdgpu_use c2_lds3
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .globl c2_lds0
+ .amdgpu_lds c2_lds0, 16, 4
+ .globl c2_lds1
+ .amdgpu_lds c2_lds1, 12, 8
+ .globl c2_lds2
+ .amdgpu_lds c2_lds2, 12, 8
+ .globl c2_lds3
+ .amdgpu_lds c2_lds3, 4, 4
+
+## Case 3
+ kernel c3_k0
+ kernel c3_k1
+
+ .amdgpu_info c3_k0
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use c3_lds0
+ .amdgpu_use c3_lds1
+ .amdgpu_use c3_lds2
+ .amdgpu_use c3_lds3
+ .amdgpu_use c3_lds4
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info c3_k1
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use c3_lds1
+ .amdgpu_use c3_lds2
+ .amdgpu_use c3_lds3
+ .amdgpu_use c3_lds4
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .globl c3_lds0
+ .amdgpu_lds c3_lds0, 12, 4
+ .globl c3_lds1
+ .amdgpu_lds c3_lds1, 32, 16
+ .globl c3_lds2
+ .amdgpu_lds c3_lds2, 32, 4
+ .globl c3_lds3
+ .amdgpu_lds c3_lds3, 48, 8
+ .globl c3_lds4
+ .amdgpu_lds c3_lds4, 32, 4
+
+## Case 4
+ kernel c4_k0
+ kernel c4_k1
+
+ .amdgpu_info c4_k0
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use c4_lds0
+ .amdgpu_use c4_lds2
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info c4_k1
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use c4_lds1
+ .amdgpu_use c4_lds2
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .globl c4_lds0
+ .amdgpu_lds c4_lds0, 48, 16
+ .globl c4_lds1
+ .amdgpu_lds c4_lds1, 24, 8
+ .globl c4_lds2
+ .amdgpu_lds c4_lds2, 20, 8
+
+## Case 5
+ kernel c5_k0
+ kernel c5_k1
+ kernel c5_k2
+ kernel c5_k3
+ kernel c5_k4
+ kernel c5_k5
+
+ .amdgpu_info c5_k0
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use c5_lds0
+ .amdgpu_use c5_lds1
+ .amdgpu_use c5_lds3
+ .amdgpu_use c5_lds5
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info c5_k1
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use c5_lds0
+ .amdgpu_use c5_lds4
+ .amdgpu_use c5_lds5
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info c5_k2
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use c5_lds0
+ .amdgpu_use c5_lds4
+ .amdgpu_use c5_lds5
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info c5_k3
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use c5_lds1
+ .amdgpu_use c5_lds3
+ .amdgpu_use c5_lds4
+ .amdgpu_use c5_lds5
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info c5_k4
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use c5_lds0
+ .amdgpu_use c5_lds1
+ .amdgpu_use c5_lds3
+ .amdgpu_use c5_lds4
+ .amdgpu_use c5_lds5
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info c5_k5
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use c5_lds2
+ .amdgpu_use c5_lds3
+ .amdgpu_use c5_lds5
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .globl c5_lds0
+ .amdgpu_lds c5_lds0, 12, 8
+ .globl c5_lds1
+ .amdgpu_lds c5_lds1, 8, 8
+ .globl c5_lds2
+ .amdgpu_lds c5_lds2, 4, 8
+ .globl c5_lds3
+ .amdgpu_lds c5_lds3, 32, 16
+ .globl c5_lds4
+ .amdgpu_lds c5_lds4, 32, 8
+ .globl c5_lds5
+ .amdgpu_lds c5_lds5, 12, 4
diff --git a/lld/test/ELF/amdgpu-lds-link-time-recursive-scc.s b/lld/test/ELF/amdgpu-lds-link-time-recursive-scc.s
new file mode 100644
index 0000000000000..487e79b7d074c
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-recursive-scc.s
@@ -0,0 +1,161 @@
+# REQUIRES: amdgpu
+
+## Test LDS reachability through a recursive SCC. The kernel reaches lds_data
+## only through the H/A/M cycle, where A is the direct LDS user.
+
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %s -o %t.o
+# RUN: ld.lld %t.o -o %t
+# RUN: llvm-readobj --notes %t | FileCheck %s
+
+# CHECK: .group_segment_fixed_size: 128
+# CHECK: .name: kernel
+# CHECK: .uses_dynamic_stack: true
+
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+
+ .text
+ .globl kernel
+ .p2align 8
+ .type kernel, at function
+kernel:
+ s_endpgm
+.Lkernel_end:
+ .size kernel, .Lkernel_end-kernel
+
+ .globl H
+ .p2align 6
+ .type H, at function
+H:
+ s_setpc_b64 s[30:31]
+.LH_end:
+ .size H, .LH_end-H
+
+ .globl A
+ .p2align 6
+ .type A, at function
+A:
+ s_setpc_b64 s[30:31]
+.LA_end:
+ .size A, .LA_end-A
+
+ .globl M
+ .p2align 6
+ .type M, at function
+M:
+ s_setpc_b64 s[30:31]
+.LM_end:
+ .size M, .LM_end-M
+
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl kernel.kd
+ .type kernel.kd, at object
+ .size kernel.kd, 64
+ .protected kernel
+kernel.kd:
+ .long 0
+ .long 0
+ .long 256
+ .long 0
+ .quad kernel at rel64-kernel.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11469063
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+
+ .text
+ .amdgpu_info kernel
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 10
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 0
+ .amdgpu_call H
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info H
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 10
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 0
+ .amdgpu_call A
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info A
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 10
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 0
+ .amdgpu_use lds_data
+ .amdgpu_call M
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info M
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 10
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 0
+ .amdgpu_call H
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 10
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 32
+ .set amdgpu.max_num_named_barrier, 0
+ .globl lds_data
+ .amdgpu_lds lds_data, 128, 16
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 256
+ .max_flat_workgroup_size: 1024
+ .name: kernel
+ .private_segment_fixed_size: 0
+ .sgpr_count: 32
+ .sgpr_spill_count: 0
+ .symbol: kernel.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 10
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-lds-link-time-symbol-resolution.s b/lld/test/ELF/amdgpu-lds-link-time-symbol-resolution.s
new file mode 100644
index 0000000000000..54f5be147e6c9
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time-symbol-resolution.s
@@ -0,0 +1,169 @@
+# REQUIRES: amdgpu
+
+## Verify that raw SHN_AMDGPU_LDS rescanning preserves normal common-symbol
+## resolution semantics.
+
+# RUN: split-file %s %t
+
+## Multiple LDS declarations merge their maximum size and alignment. A common
+## LDS declaration also overrides a weak regular definition.
+# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa -mcpu=gfx900 %t/dup-a.s -o %t/dup-a.o
+# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa -mcpu=gfx900 %t/dup-b.s -o %t/dup-b.o
+# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa -mcpu=gfx900 %t/weak.s -o %t/weak.o
+# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa -mcpu=gfx900 %t/dup-kernel.s -o %t/dup-kernel.o
+# RUN: ld.lld %t/weak.o %t/dup-a.o %t/dup-b.o %t/dup-kernel.o -o %t/dup
+# RUN: llvm-readobj --symbols %t/dup | FileCheck %s --check-prefix=MERGED
+
+# MERGED: Name: weak_lds
+# MERGED-NEXT: Value: 0x30
+# MERGED-NEXT: Size: 16
+# MERGED: Section: Absolute
+# MERGED: Name: merged
+# MERGED-NEXT: Value: 0x0
+# MERGED-NEXT: Size: 32
+# MERGED: Section: Absolute
+# MERGED: Name: prefix
+# MERGED-NEXT: Value: 0x20
+# MERGED-NEXT: Size: 16
+# MERGED: Section: Absolute
+
+## A strong regular definition overrides an LDS common declaration.
+# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa -mcpu=gfx900 %t/strong-lds.s -o %t/strong-lds.o
+# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa -mcpu=gfx900 %t/strong-def.s -o %t/strong-def.o
+# RUN: ld.lld %t/strong-lds.o %t/strong-def.o -o %t/strong
+# RUN: llvm-readobj --symbols %t/strong | FileCheck %s --check-prefix=STRONG
+
+# STRONG: Name: strong_lds
+# STRONG: Size: 4
+# STRONG: Section: .data
+
+## A regular common and an LDS common cannot describe the same symbol.
+# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa -mcpu=gfx900 %t/mixed-lds.s -o %t/mixed-lds.o
+# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa -mcpu=gfx900 %t/mixed-common.s -o %t/mixed-common.o
+# RUN: not ld.lld %t/mixed-lds.o %t/mixed-common.o -o /dev/null 2>&1 | FileCheck %s --check-prefix=MIXED
+
+# MIXED: error: symbol 'mixed' is defined as both an AMDGPU LDS symbol and a regular common symbol
+
+## --fortran-common must not extract an archive member whose definition is
+## another common-like SHN_AMDGPU_LDS symbol.
+# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa -mcpu=gfx900 %t/archive-main.s -o %t/archive-main.o
+# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa -mcpu=gfx900 %t/archive-member.s -o %t/archive-member.o
+# RUN: llvm-ar crs %t/liblds.a %t/archive-member.o
+# RUN: ld.lld --fortran-common %t/archive-main.o %t/liblds.a -o %t/archive
+# RUN: llvm-readobj --symbols %t/archive | FileCheck %s --check-prefix=ARCHIVE --implicit-check-not=member_marker
+
+# ARCHIVE: Name: archive_lds
+# ARCHIVE-NEXT: Value: 0x0
+# ARCHIVE-NEXT: Size: 8
+# ARCHIVE: Section: Absolute
+
+#--- dup-a.s
+ .globl merged
+ .amdgpu_lds merged, 32, 4
+
+ .globl prefix
+ .amdgpu_lds prefix, 16, 16
+
+ .globl weak_lds
+ .amdgpu_lds weak_lds, 16, 8
+
+#--- dup-b.s
+ .globl merged
+ .amdgpu_lds merged, 8, 64
+
+#--- weak.s
+ .data
+ .weak weak_lds
+ .type weak_lds, at object
+weak_lds:
+ .long 0
+ .size weak_lds, .-weak_lds
+
+#--- dup-kernel.s
+ .text
+ .globl dup_kernel
+ .p2align 8
+ .type dup_kernel, at function
+dup_kernel:
+ s_endpgm
+ .size dup_kernel, .-dup_kernel
+
+ .amdgpu_info dup_kernel
+ .amdgpu_flags 3
+ .amdgpu_num_vgpr 32
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 33
+ .amdgpu_private_segment_size 0
+ .amdgpu_use merged
+ .amdgpu_use prefix
+ .amdgpu_use weak_lds
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .rodata,"a", at progbits
+ .p2align 6
+ .globl dup_kernel.kd
+ .type dup_kernel.kd, at object
+ .size dup_kernel.kd, 64
+dup_kernel.kd:
+ .zero 64
+
+#--- strong-lds.s
+ .globl strong_lds
+ .amdgpu_lds strong_lds, 64, 16
+
+#--- strong-def.s
+ .data
+ .globl strong_lds
+ .type strong_lds, at object
+strong_lds:
+ .long 0
+ .size strong_lds, .-strong_lds
+
+#--- mixed-lds.s
+ .globl mixed
+ .amdgpu_lds mixed, 16, 16
+
+#--- mixed-common.s
+ .comm mixed, 8, 4
+
+#--- archive-main.s
+ .globl archive_lds
+ .amdgpu_lds archive_lds, 8, 4
+
+ .text
+ .globl archive_kernel
+ .p2align 8
+ .type archive_kernel, at function
+archive_kernel:
+ s_endpgm
+ .size archive_kernel, .-archive_kernel
+
+ .amdgpu_info archive_kernel
+ .amdgpu_flags 3
+ .amdgpu_num_vgpr 32
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 33
+ .amdgpu_private_segment_size 0
+ .amdgpu_use archive_lds
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .rodata,"a", at progbits
+ .p2align 6
+ .globl archive_kernel.kd
+ .type archive_kernel.kd, at object
+ .size archive_kernel.kd, 64
+archive_kernel.kd:
+ .zero 64
+
+#--- archive-member.s
+ .globl archive_lds
+ .amdgpu_lds archive_lds, 64, 64
+
+ .text
+ .globl member_marker
+ .type member_marker, at function
+member_marker:
+ s_endpgm
+ .size member_marker, .-member_marker
diff --git a/lld/test/ELF/amdgpu-lds-link-time.s b/lld/test/ELF/amdgpu-lds-link-time.s
new file mode 100644
index 0000000000000..a98c1ec621b3c
--- /dev/null
+++ b/lld/test/ELF/amdgpu-lds-link-time.s
@@ -0,0 +1,134 @@
+# REQUIRES: amdgpu
+
+## Test that lld resolves SHN_AMDGPU_LDS symbols to absolute symbols with
+## assigned offsets, and patches R_AMDGPU_ABS32_LO relocations in code.
+
+## Build two object files with LDS symbols and one kernel object that uses both
+## symbols.
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa -mcpu=gfx900 %t/a.s -o %t/a.o
+# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa -mcpu=gfx900 %t/b.s -o %t/b.o
+# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa -mcpu=gfx900 %t/kernel.s -o %t/kernel.o
+
+## Verify that the object files have SHN_AMDGPU_LDS symbols.
+# RUN: llvm-readobj --syms %t/a.o | FileCheck %s --check-prefix=OBJ-A
+# RUN: llvm-readobj --syms %t/b.o | FileCheck %s --check-prefix=OBJ-B
+
+## A final link cannot assign LDS without a kernel descriptor.
+# RUN: not ld.lld %t/a.o %t/b.o -o %t/no-kernel 2>&1 | FileCheck %s --check-prefix=NO-KERNEL
+
+## Link and verify the symbols are resolved to absolute with correct offsets.
+# RUN: ld.lld %t/a.o %t/b.o %t/kernel.o -o %t/out
+# RUN: llvm-readobj --syms %t/out | FileCheck %s --check-prefix=LINKED
+# RUN: llvm-objdump -d %t/out | FileCheck %s --check-prefix=DISASM
+
+# NO-KERNEL: error: cannot resolve AMDGPU LDS symbols without a kernel descriptor and .amdgpu.info metadata
+# NO-KERNEL-NOT: common symbol reached writer
+# NO-KERNEL-NOT: UNREACHABLE
+
+# OBJ-A: Symbol {
+# OBJ-A: Name: lds_a
+# OBJ-A-NEXT: Value: 0x10
+# OBJ-A-NEXT: Size: 256
+# OBJ-A-NEXT: Binding: Global
+# OBJ-A-NEXT: Type: Object
+# OBJ-A-NEXT: Other: 0
+# OBJ-A-NEXT: Section: Processor Specific (0xFF00)
+# OBJ-A-NEXT: }
+
+# OBJ-B: Symbol {
+# OBJ-B: Name: lds_b
+# OBJ-B-NEXT: Value: 0x4
+# OBJ-B-NEXT: Size: 128
+# OBJ-B-NEXT: Binding: Global
+# OBJ-B-NEXT: Type: Object
+# OBJ-B-NEXT: Other: 0
+# OBJ-B-NEXT: Section: Processor Specific (0xFF00)
+# OBJ-B-NEXT: }
+
+## After linking, LDS symbols become absolute with assigned offsets.
+## lds_a: align=16, size=256 -> offset 0
+## lds_b: align=4, size=128 -> offset 256
+
+# LINKED: Symbol {
+# LINKED: Name: lds_a
+# LINKED-NEXT: Value: 0x0
+# LINKED-NEXT: Size: 256
+# LINKED-NEXT: Binding: Global
+# LINKED-NEXT: Type: None
+# LINKED: Section: Absolute
+# LINKED-NEXT: }
+
+# LINKED: Symbol {
+# LINKED: Name: lds_b
+# LINKED-NEXT: Value: 0x100
+# LINKED-NEXT: Size: 128
+# LINKED-NEXT: Binding: Global
+# LINKED-NEXT: Type: None
+# LINKED: Section: Absolute
+# LINKED-NEXT: }
+
+## The s_mov_b32 instructions should be patched with the resolved offsets.
+# DISASM: s_mov_b32 s0, lit(0x0)
+# DISASM: s_mov_b32 s0, 0x100
+
+#--- a.s
+ .text
+ .globl use_lds_a
+ .p2align 8
+ .type use_lds_a, at function
+use_lds_a:
+ s_mov_b32 s0, lds_a at abs32@lo
+ v_lshl_add_u32 v1, v0, 2, s0
+ ds_read_b32 v2, v1
+ s_endpgm
+.Lfunc_end_a:
+ .size use_lds_a, .Lfunc_end_a-use_lds_a
+
+ .globl lds_a
+ .amdgpu_lds lds_a, 256, 16
+
+#--- b.s
+ .text
+ .globl use_lds_b
+ .p2align 8
+ .type use_lds_b, at function
+use_lds_b:
+ s_mov_b32 s0, lds_b at abs32@lo
+ v_lshl_add_u32 v1, v0, 2, s0
+ ds_write_b32 v1, v2
+ s_endpgm
+.Lfunc_end_b:
+ .size use_lds_b, .Lfunc_end_b-use_lds_b
+
+ .globl lds_b
+ .amdgpu_lds lds_b, 128, 4
+
+#--- kernel.s
+ .text
+ .globl test_kernel
+ .p2align 8
+ .type test_kernel, at function
+test_kernel:
+ s_endpgm
+.Lfunc_end_kernel:
+ .size test_kernel, .Lfunc_end_kernel-test_kernel
+
+ .amdgpu_info test_kernel
+ .amdgpu_flags 3
+ .amdgpu_num_vgpr 32
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 33
+ .amdgpu_private_segment_size 0
+ .amdgpu_use lds_a
+ .amdgpu_use lds_b
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .rodata,"a", at progbits
+ .p2align 6
+ .globl test_kernel.kd
+ .type test_kernel.kd, at object
+ .size test_kernel.kd, 64
+test_kernel.kd:
+ .zero 64
diff --git a/lld/test/ELF/amdgpu-named-barrier-cross-tu.s b/lld/test/ELF/amdgpu-named-barrier-cross-tu.s
new file mode 100644
index 0000000000000..7a94d51e51b90
--- /dev/null
+++ b/lld/test/ELF/amdgpu-named-barrier-cross-tu.s
@@ -0,0 +1,193 @@
+# REQUIRES: amdgpu
+
+## Cross-TU named barrier collision test.
+##
+## Without link-time resolution, each TU would independently assign barrier
+## ID 1 to its local named barrier. When a kernel reaches both barriers
+## transitively, this causes a collision. The linker must assign distinct
+## barrier IDs to avoid this.
+##
+## TU A: kern uses bar_a (1 slot), calls helper
+## TU B: helper uses bar_b (1 slot)
+## After linking, kern reaches both bar_a and bar_b. The linker assigns
+## distinct IDs (e.g., bar_a=1, bar_b=2 or vice versa). NAMED_BAR_CNT=1
+## since ceil(2/4)=1.
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx1250 -filetype=obj %t/a.s -o %t/a.o
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx1250 -filetype=obj %t/b.s -o %t/b.o
+# RUN: ld.lld %t/a.o %t/b.o -o %t/out
+
+## Verify the resolved barrier symbols have distinct encoded addresses.
+## Each address is 0x802000 | (scope << 9) | (barId << 4).
+## bar_a and bar_b must have different barIds.
+# RUN: llvm-nm --format=posix %t/out | FileCheck %s --check-prefix=SYMS
+
+## Verify NAMED_BAR_CNT is correct in compute_pgm_rsrc3.
+## NAMED_BAR_CNT=1 is encoded as 0x00004000, printed below as little-endian
+## bytes 00400000.
+# RUN: llvm-objdump -s -j .rodata %t/out | FileCheck %s --check-prefix=KD
+
+## The two barrier symbols must have different addresses (distinct barrier IDs).
+## Both should match the barrier encoding pattern: 0x802010 (ID=1) or 0x802020 (ID=2).
+# SYMS-DAG: __amdgpu_named_barrier.bar_a{{[^ ]*}} A {{[0-9a-f]+}}
+# SYMS-DAG: __amdgpu_named_barrier.bar_b{{[^ ]*}} A {{[0-9a-f]+}}
+
+# KD: {{[0-9a-f]+}} 00000000 00000000 00000000 00400000
+
+#--- a.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx1250"
+ .amdhsa_code_object_version 6
+ .text
+ .globl kern ; -- Begin function kern
+ .p2align 8
+ .type kern, at function
+kern:
+ s_endpgm
+.Lfunc_end0:
+ .size kern, .Lfunc_end0-kern
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl kern.kd
+ .type kern.kd, at object
+ .size kern.kd, 64
+ .protected kern
+kern.kd:
+ .long 0
+ .long 0
+ .long 256
+ .long 0
+ .quad kern at rel64-kern.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 3222208513
+ .long 5008
+ .short 1054
+ .short 0
+ .long 0
+ .text
+ .set .Lkern.num_vgpr, 32
+ .set .Lkern.num_agpr, 0
+ .set .Lkern.numbered_sgpr, 33
+ .set .Lkern.num_named_barrier, 0
+ .set .Lkern.private_seg_size, 0
+ .set .Lkern.uses_vcc, 1
+ .set .Lkern.uses_flat_scratch, 0
+ .set .Lkern.has_dyn_sized_stack, 0
+ .set .Lkern.has_recursion, 0
+ .set .Lkern.has_indirect_call, 0
+ .text
+ .amdgpu_info kern
+ .amdgpu_flags 17
+ .amdgpu_num_vgpr 32
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 33
+ .amdgpu_private_segment_size 0
+ .amdgpu_use __amdgpu_named_barrier.bar_a.a8fb08c539c29f35a63da55cd03bd885
+ .amdgpu_call helper
+ .amdgpu_occupancy 8
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 0
+ .set amdgpu.max_num_named_barrier, 0
+ .text
+ .globl __amdgpu_named_barrier.bar_a.a8fb08c539c29f35a63da55cd03bd885
+ .amdgpu_lds __amdgpu_named_barrier.bar_a.a8fb08c539c29f35a63da55cd03bd885, 16, 4
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 256
+ .max_flat_workgroup_size: 1024
+ .name: kern
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: kern.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 32
+amdhsa.target: amdgcn-amd-amdhsa--gfx1250
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- b.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx1250"
+ .amdhsa_code_object_version 6
+ .text
+ .globl helper ; -- Begin function helper
+ .p2align 7
+ .type helper, at function
+helper:
+ s_setpc_b64 s[30:31]
+.Lfunc_end0:
+ .size helper, .Lfunc_end0-helper
+ .set .Lhelper.num_vgpr, 0
+ .set .Lhelper.num_agpr, 0
+ .set .Lhelper.numbered_sgpr, 32
+ .set .Lhelper.num_named_barrier, 0
+ .set .Lhelper.private_seg_size, 0
+ .set .Lhelper.uses_vcc, 0
+ .set .Lhelper.uses_flat_scratch, 0
+ .set .Lhelper.has_dyn_sized_stack, 0
+ .set .Lhelper.has_recursion, 0
+ .set .Lhelper.has_indirect_call, 0
+ .text
+ .amdgpu_info helper
+ .amdgpu_flags 16
+ .amdgpu_num_vgpr 0
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 0
+ .amdgpu_use __amdgpu_named_barrier.bar_b.957946e05970693deaaae396388e1c50
+ .amdgpu_occupancy 8
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 32
+ .set amdgpu.max_num_named_barrier, 0
+ .text
+ .globl __amdgpu_named_barrier.bar_b.957946e05970693deaaae396388e1c50
+ .amdgpu_lds __amdgpu_named_barrier.bar_b.957946e05970693deaaae396388e1c50, 16, 4
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels: []
+amdhsa.target: amdgcn-amd-amdhsa--gfx1250
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-named-barrier-grouped.s b/lld/test/ELF/amdgpu-named-barrier-grouped.s
new file mode 100644
index 0000000000000..f531009c104fe
--- /dev/null
+++ b/lld/test/ELF/amdgpu-named-barrier-grouped.s
@@ -0,0 +1,109 @@
+# REQUIRES: amdgpu
+
+## Test per-kernel named-barrier frontiers. A and B are allocated first because
+## they have the most users. C can still reuse B's IDs because no kernel reaches
+## both B and C, even though A and B connect all three kernels into one old-style
+## barrier group.
+##
+## K1: A, C
+## K2: A, B
+## K3: B, D
+##
+## With connected-component allocation, C would be assigned after A and B, so K1
+## would require 6 named barriers. The frontier allocator places C at ID 3, so
+## K1 only requires 4 named barriers and NAMED_BAR_CNT stays 1.
+
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx1250 -filetype=obj %s -o %t.o
+# RUN: ld.lld %t.o -o %t
+# RUN: llvm-nm --format=posix %t | FileCheck %s --check-prefix=SYM
+# RUN: llvm-objdump -s -j .rodata %t | FileCheck %s --check-prefix=KD
+
+# SYM-DAG: __amdgpu_named_barrier.a A {{0*}}802010
+# SYM-DAG: __amdgpu_named_barrier.b A {{0*}}802030
+# SYM-DAG: __amdgpu_named_barrier.c A {{0*}}802030
+# SYM-DAG: __amdgpu_named_barrier.d A {{0*}}802050
+
+## K1 and K2 use IDs 1..4, so NAMED_BAR_CNT=1. K3 uses IDs 3..6, so
+## NAMED_BAR_CNT=2.
+# KD: {{[0-9a-f]+}} 00000000 00000000 00000000 00400000
+# KD: {{[0-9a-f]+}} 00000000 00000000 00000000 00400000
+# KD: {{[0-9a-f]+}} 00000000 00000000 00000000 00800000
+
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx1250"
+ .amdhsa_code_object_version 6
+
+ .macro kernel name
+ .text
+ .globl \name
+ .p2align 8
+ .type \name, at function
+\name:
+ s_endpgm
+.L\name\()_end:
+ .size \name, .L\name\()_end-\name
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl \name\().kd
+ .type \name\().kd, at object
+ .size \name\().kd, 64
+ .protected \name
+\name\().kd:
+ .long 0
+ .long 0
+ .long 0
+ .long 0
+ .quad \name at rel64-\name\().kd
+ .zero 20
+ .long 0
+ .long 0
+ .long 0
+ .short 1024
+ .short 0
+ .long 0
+ .endm
+
+ kernel K1
+ kernel K2
+ kernel K3
+
+ .amdgpu_info K1
+ .amdgpu_flags 16
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use __amdgpu_named_barrier.a
+ .amdgpu_use __amdgpu_named_barrier.c
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info K2
+ .amdgpu_flags 16
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use __amdgpu_named_barrier.a
+ .amdgpu_use __amdgpu_named_barrier.b
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info K3
+ .amdgpu_flags 16
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_use __amdgpu_named_barrier.b
+ .amdgpu_use __amdgpu_named_barrier.d
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .globl __amdgpu_named_barrier.a
+ .amdgpu_lds __amdgpu_named_barrier.a, 32, 4
+ .globl __amdgpu_named_barrier.b
+ .amdgpu_lds __amdgpu_named_barrier.b, 32, 4
+ .globl __amdgpu_named_barrier.c
+ .amdgpu_lds __amdgpu_named_barrier.c, 32, 4
+ .globl __amdgpu_named_barrier.d
+ .amdgpu_lds __amdgpu_named_barrier.d, 32, 4
diff --git a/lld/test/ELF/amdgpu-object-linking-asm-roundtrip.s b/lld/test/ELF/amdgpu-object-linking-asm-roundtrip.s
new file mode 100644
index 0000000000000..62a7b4b91d446
--- /dev/null
+++ b/lld/test/ELF/amdgpu-object-linking-asm-roundtrip.s
@@ -0,0 +1,145 @@
+# REQUIRES: amdgpu
+
+## Test that the assembly form of AMDGPU object-linking metadata can be
+## assembled by llvm-mc and consumed by ld.lld. This covers the save-temps
+## style path without depending on compiler code generation in an lld test.
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/a.s -o %t/a.o
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/b.s -o %t/b.o
+# RUN: ld.lld %t/a.o %t/b.o -o %t/out
+# RUN: llvm-readobj --notes %t/out | FileCheck %s
+
+# CHECK: .name: kernel
+# CHECK: .sgpr_count:
+# CHECK: .vgpr_count:
+
+#--- a.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl kernel
+ .p2align 8
+ .type kernel, at function
+kernel:
+ s_endpgm
+.Lkernel_end:
+ .size kernel, .Lkernel_end-kernel
+
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl kernel.kd
+ .type kernel.kd, at object
+ .size kernel.kd, 64
+ .protected kernel
+kernel.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad kernel at rel64-kernel.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11469063
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+
+ .text
+ .amdgpu_info kernel
+ .amdgpu_flags 3
+ .amdgpu_num_vgpr 32
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 33
+ .amdgpu_private_segment_size 0
+ .amdgpu_call helper
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 0
+ .set amdgpu.max_num_named_barrier, 0
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: kernel
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .sgpr_spill_count: 0
+ .symbol: kernel.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- b.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl helper
+ .p2align 6
+ .type helper, at function
+helper:
+ s_setpc_b64 s[30:31]
+.Lhelper_end:
+ .size helper, .Lhelper_end-helper
+
+ .amdgpu_info helper
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 8
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 8
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 32
+ .set amdgpu.max_num_named_barrier, 0
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels: []
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-object-linking-eflags.s b/lld/test/ELF/amdgpu-object-linking-eflags.s
new file mode 100644
index 0000000000000..56987280ac2e2
--- /dev/null
+++ b/lld/test/ELF/amdgpu-object-linking-eflags.s
@@ -0,0 +1,220 @@
+# REQUIRES: amdgpu
+
+## Test that the linker rejects objects with incompatible AMDGPU e_flags before
+## AMDGPU object linking.
+
+# RUN: split-file %s %t
+
+## --- Incompatible GPU architecture ---
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/func-gfx900.s -o %t/func-gfx900.o
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx1030 -filetype=obj %t/func-gfx1030.s -o %t/func-gfx1030.o
+# RUN: not ld.lld %t/func-gfx900.o %t/func-gfx1030.o -o /dev/null 2>&1 | FileCheck %s --check-prefix=ARCH
+
+# ARCH: error: incompatible mach:
+
+## --- Incompatible xnack ---
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack -filetype=obj %t/func-xnack-on.s -o %t/func-xnack-on.o
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=-xnack -filetype=obj %t/func-xnack-off.s -o %t/func-xnack-off.o
+# RUN: not ld.lld %t/func-xnack-on.o %t/func-xnack-off.o -o /dev/null 2>&1 | FileCheck %s --check-prefix=XNACK
+
+# XNACK: error: incompatible xnack:
+
+## --- Incompatible sramecc ---
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx908 -mattr=+sramecc -filetype=obj %t/func-sramecc-on.s -o %t/func-sramecc-on.o
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx908 -mattr=-sramecc -filetype=obj %t/func-sramecc-off.s -o %t/func-sramecc-off.o
+# RUN: not ld.lld %t/func-sramecc-on.o %t/func-sramecc-off.o -o /dev/null 2>&1 | FileCheck %s --check-prefix=SRAMECC
+# RUN: ld.lld %t/func-sramecc-on.o -o %t/sramecc-on
+
+# SRAMECC: error: incompatible sramecc:
+
+#--- func-gfx900.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl func_a
+ .p2align 2
+ .type func_a, at function
+func_a:
+ s_setpc_b64 s[30:31]
+.Lfunc_end_a:
+ .size func_a, .Lfunc_end_a-func_a
+
+ .amdgpu_info func_a
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_sgpr 4
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 8
+ .end_amdgpu_info
+
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels: []
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- func-gfx1030.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx1030"
+ .amdhsa_code_object_version 6
+ .text
+ .globl func_b
+ .p2align 2
+ .type func_b, at function
+func_b:
+ s_setpc_b64 s[30:31]
+.Lfunc_end_b:
+ .size func_b, .Lfunc_end_b-func_b
+
+ .amdgpu_info func_b
+ .amdgpu_flags 16
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_sgpr 4
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 8
+ .end_amdgpu_info
+
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels: []
+amdhsa.target: amdgcn-amd-amdhsa--gfx1030
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- func-xnack-on.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900:xnack+"
+ .amdhsa_code_object_version 6
+ .text
+ .globl func_c
+ .p2align 2
+ .type func_c, at function
+func_c:
+ s_setpc_b64 s[30:31]
+.Lfunc_end_c:
+ .size func_c, .Lfunc_end_c-func_c
+
+ .amdgpu_info func_c
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_sgpr 4
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 8
+ .end_amdgpu_info
+
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels: []
+amdhsa.target: amdgcn-amd-amdhsa--gfx900:xnack+
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- func-xnack-off.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900:xnack-"
+ .amdhsa_code_object_version 6
+ .text
+ .globl func_d
+ .p2align 2
+ .type func_d, at function
+func_d:
+ s_setpc_b64 s[30:31]
+.Lfunc_end_d:
+ .size func_d, .Lfunc_end_d-func_d
+
+ .amdgpu_info func_d
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_sgpr 4
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 8
+ .end_amdgpu_info
+
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels: []
+amdhsa.target: amdgcn-amd-amdhsa--gfx900:xnack-
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- func-sramecc-on.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx908:sramecc+"
+ .amdhsa_code_object_version 6
+ .text
+ .globl func_e
+ .p2align 2
+ .type func_e, at function
+func_e:
+ s_setpc_b64 s[30:31]
+.Lfunc_end_e:
+ .size func_e, .Lfunc_end_e-func_e
+
+ .amdgpu_info func_e
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_sgpr 4
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 8
+ .end_amdgpu_info
+
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels: []
+amdhsa.target: amdgcn-amd-amdhsa--gfx908:sramecc+
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- func-sramecc-off.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx908:sramecc-"
+ .amdhsa_code_object_version 6
+ .text
+ .globl func_f
+ .p2align 2
+ .type func_f, at function
+func_f:
+ s_setpc_b64 s[30:31]
+.Lfunc_end_f:
+ .size func_f, .Lfunc_end_f-func_f
+
+ .amdgpu_info func_f
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_sgpr 4
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 8
+ .end_amdgpu_info
+
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels: []
+amdhsa.target: amdgcn-amd-amdhsa--gfx908:sramecc-
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-object-linking-lto.ll b/lld/test/ELF/amdgpu-object-linking-lto.ll
new file mode 100644
index 0000000000000..68b2c5ab1c351
--- /dev/null
+++ b/lld/test/ELF/amdgpu-object-linking-lto.ll
@@ -0,0 +1,31 @@
+; REQUIRES: amdgpu
+
+; Verify that the target symbol-finalization hook sees objects produced by LTO
+; and resolves their SHN_AMDGPU_LDS symbols.
+
+; RUN: llvm-as < %s -o %t.bc
+; RUN: ld.lld --save-temps -shared -mllvm -amdgpu-enable-object-linking \
+; RUN: -mllvm -mcpu=gfx900 %t.bc -o %t
+; RUN: llvm-readobj --symbols %t.lto.o | FileCheck %s --check-prefix=OBJECT
+; RUN: llvm-readobj --symbols %t | FileCheck %s --check-prefix=LINKED
+
+; OBJECT: Name: lds
+; OBJECT-NEXT: Value: 0x4
+; OBJECT-NEXT: Size: 4
+; OBJECT: Section: Processor Specific (0xFF00)
+
+; LINKED: Name: lds
+; LINKED-NEXT: Value: 0x0
+; LINKED-NEXT: Size: 4
+; LINKED: Section: Absolute
+
+target triple = "amdgcn-amd-amdhsa"
+target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+ at lds = external addrspace(3) global i32, align 4
+
+define amdgpu_kernel void @kernel() {
+entry:
+ store volatile i32 0, ptr addrspace(3) @lds, align 4
+ ret void
+}
diff --git a/lld/test/ELF/amdgpu-object-linking-malformed-strtab.s b/lld/test/ELF/amdgpu-object-linking-malformed-strtab.s
new file mode 100644
index 0000000000000..6919609052a77
--- /dev/null
+++ b/lld/test/ELF/amdgpu-object-linking-malformed-strtab.s
@@ -0,0 +1,209 @@
+# REQUIRES: amdgpu
+
+## The type-id string in .amdgpu.strtab is intentionally not NUL-terminated.
+## The linker should bound the read to the section contents and still match the
+## indirect call to the address-taken function.
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/a.s -o %t/a.o
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/b.s -o %t/b.o
+# RUN: ld.lld %t/a.o %t/b.o -o %t/out
+# RUN: llvm-readobj --notes %t/out | FileCheck %s
+
+# CHECK: .group_segment_fixed_size: 128
+# CHECK: .name: kernel
+
+#--- a.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl target_func
+ .p2align 6
+ .type target_func, at function
+target_func:
+ s_setpc_b64 s[30:31]
+.Ltarget_func_end:
+ .size target_func, .Ltarget_func_end-target_func
+
+ .globl lds_var
+ .amdgpu_lds lds_var, 128, 16
+
+ .section .amdgpu.info,"e", at progbits
+ .byte 1
+ .byte 8
+ .quad target_func
+ .byte 2
+ .byte 4
+ .long 0
+ .byte 4
+ .byte 4
+ .long 4
+ .byte 3
+ .byte 4
+ .long 4
+ .byte 6
+ .byte 4
+ .long 0
+ .byte 11
+ .byte 4
+ .long 8
+ .byte 12
+ .byte 4
+ .long 64
+ .byte 7
+ .byte 8
+ .quad lds_var
+ .byte 10
+ .byte 4
+ .long 0
+
+ .section .amdgpu.strtab,"e", at progbits
+ .ascii "vi"
+
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels: []
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- b.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl caller
+ .p2align 6
+ .type caller, at function
+caller:
+ s_setpc_b64 s[30:31]
+.Lcaller_end:
+ .size caller, .Lcaller_end-caller
+
+ .globl kernel
+ .p2align 8
+ .type kernel, at function
+kernel:
+ s_endpgm
+.Lkernel_end:
+ .size kernel, .Lkernel_end-kernel
+
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl kernel.kd
+ .type kernel.kd, at object
+ .size kernel.kd, 64
+ .protected kernel
+kernel.kd:
+ .long 0
+ .long 0
+ .long 0
+ .long 0
+ .long 0
+ .quad kernel at rel64-kernel.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 0
+ .long 0
+ .short 0
+ .short 0
+ .long 0
+
+ .section .amdgpu.info,"e", at progbits
+ .byte 1
+ .byte 8
+ .quad caller
+ .byte 2
+ .byte 4
+ .long 0
+ .byte 4
+ .byte 4
+ .long 4
+ .byte 3
+ .byte 4
+ .long 4
+ .byte 6
+ .byte 4
+ .long 0
+ .byte 11
+ .byte 4
+ .long 8
+ .byte 12
+ .byte 4
+ .long 64
+ .byte 9
+ .byte 4
+ .long 0
+ .byte 1
+ .byte 8
+ .quad kernel
+ .byte 2
+ .byte 4
+ .long 0
+ .byte 4
+ .byte 4
+ .long 4
+ .byte 3
+ .byte 4
+ .long 4
+ .byte 6
+ .byte 4
+ .long 0
+ .byte 11
+ .byte 4
+ .long 8
+ .byte 12
+ .byte 4
+ .long 64
+ .byte 8
+ .byte 8
+ .quad caller
+
+ .section .amdgpu.strtab,"e", at progbits
+ .ascii "vi"
+
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 0
+ .max_flat_workgroup_size: 1024
+ .name: kernel
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .symbol: kernel.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-object-linking-wave-size.s b/lld/test/ELF/amdgpu-object-linking-wave-size.s
new file mode 100644
index 0000000000000..d64cb05fa0c3e
--- /dev/null
+++ b/lld/test/ELF/amdgpu-object-linking-wave-size.s
@@ -0,0 +1,317 @@
+# REQUIRES: amdgpu
+
+## Test that the linker rejects cross-TU calls between functions compiled with
+## different wavefront sizes.
+
+# RUN: split-file %s %t
+
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx1030 -filetype=obj %t/a.s -o %t/a.o
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx1030 -filetype=obj %t/b.s -o %t/b.o
+# RUN: not ld.lld %t/a.o %t/b.o -o /dev/null 2>&1 | FileCheck %s
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx1030 -filetype=obj %t/wgp-direct.s -o %t/wgp-direct.o
+# RUN: not ld.lld %t/wgp-direct.o -o /dev/null 2>&1 | FileCheck %s --check-prefix=MODE-DIRECT
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx1030 -filetype=obj %t/wgp-indirect.s -o %t/wgp-indirect.o
+# RUN: not ld.lld %t/wgp-indirect.o -o /dev/null 2>&1 | FileCheck %s --check-prefix=MODE-INDIRECT
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx1250 -filetype=obj %t/gfx1250-wave64.s -o %t/gfx1250-wave64.o
+# RUN: not ld.lld %t/gfx1250-wave64.o -o /dev/null 2>&1 | FileCheck %s --check-prefix=GFX1250-W64
+
+# CHECK: error: AMDGPU object linking: wave size mismatch in call from 'kernel' (wave32) to 'helper' (wave64)
+# MODE-DIRECT: error: AMDGPU object linking: CU/WGP mode mismatch in call from 'wgp_caller' (WGP) to 'cu_callee' (CU)
+# MODE-INDIRECT: error: AMDGPU object linking: CU/WGP mode mismatch in call from 'wgp_indirect_caller' (WGP) to 'cu_addr_taker' (CU)
+# GFX1250-W64: error: AMDGPU: function 'kernel_wave64' uses unsupported ABI wave64 for gfx1250
+
+#--- a.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx1030"
+ .amdhsa_code_object_version 6
+ .text
+ .globl kernel
+ .p2align 8
+ .type kernel, at function
+kernel:
+ s_endpgm
+.Lfunc_end0:
+ .size kernel, .Lfunc_end0-kernel
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl kernel.kd
+ .type kernel.kd, at object
+ .size kernel.kd, 64
+ .protected kernel
+kernel.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .long 0
+ .quad kernel at rel64-kernel.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11469063
+ .long 5020
+ .short 1063
+ .short 0
+ .long 0
+ .text
+ .amdgpu_info kernel
+ .amdgpu_flags 19
+ .amdgpu_num_vgpr 32
+ .amdgpu_num_sgpr 33
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 4
+ .amdgpu_call helper
+ .end_amdgpu_info
+
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: kernel
+ .private_segment_fixed_size: 0
+ .sgpr_count: 39
+ .symbol: kernel.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 32
+ .wavefront_size: 32
+amdhsa.target: amdgcn-amd-amdhsa--gfx1030
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- gfx1250-wave64.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx1250"
+ .amdhsa_code_object_version 6
+ .text
+ .globl kernel_wave64
+ .p2align 8
+ .type kernel_wave64, at function
+kernel_wave64:
+ s_endpgm
+.Lkernel_wave64_end:
+ .size kernel_wave64, .Lkernel_wave64_end-kernel_wave64
+
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl kernel_wave64.kd
+ .type kernel_wave64.kd, at object
+ .size kernel_wave64.kd, 64
+ .protected kernel_wave64
+kernel_wave64.kd:
+ .long 0
+ .long 0
+ .long 0
+ .long 0
+ .quad kernel_wave64 at rel64-kernel_wave64.kd
+ .zero 20
+ .long 0
+ .long 0
+ .long 0
+ .short 1024
+ .short 0
+ .long 0
+
+ .text
+ .amdgpu_info kernel_wave64
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 1
+ .amdgpu_num_sgpr 1
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 0
+ .max_flat_workgroup_size: 1024
+ .name: kernel_wave64
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .symbol: kernel_wave64.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .wavefront_size: 32
+amdhsa.target: amdgcn-amd-amdhsa--gfx1250
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- wgp-direct.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx1030"
+ .amdhsa_code_object_version 6
+ .text
+ .globl wgp_caller
+ .p2align 2
+ .type wgp_caller, at function
+wgp_caller:
+ s_setpc_b64 s[30:31]
+.Lwgp_caller_end:
+ .size wgp_caller, .Lwgp_caller_end-wgp_caller
+
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl wgp_caller.kd
+ .type wgp_caller.kd, at object
+ .size wgp_caller.kd, 64
+wgp_caller.kd:
+ .zero 64
+
+ .text
+ .globl cu_callee
+ .p2align 2
+ .type cu_callee, at function
+cu_callee:
+ s_setpc_b64 s[30:31]
+.Lcu_callee_end:
+ .size cu_callee, .Lcu_callee_end-cu_callee
+
+ .amdgpu_info wgp_caller
+ .amdgpu_flags 8
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_sgpr 4
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 4
+ .amdgpu_call cu_callee
+ .end_amdgpu_info
+
+ .amdgpu_info cu_callee
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_sgpr 4
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels: []
+amdhsa.target: amdgcn-amd-amdhsa--gfx1030
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- wgp-indirect.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx1030"
+ .amdhsa_code_object_version 6
+ .text
+ .globl wgp_indirect_caller
+ .p2align 2
+ .type wgp_indirect_caller, at function
+wgp_indirect_caller:
+ s_setpc_b64 s[30:31]
+.Lwgp_indirect_caller_end:
+ .size wgp_indirect_caller, .Lwgp_indirect_caller_end-wgp_indirect_caller
+
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl wgp_indirect_caller.kd
+ .type wgp_indirect_caller.kd, at object
+ .size wgp_indirect_caller.kd, 64
+wgp_indirect_caller.kd:
+ .zero 64
+
+ .text
+ .globl cu_addr_taker
+ .p2align 2
+ .type cu_addr_taker, at function
+cu_addr_taker:
+ s_setpc_b64 s[30:31]
+.Lcu_addr_taker_end:
+ .size cu_addr_taker, .Lcu_addr_taker_end-cu_addr_taker
+
+ .amdgpu_info wgp_indirect_caller
+ .amdgpu_flags 8
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_sgpr 4
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 4
+ .amdgpu_indirect_call "v"
+ .end_amdgpu_info
+
+ .amdgpu_info cu_addr_taker
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_sgpr 4
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 4
+ .amdgpu_typeid "v"
+ .end_amdgpu_info
+
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels: []
+amdhsa.target: amdgcn-amd-amdhsa--gfx1030
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- b.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx1030"
+ .amdhsa_code_object_version 6
+ .text
+ .globl helper
+ .p2align 2
+ .type helper, at function
+helper:
+ s_setpc_b64 s[30:31]
+.Lfunc_end1:
+ .size helper, .Lfunc_end1-helper
+
+ .amdgpu_info helper
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 10
+ .amdgpu_num_sgpr 8
+ .amdgpu_private_segment_size 16
+ .amdgpu_occupancy 8
+ .end_amdgpu_info
+
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels: []
+amdhsa.target: amdgcn-amd-amdhsa--gfx1030
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-resource-usage-alias.s b/lld/test/ELF/amdgpu-resource-usage-alias.s
new file mode 100644
index 0000000000000..34d452c191e94
--- /dev/null
+++ b/lld/test/ELF/amdgpu-resource-usage-alias.s
@@ -0,0 +1,173 @@
+# REQUIRES: amdgpu
+
+## Test that the linker resolves function aliases during resource propagation.
+## In the C++ Itanium ABI, C1 (complete object) constructors are often aliases
+## to C2 (base object) constructors. The compiler emits a resource entry
+## only for C2; C1 appears as a stub without has_local_res.
+## Without alias resolution, the linker would report "incomplete resource usage".
+##
+## Call graph: kernel -> _ZN3FooC1Ev (alias) -> _ZN3FooC2Ev (real definition)
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/a.s -o %t/a.o
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/b.s -o %t/b.o
+# RUN: ld.lld %t/a.o %t/b.o -o %t/out
+
+## Verify the linker succeeds and patches HSA metadata.
+# RUN: llvm-readobj --notes %t/out | FileCheck %s
+
+# CHECK: .name: test_kernel
+# CHECK: .sgpr_count:
+# CHECK: .vgpr_count:
+
+#--- a.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl test_kernel ; -- Begin function test_kernel
+ .p2align 8
+ .type test_kernel, at function
+test_kernel:
+ s_endpgm
+.Lfunc_end0:
+ .size test_kernel, .Lfunc_end0-test_kernel
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl test_kernel.kd
+ .type test_kernel.kd, at object
+ .size test_kernel.kd, 64
+ .protected test_kernel
+test_kernel.kd:
+ .long 0
+ .long 0
+ .long 256
+ .long 0
+ .quad test_kernel at rel64-test_kernel.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11469063
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .Ltest_kernel.num_vgpr, 32
+ .set .Ltest_kernel.num_agpr, 0
+ .set .Ltest_kernel.numbered_sgpr, 33
+ .set .Ltest_kernel.num_named_barrier, 0
+ .set .Ltest_kernel.private_seg_size, 0
+ .set .Ltest_kernel.uses_vcc, 1
+ .set .Ltest_kernel.uses_flat_scratch, 1
+ .set .Ltest_kernel.has_dyn_sized_stack, 0
+ .set .Ltest_kernel.has_recursion, 0
+ .set .Ltest_kernel.has_indirect_call, 0
+ .amdgpu_info test_kernel
+ .amdgpu_flags 3
+ .amdgpu_num_vgpr 32
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 33
+ .amdgpu_private_segment_size 0
+ .amdgpu_call _ZN3FooC1Ev
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 0
+ .set amdgpu.max_num_named_barrier, 0
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 256
+ .max_flat_workgroup_size: 1024
+ .name: test_kernel
+ .private_segment_fixed_size: 0
+ .sgpr_count: 39
+ .sgpr_spill_count: 0
+ .symbol: test_kernel.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 32
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- b.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl _ZN3FooC2Ev ; -- Begin function _ZN3FooC2Ev
+ .p2align 6
+ .type _ZN3FooC2Ev, at function
+_ZN3FooC2Ev:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end0:
+ .size _ZN3FooC2Ev, .Lfunc_end0-_ZN3FooC2Ev
+ .set .L_ZN3FooC2Ev.num_vgpr, 8
+ .set .L_ZN3FooC2Ev.num_agpr, 0
+ .set .L_ZN3FooC2Ev.numbered_sgpr, 32
+ .set .L_ZN3FooC2Ev.num_named_barrier, 0
+ .set .L_ZN3FooC2Ev.private_seg_size, 0
+ .set .L_ZN3FooC2Ev.uses_vcc, 0
+ .set .L_ZN3FooC2Ev.uses_flat_scratch, 0
+ .set .L_ZN3FooC2Ev.has_dyn_sized_stack, 0
+ .set .L_ZN3FooC2Ev.has_recursion, 0
+ .set .L_ZN3FooC2Ev.has_indirect_call, 0
+ .amdgpu_info _ZN3FooC2Ev
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 8
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 0
+ .amdgpu_typeid "v"
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 8
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 32
+ .set amdgpu.max_num_named_barrier, 0
+ .globl _ZN3FooC1Ev
+ .type _ZN3FooC1Ev, at function
+.set _ZN3FooC1Ev, _ZN3FooC2Ev
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels: []
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-resource-usage-recursive-scc.s b/lld/test/ELF/amdgpu-resource-usage-recursive-scc.s
new file mode 100644
index 0000000000000..48a616c317b69
--- /dev/null
+++ b/lld/test/ELF/amdgpu-resource-usage-recursive-scc.s
@@ -0,0 +1,257 @@
+# REQUIRES: amdgpu
+
+## Resource usage propagation through recursive SCCs must reach a fixed point.
+## In this graph, H/A/M form a cycle and A calls a high-resource leaf C:
+##
+## kernel_h -> H -> A -> M -> H
+## `-> C
+## kernel_m -> M
+##
+## Both kernels must receive C's resource usage through the SCC, regardless of
+## which SCC member they enter through.
+
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %s -o %t.o
+# RUN: ld.lld %t.o -o %t
+# RUN: llvm-readobj --notes %t | FileCheck %s
+
+# CHECK: .name: kernel_h
+# CHECK: .private_segment_fixed_size: 80
+# CHECK: .uses_dynamic_stack: true
+# CHECK: .vgpr_count: 100
+# CHECK: .name: kernel_m
+# CHECK: .private_segment_fixed_size: 80
+# CHECK: .uses_dynamic_stack: true
+# CHECK: .vgpr_count: 100
+
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+
+ .text
+ .globl kernel_h
+ .p2align 8
+ .type kernel_h, at function
+kernel_h:
+ s_endpgm
+.Lkernel_h_end:
+ .size kernel_h, .Lkernel_h_end-kernel_h
+
+ .globl kernel_m
+ .p2align 8
+ .type kernel_m, at function
+kernel_m:
+ s_endpgm
+.Lkernel_m_end:
+ .size kernel_m, .Lkernel_m_end-kernel_m
+
+ .globl H
+ .p2align 6
+ .type H, at function
+H:
+ s_setpc_b64 s[30:31]
+.LH_end:
+ .size H, .LH_end-H
+
+ .globl A
+ .p2align 6
+ .type A, at function
+A:
+ s_setpc_b64 s[30:31]
+.LA_end:
+ .size A, .LA_end-A
+
+ .globl M
+ .p2align 6
+ .type M, at function
+M:
+ s_setpc_b64 s[30:31]
+.LM_end:
+ .size M, .LM_end-M
+
+ .globl C
+ .p2align 6
+ .type C, at function
+C:
+ s_setpc_b64 s[30:31]
+.LC_end:
+ .size C, .LC_end-C
+
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl kernel_h.kd
+ .type kernel_h.kd, at object
+ .size kernel_h.kd, 64
+ .protected kernel_h
+kernel_h.kd:
+ .long 0
+ .long 0
+ .long 256
+ .long 0
+ .quad kernel_h at rel64-kernel_h.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11469063
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+
+ .p2align 6, 0x0
+ .globl kernel_m.kd
+ .type kernel_m.kd, at object
+ .size kernel_m.kd, 64
+ .protected kernel_m
+kernel_m.kd:
+ .long 0
+ .long 0
+ .long 256
+ .long 0
+ .quad kernel_m at rel64-kernel_m.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11469063
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+
+ .text
+ .amdgpu_info kernel_h
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 10
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 0
+ .amdgpu_call H
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info kernel_m
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 10
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 0
+ .amdgpu_call M
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info H
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 10
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 16
+ .amdgpu_call A
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info A
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 10
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 16
+ .amdgpu_call M
+ .amdgpu_call C
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info M
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 10
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 16
+ .amdgpu_call H
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info C
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 100
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 64
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 100
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 32
+ .set amdgpu.max_num_named_barrier, 0
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 256
+ .max_flat_workgroup_size: 1024
+ .name: kernel_h
+ .private_segment_fixed_size: 0
+ .sgpr_count: 32
+ .sgpr_spill_count: 0
+ .symbol: kernel_h.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 10
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 256
+ .max_flat_workgroup_size: 1024
+ .name: kernel_m
+ .private_segment_fixed_size: 0
+ .sgpr_count: 32
+ .sgpr_spill_count: 0
+ .symbol: kernel_m.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 10
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-resource-usage-section-sym.s b/lld/test/ELF/amdgpu-resource-usage-section-sym.s
new file mode 100644
index 0000000000000..4198258f27597
--- /dev/null
+++ b/lld/test/ELF/amdgpu-resource-usage-section-sym.s
@@ -0,0 +1,236 @@
+# REQUIRES: amdgpu
+
+## Test that link-time resource propagation handles functions referenced via
+## STT_SECTION symbols in relocations against the `.amdgpu.info` section.
+##
+## When a local (internal-linkage) function is placed in its own text section
+## (e.g. .text.unlikely. due to the cold attribute), ELF assemblers
+## canonically convert relocations targeting that function from the named
+## local symbol to section_symbol + addend. The linker must resolve these
+## section symbol references back to the named function symbol so that it
+## can build the call graph and propagate resource usage from `.amdgpu.info`
+## correctly.
+##
+## Without this handling, the callee's resource info (scratch size, VGPR
+## count, etc.) would be silently dropped and the kernel descriptor would
+## be patched with incorrect values.
+
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %s -o %t.o
+# RUN: llvm-readobj -S %t.o | FileCheck %s --check-prefix=SHDR
+# RUN: ld.lld %t.o -o %t.out
+# RUN: llvm-readobj --notes %t.out | FileCheck %s
+
+# SHDR: Name: .amdgpu.info
+
+## The kernel itself has private_segment_fixed_size 0 but calls callee which
+## has 132 bytes of scratch. After propagation, the kernel should reflect the
+## callee's scratch requirement.
+# CHECK: .name: kernel
+# CHECK: .private_segment_fixed_size: 132
+
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+
+## callee lives in .text.unlikely. — a separate text section.
+## Because callee is local (not .globl), llvm-mc will emit its relocations
+## in `.amdgpu.info` using the STT_SECTION symbol for
+## .text.unlikely. rather than the named symbol "callee".
+ .section .text.unlikely.,"ax", at progbits
+ .p2align 6
+ .type callee, at function
+callee:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_lshr_b32 s4, s32, 6
+ v_lshl_add_u32 v1, v0, 2, s4
+ buffer_store_dword v0, v1, s[0:3], 0 offen
+ s_waitcnt vmcnt(0)
+ buffer_load_dword v0, v1, s[0:3], 0 offen glc
+ s_waitcnt vmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end0:
+ .size callee, .Lfunc_end0-callee
+
+ .text
+ .globl kernel
+ .p2align 8
+ .type kernel, at function
+kernel:
+ s_add_u32 flat_scratch_lo, s12, s17
+ s_addc_u32 flat_scratch_hi, s13, 0
+ s_mov_b32 s13, s15
+ s_load_dwordx2 s[18:19], s[8:9], 0x0
+ s_load_dword s15, s[8:9], 0x8
+ s_add_u32 s0, s0, s17
+ s_addc_u32 s1, s1, 0
+ s_add_u32 s8, s8, 16
+ s_addc_u32 s9, s9, 0
+ v_lshlrev_b32_e32 v2, 20, v2
+ v_lshlrev_b32_e32 v1, 10, v1
+ s_mov_b32 s12, s14
+ s_getpc_b64 s[20:21]
+ s_add_u32 s20, s20, callee at rel32@lo+4
+ s_addc_u32 s21, s21, callee at rel32@hi+12
+ v_or3_b32 v31, v0, v1, v2
+ s_mov_b32 s14, s16
+ s_waitcnt lgkmcnt(0)
+ v_mov_b32_e32 v0, s15
+ s_mov_b32 s32, 0
+ v_mov_b32_e32 v3, 0
+ s_swappc_b64 s[30:31], s[20:21]
+ global_store_dword v3, v0, s[18:19]
+ s_endpgm
+
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl kernel.kd
+ .type kernel.kd, at object
+ .size kernel.kd, 64
+ .protected kernel
+kernel.kd:
+ .long 0
+ .long 0
+ .long 272
+ .long 0
+ .quad kernel at rel64-kernel.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11469063
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+
+ .text
+.Lfunc_end1:
+ .size kernel, .Lfunc_end1-kernel
+
+## Object linking metadata: callee has 132 bytes of scratch, kernel has 0.
+## Flags: 0x1=is_kernel, 0x4=uses_flat_scratch (kernel only).
+ .amdgpu_info callee
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 2
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 33
+ .amdgpu_private_segment_size 132
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_info kernel
+ .amdgpu_flags 2
+ .amdgpu_num_vgpr 32
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 33
+ .amdgpu_private_segment_size 0
+ .amdgpu_call callee
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .args:
+ - .address_space: global
+ .name: out
+ .offset: 0
+ .size: 8
+ .value_kind: global_buffer
+ - .name: x
+ .offset: 8
+ .size: 4
+ .value_kind: by_value
+ - .offset: 16
+ .size: 4
+ .value_kind: hidden_block_count_x
+ - .offset: 20
+ .size: 4
+ .value_kind: hidden_block_count_y
+ - .offset: 24
+ .size: 4
+ .value_kind: hidden_block_count_z
+ - .offset: 28
+ .size: 2
+ .value_kind: hidden_group_size_x
+ - .offset: 30
+ .size: 2
+ .value_kind: hidden_group_size_y
+ - .offset: 32
+ .size: 2
+ .value_kind: hidden_group_size_z
+ - .offset: 34
+ .size: 2
+ .value_kind: hidden_remainder_x
+ - .offset: 36
+ .size: 2
+ .value_kind: hidden_remainder_y
+ - .offset: 38
+ .size: 2
+ .value_kind: hidden_remainder_z
+ - .offset: 56
+ .size: 8
+ .value_kind: hidden_global_offset_x
+ - .offset: 64
+ .size: 8
+ .value_kind: hidden_global_offset_y
+ - .offset: 72
+ .size: 8
+ .value_kind: hidden_global_offset_z
+ - .offset: 80
+ .size: 2
+ .value_kind: hidden_grid_dims
+ - .offset: 96
+ .size: 8
+ .value_kind: hidden_hostcall_buffer
+ - .offset: 104
+ .size: 8
+ .value_kind: hidden_multigrid_sync_arg
+ - .offset: 112
+ .size: 8
+ .value_kind: hidden_heap_v1
+ - .offset: 120
+ .size: 8
+ .value_kind: hidden_default_queue
+ - .offset: 128
+ .size: 8
+ .value_kind: hidden_completion_action
+ - .offset: 216
+ .size: 8
+ .value_kind: hidden_queue_ptr
+ .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 272
+ .max_flat_workgroup_size: 1024
+ .name: kernel
+ .private_segment_fixed_size: 0
+ .sgpr_count: 39
+ .sgpr_spill_count: 0
+ .symbol: kernel.kd
+ .uses_dynamic_stack: true
+ .vgpr_count: 32
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-resource-usage-stale-info.s b/lld/test/ELF/amdgpu-resource-usage-stale-info.s
new file mode 100644
index 0000000000000..4f57a1d75de6a
--- /dev/null
+++ b/lld/test/ELF/amdgpu-resource-usage-stale-info.s
@@ -0,0 +1,230 @@
+# REQUIRES: amdgpu
+
+## Stale .amdgpu.info records from losing weak definitions or discarded COMDAT
+## groups must not overwrite the resource usage of the prevailing definition.
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/kernel.s -o %t/kernel.o
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/weak-winner.s -o %t/weak-winner.o
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/weak-stale.s -o %t/weak-stale.o
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/comdat-winner.s -o %t/comdat-winner.o
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/comdat-stale.s -o %t/comdat-stale.o
+# RUN: ld.lld %t/kernel.o %t/weak-winner.o %t/weak-stale.o %t/comdat-winner.o %t/comdat-stale.o -o %t/out
+# RUN: llvm-readobj --notes %t/out | FileCheck %s
+
+# CHECK: .name: kernel
+# CHECK: .private_segment_fixed_size: 32
+
+#--- kernel.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl kernel
+ .p2align 8
+ .type kernel, at function
+kernel:
+ s_endpgm
+.Lkernel_end:
+ .size kernel, .Lkernel_end-kernel
+
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl kernel.kd
+ .type kernel.kd, at object
+ .size kernel.kd, 64
+ .protected kernel
+kernel.kd:
+ .long 0
+ .long 0
+ .long 0
+ .long 0
+ .long 0
+ .quad kernel at rel64-kernel.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 0
+ .long 0
+ .short 0
+ .short 0
+ .long 0
+
+ .text
+ .amdgpu_info kernel
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 4
+ .amdgpu_num_sgpr 4
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 8
+ .amdgpu_call weak_helper
+ .amdgpu_call comdat_helper
+ .end_amdgpu_info
+
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 0
+ .max_flat_workgroup_size: 1024
+ .name: kernel
+ .private_segment_fixed_size: 0
+ .sgpr_count: 0
+ .symbol: kernel.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- weak-winner.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .weak weak_helper
+ .p2align 6
+ .type weak_helper, at function
+weak_helper:
+ s_setpc_b64 s[30:31]
+.Lweak_helper_end:
+ .size weak_helper, .Lweak_helper_end-weak_helper
+
+ .amdgpu_info weak_helper
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 8
+ .amdgpu_num_sgpr 8
+ .amdgpu_private_segment_size 16
+ .amdgpu_occupancy 8
+ .end_amdgpu_info
+
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels: []
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- weak-stale.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .weak weak_helper
+ .p2align 6
+ .type weak_helper, at function
+weak_helper:
+ s_setpc_b64 s[30:31]
+.Lweak_helper_end:
+ .size weak_helper, .Lweak_helper_end-weak_helper
+
+ .amdgpu_info weak_helper
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 64
+ .amdgpu_num_sgpr 64
+ .amdgpu_private_segment_size 128
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels: []
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- comdat-winner.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .section .text.comdat_helper,"axG", at progbits,comdat_helper,comdat
+ .globl comdat_helper
+ .p2align 6
+ .type comdat_helper, at function
+comdat_helper:
+ s_setpc_b64 s[30:31]
+.Lcomdat_helper_end:
+ .size comdat_helper, .Lcomdat_helper_end-comdat_helper
+
+ .amdgpu_info comdat_helper
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 12
+ .amdgpu_num_sgpr 12
+ .amdgpu_private_segment_size 32
+ .amdgpu_occupancy 8
+ .end_amdgpu_info
+
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels: []
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- comdat-stale.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .section .text.comdat_helper,"axG", at progbits,comdat_helper,comdat
+ .globl comdat_helper
+ .p2align 6
+ .type comdat_helper, at function
+comdat_helper:
+ s_setpc_b64 s[30:31]
+.Lcomdat_helper_end:
+ .size comdat_helper, .Lcomdat_helper_end-comdat_helper
+
+ .amdgpu_info comdat_helper
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 96
+ .amdgpu_num_sgpr 96
+ .amdgpu_private_segment_size 256
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels: []
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/lld/test/ELF/amdgpu-resource-usage.s b/lld/test/ELF/amdgpu-resource-usage.s
new file mode 100644
index 0000000000000..7f6034675b523
--- /dev/null
+++ b/lld/test/ELF/amdgpu-resource-usage.s
@@ -0,0 +1,400 @@
+# REQUIRES: amdgpu
+
+## Test link-time resource usage propagation for AMDGPU.
+## The linker parses AMDGPU object linking metadata in the `.amdgpu.info` section,
+## propagates resource usage
+## across the call graph, and patches the kernel descriptor and HSA metadata
+## with the propagated values.
+
+# RUN: split-file %s %t
+
+## --- GFX900 test ---
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/a.s -o %t/a.o
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %t/b.s -o %t/b.o
+# RUN: ld.lld %t/a.o %t/b.o -o %t/out
+
+## Verify the linker patches HSA metadata after resource propagation.
+# RUN: llvm-readobj --notes %t/out | FileCheck %s
+
+## kernel calls helper (defined in b.ll). After propagation, the kernel's
+## metadata should reflect the max of both functions' register usage.
+# CHECK: .name: kernel
+# CHECK: .sgpr_count:
+# CHECK: .vgpr_count:
+
+## --- GFX90A test ---
+## Test that on GFX90A the linker correctly accounts for extra SGPRs
+## and uses the proper VGPR total computation (GFX90A alignment rule).
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx90a -filetype=obj %t/a90a.s -o %t/a90a.o
+# RUN: llvm-mc -triple=amdgcn-amd-amdhsa -mcpu=gfx90a -filetype=obj %t/b90a.s -o %t/b90a.o
+# RUN: ld.lld %t/a90a.o %t/b90a.o -o %t/out90a
+# RUN: llvm-readobj --notes %t/out90a | FileCheck %s --check-prefix=GFX90A
+
+## After propagation, the kernel should have SGPRs with extras accounted for
+## and VGPR count from the callee.
+# GFX90A: .name: kernel90a
+# GFX90A: .sgpr_count:
+# GFX90A: .vgpr_count:
+
+#--- a.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl kernel ; -- Begin function kernel
+ .p2align 8
+ .type kernel, at function
+kernel:
+ s_endpgm
+.Lfunc_end0:
+ .size kernel, .Lfunc_end0-kernel
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl kernel.kd
+ .type kernel.kd, at object
+ .size kernel.kd, 64
+ .protected kernel
+kernel.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad kernel at rel64-kernel.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 0
+ .long 11469063
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .Lkernel.num_vgpr, 32
+ .set .Lkernel.num_agpr, 0
+ .set .Lkernel.numbered_sgpr, 33
+ .set .Lkernel.num_named_barrier, 0
+ .set .Lkernel.private_seg_size, 0
+ .set .Lkernel.uses_vcc, 1
+ .set .Lkernel.uses_flat_scratch, 1
+ .set .Lkernel.has_dyn_sized_stack, 0
+ .set .Lkernel.has_recursion, 0
+ .set .Lkernel.has_indirect_call, 0
+ .amdgpu_info kernel
+ .amdgpu_flags 3
+ .amdgpu_num_vgpr 32
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 33
+ .amdgpu_private_segment_size 0
+ .amdgpu_call helper
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 0
+ .set amdgpu.max_num_named_barrier, 0
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: kernel
+ .private_segment_fixed_size: 0
+ .sgpr_count: 39
+ .sgpr_spill_count: 0
+ .symbol: kernel.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 32
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- b.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+ .amdhsa_code_object_version 6
+ .text
+ .globl helper ; -- Begin function helper
+ .p2align 6
+ .type helper, at function
+helper:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end0:
+ .size helper, .Lfunc_end0-helper
+ .set .Lhelper.num_vgpr, 0
+ .set .Lhelper.num_agpr, 0
+ .set .Lhelper.numbered_sgpr, 32
+ .set .Lhelper.num_named_barrier, 0
+ .set .Lhelper.private_seg_size, 0
+ .set .Lhelper.uses_vcc, 0
+ .set .Lhelper.uses_flat_scratch, 0
+ .set .Lhelper.has_dyn_sized_stack, 0
+ .set .Lhelper.has_recursion, 0
+ .set .Lhelper.has_indirect_call, 0
+ .amdgpu_info helper
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 0
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 32
+ .set amdgpu.max_num_named_barrier, 0
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels: []
+amdhsa.target: amdgcn-amd-amdhsa--gfx900
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- a90a.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx90a"
+ .amdhsa_code_object_version 6
+ .text
+ .globl kernel90a ; -- Begin function kernel90a
+ .p2align 8
+ .type kernel90a, at function
+kernel90a:
+ s_endpgm
+.Lfunc_end0:
+ .size kernel90a, .Lfunc_end0-kernel90a
+ .section .rodata,"a", at progbits
+ .p2align 6, 0x0
+ .globl kernel90a.kd
+ .type kernel90a.kd, at object
+ .size kernel90a.kd, 64
+ .protected kernel90a
+kernel90a.kd:
+ .long 0
+ .long 0
+ .long 264
+ .long 0
+ .quad kernel90a at rel64-kernel90a.kd
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .byte 0
+ .long 7
+ .long 11469059
+ .long 5020
+ .short 63
+ .short 0
+ .long 0
+ .text
+ .set .Lkernel90a.num_vgpr, 32
+ .set .Lkernel90a.num_agpr, 0
+ .set .Lkernel90a.numbered_sgpr, 33
+ .set .Lkernel90a.num_named_barrier, 0
+ .set .Lkernel90a.private_seg_size, 0
+ .set .Lkernel90a.uses_vcc, 1
+ .set .Lkernel90a.uses_flat_scratch, 1
+ .set .Lkernel90a.has_dyn_sized_stack, 0
+ .set .Lkernel90a.has_recursion, 0
+ .set .Lkernel90a.has_indirect_call, 0
+ .text
+ .amdgpu_info kernel90a
+ .amdgpu_flags 3
+ .amdgpu_num_vgpr 32
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 33
+ .amdgpu_private_segment_size 0
+ .amdgpu_call mfma_helper
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 0
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 0
+ .set amdgpu.max_num_named_barrier, 0
+ .text
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels:
+ - .agpr_count: 0
+ .args:
+ - .name: x
+ .offset: 0
+ .size: 4
+ .value_kind: by_value
+ - .offset: 8
+ .size: 4
+ .value_kind: hidden_block_count_x
+ - .offset: 12
+ .size: 4
+ .value_kind: hidden_block_count_y
+ - .offset: 16
+ .size: 4
+ .value_kind: hidden_block_count_z
+ - .offset: 20
+ .size: 2
+ .value_kind: hidden_group_size_x
+ - .offset: 22
+ .size: 2
+ .value_kind: hidden_group_size_y
+ - .offset: 24
+ .size: 2
+ .value_kind: hidden_group_size_z
+ - .offset: 26
+ .size: 2
+ .value_kind: hidden_remainder_x
+ - .offset: 28
+ .size: 2
+ .value_kind: hidden_remainder_y
+ - .offset: 30
+ .size: 2
+ .value_kind: hidden_remainder_z
+ - .offset: 48
+ .size: 8
+ .value_kind: hidden_global_offset_x
+ - .offset: 56
+ .size: 8
+ .value_kind: hidden_global_offset_y
+ - .offset: 64
+ .size: 8
+ .value_kind: hidden_global_offset_z
+ - .offset: 72
+ .size: 2
+ .value_kind: hidden_grid_dims
+ - .offset: 88
+ .size: 8
+ .value_kind: hidden_hostcall_buffer
+ - .offset: 96
+ .size: 8
+ .value_kind: hidden_multigrid_sync_arg
+ - .offset: 104
+ .size: 8
+ .value_kind: hidden_heap_v1
+ - .offset: 112
+ .size: 8
+ .value_kind: hidden_default_queue
+ - .offset: 120
+ .size: 8
+ .value_kind: hidden_completion_action
+ - .offset: 208
+ .size: 8
+ .value_kind: hidden_queue_ptr
+ .group_segment_fixed_size: 0
+ .kernarg_segment_align: 8
+ .kernarg_segment_size: 264
+ .max_flat_workgroup_size: 1024
+ .name: kernel90a
+ .private_segment_fixed_size: 0
+ .sgpr_count: 39
+ .sgpr_spill_count: 0
+ .symbol: kernel90a.kd
+ .uses_dynamic_stack: false
+ .vgpr_count: 32
+ .vgpr_spill_count: 0
+ .wavefront_size: 64
+amdhsa.target: amdgcn-amd-amdhsa--gfx90a
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
+
+#--- b90a.s
+ .amdgcn_target "amdgcn-amd-amdhsa--gfx90a"
+ .amdhsa_code_object_version 6
+ .text
+ .globl mfma_helper ; -- Begin function mfma_helper
+ .p2align 6
+ .type mfma_helper, at function
+mfma_helper:
+ s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ s_setpc_b64 s[30:31]
+.Lfunc_end0:
+ .size mfma_helper, .Lfunc_end0-mfma_helper
+ .set .Lmfma_helper.num_vgpr, 8
+ .set .Lmfma_helper.num_agpr, 0
+ .set .Lmfma_helper.numbered_sgpr, 32
+ .set .Lmfma_helper.num_named_barrier, 0
+ .set .Lmfma_helper.private_seg_size, 0
+ .set .Lmfma_helper.uses_vcc, 0
+ .set .Lmfma_helper.uses_flat_scratch, 0
+ .set .Lmfma_helper.has_dyn_sized_stack, 0
+ .set .Lmfma_helper.has_recursion, 0
+ .set .Lmfma_helper.has_indirect_call, 0
+ .text
+ .amdgpu_info mfma_helper
+ .amdgpu_flags 0
+ .amdgpu_num_vgpr 8
+ .amdgpu_num_agpr 0
+ .amdgpu_num_sgpr 32
+ .amdgpu_private_segment_size 0
+ .amdgpu_occupancy 4
+ .end_amdgpu_info
+
+ .section .AMDGPU.gpr_maximums,"", at progbits
+ .set amdgpu.max_num_vgpr, 8
+ .set amdgpu.max_num_agpr, 0
+ .set amdgpu.max_num_sgpr, 32
+ .set amdgpu.max_num_named_barrier, 0
+ .text
+ .section ".note.GNU-stack","", at progbits
+ .amdgpu_metadata
+---
+amdhsa.kernels: []
+amdhsa.target: amdgcn-amd-amdhsa--gfx90a
+amdhsa.version:
+ - 1
+ - 2
+...
+
+ .end_amdgpu_metadata
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 40a2820b9e04f..50534a65a89c6 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -2530,8 +2530,10 @@ The AMDGPU backend supports the following LLVM IR attributes.
CLANG attribute [CLANG-ATTR]_. This is an optimization hint,
and the backend may not be able to satisfy the request. If
the specified range is incompatible with the function's
- "amdgpu-flat-work-group-size" value, the implied occupancy
- bounds by the workgroup size takes precedence.
+ "amdgpu-flat-work-group-size" value, the occupancy bounds
+ implied by the workgroup size take precedence. Under object
+ linking, this hint cannot lower the ABI occupancy budget;
+ see :ref:`amdgpu-abi-occupancy`.
"amdgpu-ieee" true/false. GFX6-GFX11 (Except GFX11.7) Only
Specify whether the function expects the IEEE field of the
@@ -2943,6 +2945,51 @@ unit's worst case (i.e, maxima) ``num_vgpr``, ``num_agpr``, and
symbolic expressions. These three symbols are ``amdgcn.max_num_vgpr``,
``amdgcn.max_num_agpr``, and ``amdgcn.max_num_sgpr``.
+.. _amdgpu-abi-occupancy:
+
+ABI Occupancy (Object Linking)
+------------------------------
+
+When object linking is enabled, the AMDGPU backend no longer assumes
+whole-program visibility. Individual translation units must agree on an ABI that
+callers and callees can rely on without seeing each other's register usage. The
+compiler enforces this by using an *ABI occupancy* as a resource-budget floor
+for separately compiled functions. The default ABI occupancy is the number of
+waves per EU required to support a 1024-workitem workgroup:
+
+``ceil(ceil(1024 / wavefront-size) / EUs-per-CU)``.
+
+For example, on ``gfx900`` (wave64, 4 EUs/CU) the default ABI occupancy is
+``ceil(ceil(1024 / 64) / 4) = 4``, yielding a VGPR budget of
+``getMaxNumVGPRs(4) = 64``. On a gfx12 target such as ``gfx1200`` (wave32,
+4 EUs/CU), the default ABI occupancy is ``ceil(ceil(1024 / 32) / 4) = 8``,
+yielding a VGPR budget of ``getMaxNumVGPRs(8) = 192``.
+
+The ``amdgpu_abi_waves_per_eu`` module flag overrides the default ABI occupancy
+floor for a module. A value lower than the default accepts a looser ABI
+contract; a value higher than the default requests a stricter one. Any compiler
+driver option that exposes this control should set the module flag in generated
+IR rather than passing the value directly to the backend.
+
+``amdgpu-flat-work-group-size`` is ABI-significant. If a function carries this
+attribute, the occupancy implied by its maximum flat workgroup size replaces
+the default ABI occupancy for that function, so it can either raise or lower the
+register budget.
+
+``amdgpu-waves-per-eu`` remains a hint, and it does not define the ABI
+occupancy. It affects the ABI register budget only if the backend accepts it as
+an effective occupancy request. In that case, a minimum higher than the ABI
+occupancy makes the register budget stricter; a lower minimum cannot lower the
+ABI budget. The maximum side of an accepted ``amdgpu-waves-per-eu`` hint can
+still cap occupancy by inflating reported resource usage. Under object linking,
+the effective maximum used for this inflation is raised to at least the ABI
+occupancy, so the emitted resource usage does not describe a looser contract
+than the ABI budget.
+
+The backend records the register-budget occupancy in each function's
+``.amdgpu.info`` metadata so link-time object linking can reject incompatible
+occupancy contracts across translation units.
+
.. _amdgpu-elf-code-object:
ELF Code Object
@@ -3379,11 +3426,10 @@ if needed.
.. _amdgpu-info-section:
``.amdgpu.info``
- Per-function metadata for AMDGPU object linking, emitted only in relocatable
- code objects when object linking is enabled
- (``-amdgpu-enable-object-linking``). The linker uses this section to
- propagate resource usage (registers, stack, LDS) and resolve call graph
- dependencies across translation units.
+ Per-function metadata emitted only in relocatable code objects when object
+ linking is enabled. The linker uses this section to propagate resource usage
+ (registers, stack, LDS) and resolve call graph dependencies across
+ translation units.
Each entry uses a tagged, length-prefixed binary encoding:
@@ -3412,6 +3458,7 @@ if needed.
8 ``INFO_CALL`` 8B symbol ref; direct call edge
9 ``INFO_INDIRECT_CALL`` u32 strtab offset; indirect call type-ID
10 ``INFO_TYPEID`` u32 strtab offset; function type-ID
+ 11 ``INFO_OCCUPANCY`` u32; occupancy used to compile the function
===== ============================== ==========================================
.. table:: AMDGPU Info Function Flags (``INFO_FLAGS``)
@@ -3423,6 +3470,8 @@ if needed.
0x1 ``FUNC_USES_VCC`` Function uses the VCC register
0x2 ``FUNC_USES_FLAT_SCRATCH`` Function uses flat scratch addressing
0x4 ``FUNC_HAS_DYN_STACK`` Function has dynamic stack allocation
+ 0x8 ``FUNC_WGP_MODE`` Function uses WGP execution mode
+ 0x10 ``FUNC_WAVE32`` Function uses wave32; otherwise wave64
===== =========================== ==========================================
Symbol references (``INFO_FUNC``, ``INFO_USE``, ``INFO_CALL``) generate
@@ -21871,6 +21920,7 @@ The following sub-directives may appear inside the block:
``.amdgpu_num_vgpr`` *value* Architectural VGPRs used (u32)
``.amdgpu_num_agpr`` *value* Accumulator VGPRs used (u32)
``.amdgpu_private_segment_size`` *n* Private segment size in bytes (u32)
+ ``.amdgpu_occupancy`` *value* Occupancy used to compile the function (u32)
``.amdgpu_use`` *symbol* Resource dependency (LDS or barrier)
``.amdgpu_call`` *symbol* Direct call edge to *symbol*
``.amdgpu_indirect_call`` *"type-id"* Indirect call with given type-ID string
@@ -21887,6 +21937,7 @@ Example:
.amdgpu_num_vgpr 32
.amdgpu_num_agpr 0
.amdgpu_private_segment_size 0
+ .amdgpu_occupancy 4
.amdgpu_use lds_var
.amdgpu_call helper
.amdgpu_indirect_call "vi"
diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h
index 46d806e1b9d8a..e080a5f20acb7 100644
--- a/llvm/include/llvm/BinaryFormat/ELF.h
+++ b/llvm/include/llvm/BinaryFormat/ELF.h
@@ -933,6 +933,18 @@ enum : unsigned {
EF_AMDGPU_GENERIC_VERSION_MAX = 0xff,
};
+inline StringRef getAMDGPUArchNameFromELFMach(unsigned Mach) {
+ switch (Mach) {
+#define X(NUM, ENUM, NAME) \
+ case ENUM: \
+ return NAME;
+ AMDGPU_MACH_LIST(X)
+#undef X
+ default:
+ return "";
+ }
+}
+
// ELF Relocation types for AMDGPU
enum {
#include "ELFRelocs/AMDGPU.def"
diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
index a0617b9137ef1..74065a8a9da01 100644
--- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -63,6 +63,7 @@ enum BlockIDs {
SYMTAB_BLOCK_ID,
SYNC_SCOPE_NAMES_BLOCK_ID,
+
};
/// Identification block contains a string that describes the producer details,
@@ -348,6 +349,19 @@ enum GlobalValueSummarySymtabCodes {
// nummib x alloc type,
// numver x version]
FS_COMBINED_ALLOC_INFO_NO_CONTEXT = 33,
+ // Optional AMDGPU per-function data, emitted after the corresponding
+ // function summary record.
+ // [calling_conv, flags, flat_wg_min, flat_wg_max, waves_min, waves_max,
+ // max_wg_x, max_wg_y, max_wg_z]
+ FS_AMDGPU_ENTRY = 34,
+};
+
+/// Flags for the FS_AMDGPU_ENTRY record.
+enum AMDGPUFunctionSummaryFlags {
+ AFS_HAS_FLAT_WORK_GROUP_SIZE = 1 << 0,
+ AFS_HAS_WAVES_PER_EU = 1 << 1,
+ AFS_HAS_WAVES_PER_EU_MAX = 1 << 2,
+ AFS_HAS_MAX_NUM_WORKGROUPS = 1 << 3,
};
enum MetadataCodes {
diff --git a/llvm/include/llvm/IR/ModuleSummaryIndex.h b/llvm/include/llvm/IR/ModuleSummaryIndex.h
index 4f68582b65c4e..7ae0d932d0cfd 100644
--- a/llvm/include/llvm/IR/ModuleSummaryIndex.h
+++ b/llvm/include/llvm/IR/ModuleSummaryIndex.h
@@ -28,6 +28,7 @@
#include "llvm/ADT/StringSet.h"
#include "llvm/ADT/iterator.h"
#include "llvm/ADT/iterator_range.h"
+#include "llvm/IR/CallingConv.h"
#include "llvm/IR/ConstantRange.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/Module.h"
@@ -1387,6 +1388,34 @@ struct TypeIdSummary {
std::map<uint64_t, WholeProgramDevirtResolution> WPDRes;
};
+/// Optional per-function AMDGPU information carried in the module summary.
+///
+/// Attribute presence is retained because an absent attribute can have a
+/// subtarget-dependent default and is not equivalent to an explicit value.
+struct AMDGPUFunctionSummary {
+ struct WavesPerEUInfo {
+ uint32_t Min = 0;
+ std::optional<uint32_t> Max;
+
+ bool operator==(const WavesPerEUInfo &Other) const {
+ return Min == Other.Min && Max == Other.Max;
+ }
+ };
+
+ CallingConv::ID CC = CallingConv::C;
+ std::optional<std::pair<uint32_t, uint32_t>> FlatWorkGroupSize;
+ std::optional<WavesPerEUInfo> WavesPerEU;
+ std::optional<std::array<uint32_t, 3>> MaxNumWorkgroups;
+
+ bool operator==(const AMDGPUFunctionSummary &Other) const {
+ return CC == Other.CC && FlatWorkGroupSize == Other.FlatWorkGroupSize &&
+ WavesPerEU == Other.WavesPerEU &&
+ MaxNumWorkgroups == Other.MaxNumWorkgroups;
+ }
+};
+
+using AMDGPUSummaryMap = DenseMap<GlobalValue::GUID, AMDGPUFunctionSummary>;
+
/// Encapsulate the names of CFI target functions. It interfaces with ThinLTO to
/// determine efficiently which of the names need to be exported for a
/// particular module.
@@ -1619,6 +1648,11 @@ class ModuleSummaryIndex {
// built via releaseTemporaryMemory.
DenseMap<uint64_t, unsigned> StackIdToIndex;
+ // Optional AMDGPU per-function occupancy data, indexed by GUID.
+ // Populated during summary parsing and used by ThinLTO to propagate
+ // kernel occupancy constraints to device functions.
+ AMDGPUSummaryMap AMDGPUSummaries;
+
// YAML I/O support.
friend yaml::MappingTraits<ModuleSummaryIndex>;
@@ -1670,6 +1704,9 @@ class ModuleSummaryIndex {
const std::vector<uint64_t> &stackIds() const { return StackIds; }
+ AMDGPUSummaryMap &getAMDGPUSummaries() { return AMDGPUSummaries; }
+ const AMDGPUSummaryMap &getAMDGPUSummaries() const { return AMDGPUSummaries; }
+
unsigned addOrGetStackIdIndex(uint64_t StackId) {
auto Inserted = StackIdToIndex.insert({StackId, StackIds.size()});
if (Inserted.second)
diff --git a/llvm/include/llvm/LTO/Config.h b/llvm/include/llvm/LTO/Config.h
index f322f753813ff..a285215e835c1 100644
--- a/llvm/include/llvm/LTO/Config.h
+++ b/llvm/include/llvm/LTO/Config.h
@@ -257,6 +257,12 @@ struct Config {
/// splitting the module.
ModuleHookFn PreCodeGenModuleHook;
+ /// This hook is called in the ThinLTO backend after finalization and import,
+ /// but before optimization. Unlike ModuleHookFn, it takes a mutable Module
+ /// reference, allowing targets to apply cross-TU propagated attributes.
+ using MutableModuleHookFn = std::function<void(unsigned Task, Module &)>;
+ MutableModuleHookFn ApplyThinLTOAttributes;
+
/// A combined index hook is called after all per-module indexes have been
/// combined (ThinLTO-specific). It can be used to implement -save-temps for
/// the combined index.
diff --git a/llvm/include/llvm/Object/ELFTypes.h b/llvm/include/llvm/Object/ELFTypes.h
index 6d5a0bdb21130..b3c5fca6fb2b7 100644
--- a/llvm/include/llvm/Object/ELFTypes.h
+++ b/llvm/include/llvm/Object/ELFTypes.h
@@ -264,6 +264,11 @@ struct Elf_Sym_Impl : Elf_Sym_Base<ELFT> {
return getType() == ELF::STT_COMMON || st_shndx == ELF::SHN_COMMON;
}
+ bool isCommon(uint16_t EMachine) const {
+ return isCommon() ||
+ (EMachine == ELF::EM_AMDGPU && st_shndx == ELF::SHN_AMDGPU_LDS);
+ }
+
bool isDefined() const { return !isUndefined(); }
bool isProcessorSpecific() const {
diff --git a/llvm/include/llvm/Support/AMDGPUObjLinkingInfo.h b/llvm/include/llvm/Support/AMDGPUObjLinkingInfo.h
index e65161e6545fc..5db9a65ca6e78 100644
--- a/llvm/include/llvm/Support/AMDGPUObjLinkingInfo.h
+++ b/llvm/include/llvm/Support/AMDGPUObjLinkingInfo.h
@@ -58,6 +58,9 @@ enum class InfoKind : uint8_t {
/// string (at the given `.amdgpu.strtab` offset) so the linker can match
/// it against INFO_INDIRECT_CALL entries. [u32]
INFO_TYPEID = 10,
+ /// Occupancy used to compile the function.
+ /// [u32]
+ INFO_OCCUPANCY = 11,
};
/// Per-function flags packed into INFO_FLAGS entries.
@@ -65,7 +68,11 @@ enum class FuncInfoFlags : uint32_t {
FUNC_USES_VCC = 1U << 0,
FUNC_USES_FLAT_SCRATCH = 1U << 1,
FUNC_HAS_DYN_STACK = 1U << 2,
- LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/FUNC_HAS_DYN_STACK),
+ /// Function uses WGP mode. If unset, the function uses CU mode.
+ FUNC_WGP_MODE = 1U << 3,
+ /// Function uses wave32. If unset, the function uses wave64.
+ FUNC_WAVE32 = 1U << 4,
+ LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/FUNC_WAVE32),
};
} // namespace AMDGPU
diff --git a/llvm/include/llvm/TargetParser/AMDGPUTargetParser.h b/llvm/include/llvm/TargetParser/AMDGPUTargetParser.h
index 5d4a87b64f86d..82d14f9689b3e 100644
--- a/llvm/include/llvm/TargetParser/AMDGPUTargetParser.h
+++ b/llvm/include/llvm/TargetParser/AMDGPUTargetParser.h
@@ -223,6 +223,21 @@ constexpr unsigned getMinWavesPerEU() { return 1; }
LLVM_ABI unsigned getMaxWavesPerEU(GPUKind AK);
LLVM_ABI unsigned getMaxWavesPerEU(Triple::SubArchType SubArch);
+class TargetID;
+
+/// Queries used to resolve object-linking resources without MCSubtargetInfo.
+LLVM_ABI bool isObjectLinkingWaveSizeSupported(const TargetID &Target,
+ unsigned WaveSize);
+LLVM_ABI unsigned getNumExtraSGPRs(const TargetID &Target, bool VCCUsed,
+ bool FlatScrUsed);
+LLVM_ABI unsigned getEncodedNumVGPRBlocks(const TargetID &Target,
+ unsigned NumVGPRs, unsigned WaveSize);
+LLVM_ABI unsigned getNumSGPRBlocks(unsigned NumSGPRs);
+LLVM_ABI bool isLDSSizeCompatibleWithOccupancy(const TargetID &Target,
+ unsigned WaveSize, bool IsCuMode,
+ uint64_t LDSBytes,
+ unsigned Occupancy);
+
/// Fills Features map with default values for given target GPU.
/// \p Features contains overriding target features and this function returns
/// default target features with entries overridden by \p Features.
diff --git a/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp b/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp
index 6574ab7a93c58..8688cd4b583c0 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp
@@ -333,6 +333,7 @@ GetCodeName(unsigned CodeID, unsigned BlockID,
STRINGIFY_CODE(FS, ALLOC_CONTEXT_IDS)
STRINGIFY_CODE(FS, CONTEXT_RADIX_TREE_ARRAY)
STRINGIFY_CODE(FS, COMBINED_ALLOC_INFO_NO_CONTEXT)
+ STRINGIFY_CODE(FS, AMDGPU_ENTRY)
}
case bitc::METADATA_ATTACHMENT_ID:
switch (CodeID) {
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index bf8f6d12ce5a8..af858ed201c44 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -7873,6 +7873,11 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
GlobalValueSummary *LastSeenSummary = nullptr;
GlobalValue::GUID LastSeenGUID = 0;
+ // Track the function associated with an optional AMDGPU summary record.
+ // Unlike LastSeenGUID, this is never set for aliases or global variables.
+ std::optional<GlobalValue::GUID> CurrentFunctionGUID;
+ bool CurrentFunctionIsPrevailing = false;
+
// Track the most recent function summary if it was prevailing, and while we
// are not done processing any subsequent memprof records. Starting with
// summary version 13 (tracked by MemProfAfterFunctionSummary), MemProf
@@ -7926,6 +7931,12 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
return MaybeBitCode.takeError();
unsigned BitCode = MaybeBitCode.get();
+ // An AMDGPU entry must immediately follow its function summary.
+ if (BitCode != bitc::FS_AMDGPU_ENTRY) {
+ CurrentFunctionGUID = std::nullopt;
+ CurrentFunctionIsPrevailing = false;
+ }
+
switch (BitCode) {
default: // Default behavior: ignore.
break;
@@ -8029,6 +8040,9 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
std::move(PendingAllocs));
FS->setModulePath(getThisModule()->first());
FS->setOriginalName(GUID);
+ LastSeenGUID = VI.getGUID();
+ CurrentFunctionGUID = LastSeenGUID;
+ CurrentFunctionIsPrevailing = IsPrevailingSym;
// Set CurrentPrevailingFS only if prevailing, so subsequent MemProf
// records are attached (new order) or skipped.
if (MemProfAfterFunctionSummary) {
@@ -8064,6 +8078,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
auto GUID = getValueInfoFromValueId(ValueID);
AS->setOriginalName(std::get<1>(GUID));
+ LastSeenGUID = std::get<0>(GUID).getGUID();
TheIndex.addGlobalValueSummary(std::get<0>(GUID), std::move(AS));
break;
}
@@ -8182,6 +8197,8 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
if (MemProfAfterFunctionSummary)
CurrentPrevailingFS = FS.get();
LastSeenGUID = VI.getGUID();
+ CurrentFunctionGUID = LastSeenGUID;
+ CurrentFunctionIsPrevailing = true;
FS->setModulePath(ModuleIdMap[ModuleId]);
TheIndex.addGlobalValueSummary(VI, std::move(FS));
break;
@@ -8237,6 +8254,54 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
TheIndex.addGlobalValueSummary(VI, std::move(FS));
break;
}
+ case bitc::FS_AMDGPU_ENTRY: {
+ if (Record.size() < 9)
+ return error("invalid AMDGPU summary entry");
+ if (!CurrentFunctionGUID)
+ return error("amdgpu summary entry does not follow a function summary");
+
+ if (Record[0] > CallingConv::MaxID ||
+ llvm::any_of(ArrayRef(Record).slice(2, 7),
+ [](uint64_t Value) { return !isUInt<32>(Value); }))
+ return error("invalid value in AMDGPU summary entry");
+
+ uint64_t Flags = Record[1];
+ if ((Flags & bitc::AFS_HAS_WAVES_PER_EU_MAX) &&
+ !(Flags & bitc::AFS_HAS_WAVES_PER_EU))
+ return error("invalid flags in AMDGPU summary entry");
+
+ AMDGPUFunctionSummary AFS;
+ AFS.CC = static_cast<CallingConv::ID>(Record[0]);
+ if (Flags & bitc::AFS_HAS_FLAT_WORK_GROUP_SIZE)
+ AFS.FlatWorkGroupSize = std::make_pair(
+ static_cast<uint32_t>(Record[2]), static_cast<uint32_t>(Record[3]));
+ if (Flags & bitc::AFS_HAS_WAVES_PER_EU) {
+ AMDGPUFunctionSummary::WavesPerEUInfo WavesPerEU{
+ static_cast<uint32_t>(Record[4]), std::nullopt};
+ if (Flags & bitc::AFS_HAS_WAVES_PER_EU_MAX)
+ WavesPerEU.Max = static_cast<uint32_t>(Record[5]);
+ AFS.WavesPerEU = WavesPerEU;
+ }
+ if (Flags & bitc::AFS_HAS_MAX_NUM_WORKGROUPS)
+ AFS.MaxNumWorkgroups = std::array<uint32_t, 3>{
+ static_cast<uint32_t>(Record[6]), static_cast<uint32_t>(Record[7]),
+ static_cast<uint32_t>(Record[8])};
+
+ if (!CurrentFunctionIsPrevailing) {
+ CurrentFunctionGUID = std::nullopt;
+ CurrentFunctionIsPrevailing = false;
+ break;
+ }
+
+ AMDGPUSummaryMap &Summaries = TheIndex.getAMDGPUSummaries();
+ auto [It, Inserted] = Summaries.try_emplace(*CurrentFunctionGUID, AFS);
+ if (!Inserted && !(It->second == AFS))
+ return error("conflicting AMDGPU summary entries for GUID " +
+ Twine(*CurrentFunctionGUID));
+ CurrentFunctionGUID = std::nullopt;
+ CurrentFunctionIsPrevailing = false;
+ break;
+ }
// FS_COMBINED_ORIGINAL_NAME: [original_name]
case bitc::FS_COMBINED_ORIGINAL_NAME: {
uint64_t OriginalName = Record[0];
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 0b9b1bccb1fb8..a3b7d9cad69ca 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -4677,6 +4677,101 @@ static void writeFunctionHeapProfileRecords(
}
}
+/// Build an AMDGPUFunctionSummary from the IR attributes on \p F.
+static AMDGPUFunctionSummary buildAMDGPUFunctionSummary(const Function &F) {
+ AMDGPUFunctionSummary AFS;
+ AFS.CC = F.getCallingConv();
+
+ Attribute A = F.getFnAttribute("amdgpu-flat-work-group-size");
+ if (A.isStringAttribute()) {
+ auto [MinS, MaxS] = A.getValueAsString().split(',');
+ uint32_t Min, Max;
+ if (!MinS.trim().getAsInteger(0, Min) && !MaxS.trim().getAsInteger(0, Max))
+ AFS.FlatWorkGroupSize = std::make_pair(Min, Max);
+ }
+
+ A = F.getFnAttribute("amdgpu-waves-per-eu");
+ if (A.isStringAttribute()) {
+ auto [MinS, MaxS] = A.getValueAsString().split(',');
+ uint32_t Min;
+ if (!MinS.trim().getAsInteger(0, Min)) {
+ AMDGPUFunctionSummary::WavesPerEUInfo WavesPerEU{Min, std::nullopt};
+ uint32_t Max;
+ if (MaxS.trim().empty() || !MaxS.trim().getAsInteger(0, Max)) {
+ if (!MaxS.trim().empty())
+ WavesPerEU.Max = Max;
+ AFS.WavesPerEU = WavesPerEU;
+ }
+ }
+ }
+
+ A = F.getFnAttribute("amdgpu-max-num-workgroups");
+ if (A.isStringAttribute()) {
+ SmallVector<StringRef, 3> Parts;
+ A.getValueAsString().split(Parts, ',');
+ if (Parts.size() == 3) {
+ uint32_t X, Y, Z;
+ if (!Parts[0].trim().getAsInteger(0, X) &&
+ !Parts[1].trim().getAsInteger(0, Y) &&
+ !Parts[2].trim().getAsInteger(0, Z))
+ AFS.MaxNumWorkgroups = std::array<uint32_t, 3>{X, Y, Z};
+ }
+ }
+
+ return AFS;
+}
+
+/// Emit an FS_AMDGPU_ENTRY record for a pre-built summary.
+static void writeAMDGPUSummaryRecord(BitstreamWriter &Stream,
+ const AMDGPUFunctionSummary &AFS) {
+ uint64_t Flags = 0;
+ uint32_t FlatWGSizeMin = 0;
+ uint32_t FlatWGSizeMax = 0;
+ uint32_t WavesPerEUMin = 0;
+ uint32_t WavesPerEUMax = 0;
+ uint32_t MaxNumWGX = 0;
+ uint32_t MaxNumWGY = 0;
+ uint32_t MaxNumWGZ = 0;
+
+ if (AFS.FlatWorkGroupSize) {
+ Flags |= bitc::AFS_HAS_FLAT_WORK_GROUP_SIZE;
+ FlatWGSizeMin = AFS.FlatWorkGroupSize->first;
+ FlatWGSizeMax = AFS.FlatWorkGroupSize->second;
+ }
+ if (AFS.WavesPerEU) {
+ Flags |= bitc::AFS_HAS_WAVES_PER_EU;
+ WavesPerEUMin = AFS.WavesPerEU->Min;
+ if (AFS.WavesPerEU->Max) {
+ Flags |= bitc::AFS_HAS_WAVES_PER_EU_MAX;
+ WavesPerEUMax = *AFS.WavesPerEU->Max;
+ }
+ }
+ if (AFS.MaxNumWorkgroups) {
+ Flags |= bitc::AFS_HAS_MAX_NUM_WORKGROUPS;
+ MaxNumWGX = (*AFS.MaxNumWorkgroups)[0];
+ MaxNumWGY = (*AFS.MaxNumWorkgroups)[1];
+ MaxNumWGZ = (*AFS.MaxNumWorkgroups)[2];
+ }
+
+ SmallVector<uint64_t> NameVals = {AFS.CC, Flags, FlatWGSizeMin,
+ FlatWGSizeMax, WavesPerEUMin, WavesPerEUMax,
+ MaxNumWGX, MaxNumWGY, MaxNumWGZ};
+ Stream.EmitRecord(bitc::FS_AMDGPU_ENTRY, NameVals);
+}
+
+/// Emit an FS_AMDGPU_ENTRY record for \p F if it is a kernel or carries
+/// explicit occupancy attributes on an amdgcn target.
+static void writeAMDGPUSummaryRecord(BitstreamWriter &Stream,
+ const Function &F) {
+ bool IsKernel = F.getCallingConv() == CallingConv::AMDGPU_KERNEL;
+ bool HasAttrs = F.hasFnAttribute("amdgpu-flat-work-group-size") ||
+ F.hasFnAttribute("amdgpu-waves-per-eu") ||
+ F.hasFnAttribute("amdgpu-max-num-workgroups");
+ if (!IsKernel && !HasAttrs)
+ return;
+ writeAMDGPUSummaryRecord(Stream, buildAMDGPUFunctionSummary(F));
+}
+
// Helper to emit a single function summary record.
void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord(
SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary,
@@ -4713,6 +4808,11 @@ void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord(
Stream.EmitRecord(bitc::FS_PERMODULE_PROFILE, NameVals, FSCallsProfileAbbrev);
NameVals.clear();
+ // Emit optional AMDGPU per-function information immediately after the
+ // corresponding function summary.
+ if (M.getTargetTriple().isAMDGCN())
+ writeAMDGPUSummaryRecord(Stream, F);
+
writeFunctionHeapProfileRecords(
Stream, FS, CallsiteAbbrev, AllocAbbrev, ContextIdAbbvId,
/*PerModule*/ true,
@@ -5360,6 +5460,11 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
FSCallsProfileAbbrev);
NameVals.clear();
+ // Emit optional AMDGPU per-function occupancy record if present.
+ if (auto It = Index.getAMDGPUSummaries().find(I.first);
+ It != Index.getAMDGPUSummaries().end())
+ writeAMDGPUSummaryRecord(Stream, It->second);
+
writeFunctionHeapProfileRecords(
Stream, FS, CallsiteAbbrev, AllocAbbrev, /*ContextIdAbbvId*/ 0,
/*PerModule*/ false,
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index 4594c52fb5f6e..56c1216cab532 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -62,6 +62,7 @@
#include "llvm/Transforms/Utils/FunctionImportUtils.h"
#include "llvm/Transforms/Utils/SplitModule.h"
+#include <limits>
#include <optional>
#include <set>
@@ -2056,6 +2057,198 @@ ThinBackend lto::createWriteIndexesThinBackend(
return ThinBackend(Func, Parallelism);
}
+namespace {
+
+/// Values used by the current AMDGPU ThinLTO propagation policy.
+/// A missing waves-per-EU maximum remains target-dependent.
+struct AMDGPUPropagatedFunctionAttrs {
+ CallingConv::ID CC = CallingConv::C;
+ uint32_t FlatWGSizeMin = 1;
+ uint32_t FlatWGSizeMax = 1024;
+ uint32_t WavesPerEUMin = 1;
+ std::optional<uint32_t> WavesPerEUMax;
+ uint32_t MaxNumWGX = std::numeric_limits<uint32_t>::max();
+ uint32_t MaxNumWGY = std::numeric_limits<uint32_t>::max();
+ uint32_t MaxNumWGZ = std::numeric_limits<uint32_t>::max();
+
+ void mergeFrom(const AMDGPUPropagatedFunctionAttrs &Caller) {
+ FlatWGSizeMin = std::min(FlatWGSizeMin, Caller.FlatWGSizeMin);
+ FlatWGSizeMax = std::max(FlatWGSizeMax, Caller.FlatWGSizeMax);
+ WavesPerEUMin = std::max(WavesPerEUMin, Caller.WavesPerEUMin);
+ if (WavesPerEUMax && Caller.WavesPerEUMax)
+ WavesPerEUMax = std::max(*WavesPerEUMax, *Caller.WavesPerEUMax);
+ else
+ WavesPerEUMax = std::nullopt;
+ MaxNumWGX = std::max(MaxNumWGX, Caller.MaxNumWGX);
+ MaxNumWGY = std::max(MaxNumWGY, Caller.MaxNumWGY);
+ MaxNumWGZ = std::max(MaxNumWGZ, Caller.MaxNumWGZ);
+ }
+
+ bool hasValidWavesPerEU() const {
+ return !WavesPerEUMax || WavesPerEUMin <= *WavesPerEUMax;
+ }
+};
+
+using AMDGPUPropagatedAttrsMap =
+ DenseMap<GlobalValue::GUID, AMDGPUPropagatedFunctionAttrs>;
+
+static AMDGPUPropagatedFunctionAttrs
+getAMDGPUPropagatedAttrs(const AMDGPUFunctionSummary &Summary) {
+ AMDGPUPropagatedFunctionAttrs Attrs;
+ Attrs.CC = Summary.CC;
+
+ if (Summary.FlatWorkGroupSize) {
+ Attrs.FlatWGSizeMin = Summary.FlatWorkGroupSize->first;
+ Attrs.FlatWGSizeMax = Summary.FlatWorkGroupSize->second;
+ }
+
+ if (Summary.WavesPerEU) {
+ Attrs.WavesPerEUMin = Summary.WavesPerEU->Min;
+ if (Summary.WavesPerEU->Max)
+ Attrs.WavesPerEUMax = *Summary.WavesPerEU->Max;
+ }
+
+ if (Summary.MaxNumWorkgroups) {
+ Attrs.MaxNumWGX = (*Summary.MaxNumWorkgroups)[0];
+ Attrs.MaxNumWGY = (*Summary.MaxNumWorkgroups)[1];
+ Attrs.MaxNumWGZ = (*Summary.MaxNumWorkgroups)[2];
+ }
+
+ return Attrs;
+}
+
+/// Propagate AMDGPU kernel occupancy attributes (flat-work-group-size,
+/// waves-per-eu, max-num-workgroups) top-down from kernels to device functions
+/// through the combined index call graph. Sets a backend callback on \p Conf
+/// to apply the propagated attributes when each module is compiled.
+static void
+thinLTOPropagateAMDGPUAttributes(const ModuleSummaryIndex &CombinedIndex,
+ lto::Config &Conf) {
+ const AMDGPUSummaryMap &AMDGPUSummaries = CombinedIndex.getAMDGPUSummaries();
+ if (AMDGPUSummaries.empty())
+ return;
+
+ LLVM_DEBUG(dbgs() << "AMDGPU ThinLTO: read " << AMDGPUSummaries.size()
+ << " function summaries\n");
+
+ AMDGPUPropagatedAttrsMap Propagated;
+ SmallVector<GlobalValue::GUID> KernelGUIDs;
+ for (const auto &[GUID, Summary] : AMDGPUSummaries) {
+ if (Summary.CC == CallingConv::AMDGPU_KERNEL) {
+ KernelGUIDs.push_back(GUID);
+ Propagated[GUID] = getAMDGPUPropagatedAttrs(Summary);
+ }
+ }
+
+ // Per-kernel BFS: propagate each kernel's constraints to all reachable
+ // callees. When a callee is reached by multiple kernels, merge
+ // conservatively. Callees are initialized from the first reaching
+ // kernel (not from their own defaults) to avoid loose defaults
+ // diluting the propagated constraints.
+ for (GlobalValue::GUID KernelGUID : KernelGUIDs) {
+ auto KIt = AMDGPUSummaries.find(KernelGUID);
+ if (KIt == AMDGPUSummaries.end())
+ continue;
+ const AMDGPUPropagatedFunctionAttrs KernelAttrs =
+ getAMDGPUPropagatedAttrs(KIt->second);
+
+ SmallVector<GlobalValue::GUID> Worklist;
+ DenseSet<GlobalValue::GUID> Visited;
+ Worklist.push_back(KernelGUID);
+ Visited.insert(KernelGUID);
+
+ for (unsigned Idx = 0; Idx < Worklist.size(); ++Idx) {
+ GlobalValue::GUID CurGUID = Worklist[Idx];
+ ValueInfo VI = CombinedIndex.getValueInfo(CurGUID);
+ if (!VI)
+ continue;
+
+ for (const auto &Summary : VI.getSummaryList()) {
+ auto *FS = dyn_cast<FunctionSummary>(Summary->getBaseObject());
+ if (!FS)
+ continue;
+ for (const auto &Edge : FS->calls()) {
+ GlobalValue::GUID CalleeGUID = Edge.first.getGUID();
+ if (!Visited.insert(CalleeGUID).second)
+ continue;
+ Worklist.push_back(CalleeGUID);
+
+ auto [It, Inserted] = Propagated.try_emplace(CalleeGUID);
+ if (Inserted) {
+ auto CalleeIt = AMDGPUSummaries.find(CalleeGUID);
+ if (CalleeIt != AMDGPUSummaries.end())
+ It->second = getAMDGPUPropagatedAttrs(CalleeIt->second);
+ else
+ It->second = KernelAttrs;
+ }
+ It->second.mergeFrom(KernelAttrs);
+ }
+ }
+ }
+ }
+
+ LLVM_DEBUG({
+ dbgs() << "AMDGPU ThinLTO: propagated to " << Propagated.size()
+ << " functions\n";
+ for (auto &[GUID, Attrs] : Propagated) {
+ dbgs() << " GUID=" << GUID << " flat_wg=[" << Attrs.FlatWGSizeMin << ","
+ << Attrs.FlatWGSizeMax << "]"
+ << " waves=[" << Attrs.WavesPerEUMin << ",";
+ if (Attrs.WavesPerEUMax)
+ dbgs() << *Attrs.WavesPerEUMax;
+ else
+ dbgs() << "target";
+ dbgs() << "]"
+ << " max_wg=[" << Attrs.MaxNumWGX << "," << Attrs.MaxNumWGY << ","
+ << Attrs.MaxNumWGZ << "]"
+ << (Attrs.CC == CallingConv::AMDGPU_KERNEL ? " [kernel]" : "")
+ << "\n";
+ }
+ });
+
+ auto PropagatedPtr =
+ std::make_shared<AMDGPUPropagatedAttrsMap>(std::move(Propagated));
+ Conf.ApplyThinLTOAttributes = [PropagatedPtr](unsigned /*Task*/, Module &M) {
+ for (Function &F : M) {
+ if (F.isDeclaration())
+ continue;
+ auto It = PropagatedPtr->find(F.getGUID());
+ if (It == PropagatedPtr->end())
+ continue;
+ const AMDGPUPropagatedFunctionAttrs &Attrs = It->second;
+
+ if (F.getCallingConv() == CallingConv::AMDGPU_KERNEL)
+ continue;
+
+ if (!Attrs.hasValidWavesPerEU()) {
+ F.getContext().emitError(
+ "amdgpu ThinLTO: contradictory waves-per-eu constraints "
+ "propagated to function '" +
+ F.getName() + "': min=" + Twine(Attrs.WavesPerEUMin) +
+ " > max=" + Twine(*Attrs.WavesPerEUMax));
+ continue;
+ }
+
+ std::string Val;
+ Val = std::to_string(Attrs.FlatWGSizeMin) + "," +
+ std::to_string(Attrs.FlatWGSizeMax);
+ F.addFnAttr("amdgpu-flat-work-group-size", Val);
+
+ Val = std::to_string(Attrs.WavesPerEUMin);
+ if (Attrs.WavesPerEUMax)
+ Val += "," + std::to_string(*Attrs.WavesPerEUMax);
+ F.addFnAttr("amdgpu-waves-per-eu", Val);
+
+ Val = std::to_string(Attrs.MaxNumWGX) + "," +
+ std::to_string(Attrs.MaxNumWGY) + "," +
+ std::to_string(Attrs.MaxNumWGZ);
+ F.addFnAttr("amdgpu-max-num-workgroups", Val);
+ }
+ };
+}
+
+} // end anonymous namespace
+
Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache,
const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
llvm::TimeTraceScope timeScope("Run ThinLTO");
@@ -2246,6 +2439,8 @@ Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache,
thinLTOPropagateFunctionAttrs(ThinLTO.CombinedIndex, isPrevailing);
+ thinLTOPropagateAMDGPUAttributes(ThinLTO.CombinedIndex, Conf);
+
generateParamAccessSummary(ThinLTO.CombinedIndex);
if (llvm::timeTraceProfilerEnabled())
diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp
index 69bc3fdae6c57..7739c41407660 100644
--- a/llvm/lib/LTO/LTOBackend.cpp
+++ b/llvm/lib/LTO/LTOBackend.cpp
@@ -781,6 +781,9 @@ Error lto::thinBackend(const Config &Conf, unsigned Task, AddStreamFn AddStream,
if (Conf.PostImportModuleHook && !Conf.PostImportModuleHook(Task, Mod))
return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
+ if (Conf.ApplyThinLTOAttributes)
+ Conf.ApplyThinLTOAttributes(Task, Mod);
+
return OptimizeAndCodegen(Mod, TM.get(), std::move(DiagnosticOutputFile));
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
index 7afda107a59bc..6e03071f64634 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
@@ -449,7 +449,7 @@ const AMDGPUMCExpr *createOccupancy(unsigned InitOcc, const MCExpr *NumSGPRs,
}
void AMDGPUAsmPrinter::validateMCResourceInfo(Function &F) {
- if (F.isDeclaration() || !AMDGPU::isModuleEntryFunctionCC(F.getCallingConv()))
+ if (F.isDeclaration())
return;
using RIK = MCResourceInfo::ResourceInfoKind;
@@ -465,6 +465,43 @@ void AMDGPUAsmPrinter::validateMCResourceInfo(Function &F) {
return false;
};
+ // Register allocation normally respects the ABI register budget. Final
+ // resource accounting can still exceed it when the IR names fixed physical
+ // registers, e.g. inline asm clobbers. Check the resolved per-function
+ // resource symbols here so object-linking metadata is not emitted for a
+ // function that cannot satisfy its ABI occupancy.
+ if (AMDGPUTargetMachine::EnableObjectLinking) {
+ const unsigned BudgetVGPR = STM.getMaxNumVGPRs(F);
+ const unsigned BudgetSGPR = STM.getMaxNumSGPRs(F);
+ const unsigned BudgetAGPR = STM.getMaxNumAGPRs(F);
+
+ auto CheckBudget = [&](RIK Kind, unsigned Budget, const char *What) {
+ MCSymbol *Sym = RI.getSymbol(FnSym->getName(), Kind, OutContext);
+ uint64_t Used;
+ if (!Sym->isVariable() ||
+ !TryGetMCExprValue(Sym->getVariableValue(), Used))
+ return false;
+ if (Used <= Budget)
+ return false;
+ F.getContext().diagnose(DiagnosticInfoResourceLimit(
+ F, What, Used, Budget, DS_Error, DK_ResourceLimit));
+ return true;
+ };
+
+ if (CheckBudget(RIK::RIK_NumVGPR, BudgetVGPR,
+ "VGPRs under object-linking ABI"))
+ return;
+ if (CheckBudget(RIK::RIK_NumAGPR, BudgetAGPR,
+ "AGPRs under object-linking ABI"))
+ return;
+ if (CheckBudget(RIK::RIK_NumSGPR, BudgetSGPR,
+ "SGPRs under object-linking ABI"))
+ return;
+ }
+
+ if (!AMDGPU::isModuleEntryFunctionCC(F.getCallingConv()))
+ return;
+
const uint64_t MaxScratchPerWorkitem =
STM.getMaxWaveScratchSize() / STM.getWavefrontSize();
MCSymbol *ScratchSizeSymbol =
@@ -536,20 +573,17 @@ void AMDGPUAsmPrinter::validateMCResourceInfo(Function &F) {
TryGetMCExprValue(NumVgprSymbol->getVariableValue(), NumVgpr) &&
TryGetMCExprValue(NumAgprSymbol->getVariableValue(), NumAgpr)) {
const SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>();
- unsigned MaxWaves = MFI.getMaxWavesPerEU();
+ unsigned MaxWaves = STM.getWavesPerEU(F).second;
uint64_t TotalNumVgpr =
getTotalNumVGPRs(STM.hasGFX90AInsts(), NumAgpr, NumVgpr);
- uint64_t NumVGPRsForWavesPerEU =
- std::max({TotalNumVgpr, (uint64_t)1,
- (uint64_t)STM.getMinNumVGPRs(
- MaxWaves, MFI.getDynamicVGPRBlockSize())});
- uint64_t NumSGPRsForWavesPerEU = std::max(
- {NumSgpr, (uint64_t)1, (uint64_t)STM.getMinNumSGPRs(MaxWaves)});
- const MCExpr *OccupancyExpr = createOccupancy(
- STM.getOccupancyWithWorkGroupSizes(*MF).second,
- MCConstantExpr::create(NumSGPRsForWavesPerEU, OutContext),
- MCConstantExpr::create(NumVGPRsForWavesPerEU, OutContext),
- MFI.getDynamicVGPRBlockSize(), STM, OutContext);
+ auto [EffectiveSGPRs, EffectiveVGPRs] =
+ STM.getEffectiveNumGPRsForWavesPerEU(MaxWaves, NumSgpr, TotalNumVgpr,
+ MFI.getDynamicVGPRBlockSize());
+ const MCExpr *OccupancyExpr =
+ createOccupancy(STM.getOccupancyWithWorkGroupSizes(*MF).second,
+ MCConstantExpr::create(EffectiveSGPRs, OutContext),
+ MCConstantExpr::create(EffectiveVGPRs, OutContext),
+ MFI.getDynamicVGPRBlockSize(), STM, OutContext);
uint64_t Occupancy;
const auto [MinWEU, MaxWEU] = AMDGPU::getIntegerPairAttribute(
@@ -720,30 +754,25 @@ bool AMDGPUAsmPrinter::doFinalization(Module &M) {
// LDS/named-barrier use edges, indirect calls, and address-taken type IDs).
emitAMDGPUInfo(M);
- // Assign expressions which can only be resolved when all other functions are
- // known.
- RI.finalize(OutContext);
-
- // Switch section and emit all GPR maximums within the processed module.
- OutStreamer->pushSection();
- MCSectionELF *MaxGPRSection =
- OutContext.getELFSection(".AMDGPU.gpr_maximums", ELF::SHT_PROGBITS, 0);
- OutStreamer->switchSection(MaxGPRSection);
- getTargetStreamer()->EmitMCResourceMaximums(
- RI.getMaxVGPRSymbol(OutContext), RI.getMaxAGPRSymbol(OutContext),
- RI.getMaxSGPRSymbol(OutContext), RI.getMaxNamedBarrierSymbol(OutContext));
- OutStreamer->popSection();
-
- // In the object-linking pipeline per-function resource MCExprs reference
- // external callee symbols that cannot be evaluated here, so cross-TU limit
- // checks would silently no-op for every non-leaf function. Defer resource
- // sanity checking to the linker, which re-validates against the aggregated
- // call graph in the combined .amdgpu.info metadata.
+ // Finalize non-object-linking resource propagation and emit the
+ // `amdgpu.max_num_*` fallback symbols.
if (!AMDGPUTargetMachine::EnableObjectLinking) {
- for (Function &F : M.functions())
- validateMCResourceInfo(F);
+ RI.finalize(OutContext);
+
+ OutStreamer->pushSection();
+ MCSectionELF *MaxGPRSection =
+ OutContext.getELFSection(".AMDGPU.gpr_maximums", ELF::SHT_PROGBITS, 0);
+ OutStreamer->switchSection(MaxGPRSection);
+ getTargetStreamer()->EmitMCResourceMaximums(
+ RI.getMaxVGPRSymbol(OutContext), RI.getMaxAGPRSymbol(OutContext),
+ RI.getMaxSGPRSymbol(OutContext),
+ RI.getMaxNamedBarrierSymbol(OutContext));
+ OutStreamer->popSection();
}
+ for (Function &F : M.functions())
+ validateMCResourceInfo(F);
+
RI.reset();
return AsmPrinter::doFinalization(M);
@@ -905,11 +934,16 @@ bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
if (AMDGPUTargetMachine::EnableObjectLinking) {
const AMDGPUResourceUsageAnalysisWrapperPass::FunctionResourceInfo &RU =
*ResourceUsage;
+ uint32_t Occupancy =
+ static_cast<uint32_t>(STM.getWavesPerEU(MF.getFunction()).first);
FunctionInfos.push_back(
{/*NumSGPR=*/static_cast<uint32_t>(RU.NumExplicitSGPR),
/*NumArchVGPR=*/static_cast<uint32_t>(RU.NumVGPR),
/*NumAccVGPR=*/static_cast<uint32_t>(RU.NumAGPR),
/*PrivateSegmentSize=*/static_cast<uint32_t>(RU.PrivateSegmentSize),
+ /*Occupancy=*/Occupancy,
+ /*UsesWgpMode=*/STM.hasSupportsWGP() && !STM.isCuModeEnabled(),
+ /*UsesWave32=*/STM.getWavefrontSize() == 32,
/*UsesVCC=*/RU.UsesVCC,
/*UsesFlatScratch=*/RU.UsesFlatScratch,
/*HasDynStack=*/RU.HasDynamicallySizedStack,
@@ -1334,7 +1368,7 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
// Adjust number of registers used to meet default/requested minimum/maximum
// number of waves per execution unit request.
- unsigned MaxWaves = MFI->getMaxWavesPerEU();
+ unsigned MaxWaves = STM.getWavesPerEU(MF.getFunction()).second;
ProgInfo.NumSGPRsForWavesPerEU =
AMDGPUMCExpr::createMax({ProgInfo.NumSGPR, CreateExpr(1ul),
CreateExpr(STM.getMinNumSGPRs(MaxWaves))},
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
index 900b5d6717619..571b5430b973e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
@@ -514,8 +514,7 @@ MetadataStreamerMsgPackV4::getHSAKernelProps(const MachineFunction &MF,
// FIXME: The metadata treats the minimum as 16?
Kern[".kernarg_segment_align"] =
Kern.getDocument()->getNode(std::max(Align(4), MaxKernArgAlign).value());
- Kern[".wavefront_size"] =
- Kern.getDocument()->getNode(STM.getWavefrontSize());
+ Kern[".wavefront_size"] = Kern.getDocument()->getNode(STM.getWavefrontSize());
DelayedExprs->assignDocNode(Kern[".sgpr_count"], msgpack::Type::UInt,
ProgramInfo.NumSGPR);
DelayedExprs->assignDocNode(Kern[".vgpr_count"], msgpack::Type::UInt,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerExecSync.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerExecSync.cpp
index c2a937d51588f..032c97b6786a7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerExecSync.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerExecSync.cpp
@@ -191,8 +191,7 @@ static bool handleNamedBarriersForObjectLinking(Module &M) {
V->setName("__amdgpu_named_barrier." + V->getName() + ModuleId);
else if (!V->getName().starts_with("__amdgpu_named_barrier"))
V->setName("__amdgpu_named_barrier." + V->getName());
- V->setInitializer(nullptr);
- V->setLinkage(GlobalValue::ExternalLinkage);
+ externalizeLDSGlobalForObjectLinking(*V);
SmallVector<Metadata *, 4> Ops;
Ops.push_back(ValueAsMetadata::get(V));
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
index 2090981de4373..359ee68e99fd3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
@@ -985,8 +985,6 @@ class AMDGPULowerModuleLDS {
createLDSVariableReplacement(M, StructName, FuncScopeVars);
GlobalVariable *SGV = Replacement.SGV;
- SGV->setLinkage(GlobalValue::ExternalLinkage);
- SGV->setInitializer(nullptr);
FuncToLdsStruct.push_back({F, SGV});
replaceLDSVariablesWithStruct(
@@ -1008,8 +1006,6 @@ class AMDGPULowerModuleLDS {
createLDSVariableReplacement(M, StructName, InternalMultiUserVars);
GlobalVariable *SGV = Replacement.SGV;
- SGV->setLinkage(GlobalValue::ExternalLinkage);
- SGV->setInitializer(nullptr);
replaceLDSVariablesWithStruct(
M, InternalMultiUserVars, Replacement,
@@ -1027,29 +1023,14 @@ class AMDGPULowerModuleLDS {
InternalMultiUserVars.end());
}
- // Convert global-scope LDS to external declarations. Their uses remain
- // intact and ISel generates R_AMDGPU_ABS32_LO relocations for them.
- for (GlobalVariable *V : GlobalScopeVars) {
- V->setInitializer(nullptr);
- V->setLinkage(GlobalValue::ExternalLinkage);
- }
-
- // Emit amdgpu.lds.uses metadata for struct and global-scope LDS.
- {
- LLVMContext &Ctx = M.getContext();
- NamedMDNode *LdsMD = M.getOrInsertNamedMetadata("amdgpu.lds.uses");
+ // Externalize LDS allocations and record their function-use edges.
+ for (auto &[F, SGV] : FuncToLdsStruct)
+ recordLDSUseForObjectLinking(*F, *SGV);
- for (auto &[F, SGV] : FuncToLdsStruct)
- LdsMD->addOperand(MDNode::get(
- Ctx, {ValueAsMetadata::get(F), ValueAsMetadata::get(SGV)}));
-
- for (auto &[V, Funcs] : VarToFuncs) {
- if (GlobalScopeVars.count(V) && !InternalMultiUserVars.count(V)) {
- for (Function *F : Funcs) {
- LdsMD->addOperand(MDNode::get(
- Ctx, {ValueAsMetadata::get(F), ValueAsMetadata::get(V)}));
- }
- }
+ for (auto &[V, Funcs] : VarToFuncs) {
+ if (GlobalScopeVars.count(V) && !InternalMultiUserVars.count(V)) {
+ for (Function *F : Funcs)
+ recordLDSUseForObjectLinking(*F, *V);
}
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.cpp
index 1731b0463a099..0d5b7484942c9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.cpp
@@ -13,6 +13,8 @@
//===----------------------------------------------------------------------===//
#include "AMDGPUMCResourceInfo.h"
+#include "AMDGPUTargetMachine.h"
+#include "GCNSubtarget.h"
#include "SIMachineFunctionInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/ADT/StringRef.h"
@@ -262,7 +264,8 @@ void MCResourceInfo::gatherResourceInfo(
MF.getInfo<SIMachineFunctionInfo>()->isDynamicVGPREnabled();
if (!AMDGPU::isEntryFunctionCC(CC)) {
- if (!IsDynamicVGPREnabled || !IsChainCC)
+ if ((!IsDynamicVGPREnabled || !IsChainCC) &&
+ !AMDGPUTargetMachine::EnableObjectLinking)
addMaxVGPRCandidate(FRI.NumVGPR);
addMaxAGPRCandidate(FRI.NumAGPR);
addMaxSGPRCandidate(FRI.NumExplicitSGPR);
@@ -285,9 +288,35 @@ void MCResourceInfo::gatherResourceInfo(
if (AMDGPUTargetMachine::EnableObjectLinking) {
LLVM_DEBUG(dbgs() << "MCResUse: object linking enabled, no call-graph "
"propagation; emitting local resource values only\n");
- SetToLocal(FRI.NumVGPR, RIK_NumVGPR);
+ const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+ unsigned MaxWaves = ST.getWavesPerEU(MF.getFunction()).second;
+ unsigned DynamicVGPRBlockSize =
+ AMDGPU::getDynamicVGPRBlockSize(MF.getFunction());
+
+ uint64_t TotalVGPR =
+ AMDGPU::getTotalNumVGPRs(ST.hasGFX90AInsts(), FRI.NumAGPR, FRI.NumVGPR);
+ unsigned ExtraSGPRs =
+ AMDGPU::IsaInfo::getNumExtraSGPRs(ST, FRI.UsesVCC, FRI.UsesFlatScratch,
+ ST.getTargetID().isXnackOnOrAny());
+ uint64_t TotalSGPR = FRI.NumExplicitSGPR + ExtraSGPRs;
+ auto [EffectiveSGPRs, EffectiveVGPRs] = ST.getEffectiveNumGPRsForWavesPerEU(
+ MaxWaves, TotalSGPR, TotalVGPR, DynamicVGPRBlockSize);
+
+ int32_t EffectiveNumVGPR = FRI.NumVGPR;
+ if (EffectiveVGPRs > TotalVGPR) {
+ if (ST.hasGFX90AInsts())
+ EffectiveNumVGPR = static_cast<int32_t>(EffectiveVGPRs - FRI.NumAGPR);
+ else
+ EffectiveNumVGPR = static_cast<int32_t>(EffectiveVGPRs);
+ }
+
+ int32_t EffectiveNumSGPR = FRI.NumExplicitSGPR;
+ if (EffectiveSGPRs > TotalSGPR)
+ EffectiveNumSGPR += static_cast<int32_t>(EffectiveSGPRs - TotalSGPR);
+
+ SetToLocal(EffectiveNumVGPR, RIK_NumVGPR);
SetToLocal(FRI.NumAGPR, RIK_NumAGPR);
- SetToLocal(FRI.NumExplicitSGPR, RIK_NumSGPR);
+ SetToLocal(EffectiveNumSGPR, RIK_NumSGPR);
SetToLocal(FRI.NumNamedBarrier, RIK_NumNamedBarrier);
SetToLocal(FRI.PrivateSegmentSize, RIK_PrivateSegSize);
SetToLocal(FRI.UsesVCC, RIK_UsesVCC);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
index a560df6f2d911..dac9af4f49fcf 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
@@ -17,6 +17,8 @@
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Metadata.h"
+#include "llvm/IR/Module.h"
#include "llvm/IR/ReplaceConstant.h"
#define DEBUG_TYPE "amdgpu-memory-utils"
@@ -110,6 +112,23 @@ bool isLDSVariableToLower(const GlobalVariable &GV) {
return true;
}
+void externalizeLDSGlobalForObjectLinking(GlobalVariable &GV) {
+ assert(GV.getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS);
+ GV.setInitializer(nullptr);
+ GV.setLinkage(GlobalValue::ExternalLinkage);
+ GV.setComdat(nullptr);
+ GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None);
+}
+
+void recordLDSUseForObjectLinking(Function &F, GlobalVariable &GV) {
+ assert(F.getParent() == GV.getParent());
+ externalizeLDSGlobalForObjectLinking(GV);
+ NamedMDNode *LDSMD =
+ F.getParent()->getOrInsertNamedMetadata("amdgpu.lds.uses");
+ LDSMD->addOperand(MDNode::get(
+ F.getContext(), {ValueAsMetadata::get(&F), ValueAsMetadata::get(&GV)}));
+}
+
bool eliminateGVConstantExprUsesFromAllInstructions(
Module &M, function_ref<bool(const GlobalVariable &)> Filter) {
SmallVector<Constant *> Worklist;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
index 4e164d08549dd..025e0885ffe78 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
@@ -46,6 +46,12 @@ TargetExtType *isNamedBarrier(const GlobalVariable &GV);
bool isDynamicLDS(const GlobalVariable &GV);
bool isLDSVariableToLower(const GlobalVariable &GV);
+/// Convert \p GV to an external LDS declaration for link-time allocation.
+void externalizeLDSGlobalForObjectLinking(GlobalVariable &GV);
+
+/// Externalize \p GV and record that \p F uses it.
+void recordLDSUseForObjectLinking(Function &F, GlobalVariable &GV);
+
struct GVUsesInfoTy {
FunctionVariableMap DirectAccess;
FunctionVariableMap IndirectAccess;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index 3c1730397ab96..2b000a5d8ebbb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -26,6 +26,8 @@
//===----------------------------------------------------------------------===//
#include "AMDGPU.h"
+#include "AMDGPUMemoryUtils.h"
+#include "AMDGPUTargetMachine.h"
#include "GCNSubtarget.h"
#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/ADT/STLExtras.h"
@@ -44,6 +46,7 @@
#include "llvm/Pass.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetMachine.h"
+#include "llvm/Transforms/Utils/ModuleUtils.h"
#include "llvm/Transforms/Utils/SSAUpdater.h"
#define DEBUG_TYPE "amdgpu-promote-alloca"
@@ -1633,7 +1636,17 @@ bool AMDGPUPromoteAllocaImpl::tryPromoteAllocaToLDS(
Mod, GVTy, false, GlobalValue::InternalLinkage, PoisonValue::get(GVTy),
Twine(F->getName()) + Twine('.') + AA.Alloca->getName(), nullptr,
GlobalVariable::NotThreadLocal, AMDGPUAS::LOCAL_ADDRESS);
- GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
+ if (AMDGPUTargetMachine::EnableObjectLinking) {
+ if (F->hasLocalLinkage()) {
+ std::string ModuleId = getUniqueModuleId(&Mod);
+ assert(!ModuleId.empty() &&
+ "modules with promoted LDS allocas should have a unique ID");
+ GV->setName(GV->getName() + ModuleId);
+ }
+ AMDGPU::recordLDSUseForObjectLinking(*F, *GV);
+ } else {
+ GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
+ }
GV->setAlignment(AA.Alloca->getAlign());
Value *TCntY, *TCntZ;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
index b9ba99cf8db19..94eab60abfdb5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
@@ -16,6 +16,7 @@
#include "AMDGPUInstructionSelector.h"
#include "AMDGPULegalizerInfo.h"
#include "AMDGPURegisterBankInfo.h"
+#include "AMDGPUTargetMachine.h"
#include "R600Subtarget.h"
#include "SIMachineFunctionInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
@@ -25,6 +26,7 @@
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
#include "llvm/IR/MDBuilder.h"
+#include "llvm/Support/MathExtras.h"
#include <algorithm>
using namespace llvm;
@@ -40,7 +42,7 @@ AMDGPUSubtarget::getMaxLocalMemSizeWithWaveCount(unsigned NWaves,
const unsigned WaveSize = getWavefrontSize();
const unsigned WorkGroupSize = getFlatWorkGroupSizes(F).second;
const unsigned WavesPerWorkgroup =
- std::max(1u, (WorkGroupSize + WaveSize - 1) / WaveSize);
+ std::max(1u, divideCeil(WorkGroupSize, WaveSize));
const unsigned WorkGroupsPerCU =
std::max(1u, (NWaves * getNumWorkGroupSIMDs()) / WavesPerWorkgroup);
@@ -224,7 +226,35 @@ AMDGPUSubtarget::getWavesPerEU(std::pair<unsigned, unsigned> FlatWorkGroupSizes,
// Requested minimum/maximum number of waves per execution unit.
std::pair<unsigned, unsigned> Requested =
AMDGPU::getIntegerPairAttribute(F, "amdgpu-waves-per-eu", Default, true);
- return getEffectiveWavesPerEU(Requested, FlatWorkGroupSizes, LDSBytes);
+ std::pair<unsigned, unsigned> WavesPerEU =
+ getEffectiveWavesPerEU(Requested, FlatWorkGroupSizes, LDSBytes);
+ if (!AMDGPUTargetMachine::EnableObjectLinking)
+ return WavesPerEU;
+
+ unsigned FunctionABIOccupancy = getFunctionABIOccupancy(F);
+ unsigned EffectiveMinWaves =
+ F.hasFnAttribute("amdgpu-waves-per-eu")
+ ? std::max(WavesPerEU.first, FunctionABIOccupancy)
+ : FunctionABIOccupancy;
+ return {EffectiveMinWaves, std::max(WavesPerEU.second, EffectiveMinWaves)};
+}
+
+unsigned AMDGPUSubtarget::getDefaultABIOccupancy(const Module &M) const {
+ unsigned Override = AMDGPU::getAMDGPUABIWavesPerEU(M);
+ if (Override)
+ Override = std::clamp(Override, getMinWavesPerEU(), getMaxWavesPerEU());
+
+ constexpr unsigned DefaultMaxWorkGroupSize = 1024;
+ return Override ? Override
+ : getWavesPerEUForWorkGroup(DefaultMaxWorkGroupSize);
+}
+
+unsigned AMDGPUSubtarget::getFunctionABIOccupancy(const Function &F) const {
+ if (!F.hasFnAttribute("amdgpu-flat-work-group-size"))
+ return getDefaultABIOccupancy(*F.getParent());
+
+ std::pair<unsigned, unsigned> FlatWorkGroupSizes = getFlatWorkGroupSizes(F);
+ return getWavesPerEUForWorkGroup(FlatWorkGroupSizes.second);
}
std::optional<unsigned>
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
index e1f331229c463..13f121452c8b1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
@@ -25,6 +25,7 @@ enum AMDGPUDwarfFlavour : unsigned;
class Function;
class Instruction;
class MachineFunction;
+class Module;
class TargetMachine;
class AMDGPUSubtarget {
@@ -130,6 +131,13 @@ class AMDGPUSubtarget {
std::pair<unsigned, unsigned> FlatWorkGroupSizes,
unsigned LDSBytes) const;
+ /// \returns The default ABI occupancy for \p M.
+ unsigned getDefaultABIOccupancy(const Module &M) const;
+
+ /// \returns The ABI occupancy for \p F. A function-specific flat-workgroup
+ /// attribute overrides the default ABI occupancy.
+ unsigned getFunctionABIOccupancy(const Function &F) const;
+
/// Return the amount of LDS that can be used that will not restrict the
/// occupancy lower than WaveCount.
unsigned getMaxLocalMemSizeWithWaveCount(unsigned WaveCount,
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index 81bcc4ffaa286..03792413b1478 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -6903,6 +6903,8 @@ bool AMDGPUAsmParser::ParseDirectiveAMDGPUInfo() {
FI.UsesFlatScratch =
!!(Flags & AMDGPU::FuncInfoFlags::FUNC_USES_FLAT_SCRATCH);
FI.HasDynStack = !!(Flags & AMDGPU::FuncInfoFlags::FUNC_HAS_DYN_STACK);
+ FI.UsesWgpMode = !!(Flags & AMDGPU::FuncInfoFlags::FUNC_WGP_MODE);
+ FI.UsesWave32 = !!(Flags & AMDGPU::FuncInfoFlags::FUNC_WAVE32);
HasScalarAttrs = true;
} else if (Dir == "num_sgpr") {
int64_t Val;
@@ -6928,6 +6930,12 @@ bool AMDGPUAsmParser::ParseDirectiveAMDGPUInfo() {
return true;
FI.PrivateSegmentSize = static_cast<uint32_t>(Val);
HasScalarAttrs = true;
+ } else if (Dir == "occupancy") {
+ int64_t Val;
+ if (getParser().parseAbsoluteExpression(Val))
+ return true;
+ FI.Occupancy = static_cast<uint32_t>(Val);
+ HasScalarAttrs = true;
} else if (Dir == "use") {
StringRef ResName;
if (getParser().parseIdentifier(ResName))
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp b/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp
index 9443420c3d424..5fb8f1d95d5fb 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp
@@ -534,6 +534,16 @@ GCNSubtarget::computeOccupancy(const Function &F, unsigned LDSSize,
return {std::min(MinOcc, MaxOcc), MaxOcc};
}
+std::pair<uint64_t, uint64_t> GCNSubtarget::getEffectiveNumGPRsForWavesPerEU(
+ unsigned MaxWaves, uint64_t NumSGPRs, uint64_t NumVGPRs,
+ unsigned DynamicVGPRBlockSize) const {
+ // These are occupancy-reporting counts, not literal register usage. Clamp to
+ // at least one register because zero does not constrain occupancy.
+ return {std::max({NumSGPRs, uint64_t(1), uint64_t(getMinNumSGPRs(MaxWaves))}),
+ std::max({NumVGPRs, uint64_t(1),
+ uint64_t(getMinNumVGPRs(MaxWaves, DynamicVGPRBlockSize))})};
+}
+
unsigned GCNSubtarget::getBaseMaxNumSGPRs(
const Function &F, std::pair<unsigned, unsigned> WavesPerEU,
unsigned PreloadedSGPRs, unsigned ReservedNumSGPRs) const {
@@ -584,7 +594,7 @@ unsigned GCNSubtarget::getBaseMaxNumSGPRs(
unsigned GCNSubtarget::getMaxNumSGPRs(const MachineFunction &MF) const {
const Function &F = MF.getFunction();
const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
- return getBaseMaxNumSGPRs(F, MFI.getWavesPerEU(), MFI.getNumPreloadedSGPRs(),
+ return getBaseMaxNumSGPRs(F, getWavesPerEU(F), MFI.getNumPreloadedSGPRs(),
getReservedNumSGPRs(MF));
}
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index f06fccec9e264..06cc4e41a70fd 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -23,6 +23,7 @@
#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/Support/AMDHSAKernelDescriptor.h"
#include "llvm/Support/ErrorHandling.h"
+#include <cstdint>
#include <optional>
#define GET_SUBTARGETINFO_HEADER
@@ -715,6 +716,22 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
unsigned NumSGPRs = 0,
unsigned NumVGPRs = 0) const;
+ /// \returns {SGPRs, VGPRs} resource counts for occupancy-adjusted reporting.
+ ///
+ /// This is not the function's literal register usage. It is the count to use
+ /// when resource metadata or kernel descriptor fields need to encode the
+ /// effect of capping occupancy to at most \p MaxWaves waves per EU. If the
+ /// real usage is below the threshold needed to enforce that cap, the returned
+ /// count is inflated to the minimum count that does enforce it.
+ ///
+ /// The result is also clamped to at least one register in each bank. Zero
+ /// means "no register pressure" to the occupancy calculation, so it cannot
+ /// express an occupancy cap even for a function whose literal usage is zero.
+ std::pair<uint64_t, uint64_t>
+ getEffectiveNumGPRsForWavesPerEU(unsigned MaxWaves, uint64_t NumSGPRs,
+ uint64_t NumVGPRs,
+ unsigned DynamicVGPRBlockSize) const;
+
/// \returns true if the flat_scratch register should be initialized with the
/// pointer to the wave's scratch memory rather than a size and offset.
bool flatScratchIsPointer() const {
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
index be162919c46ce..52019fa82105a 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
@@ -766,6 +766,10 @@ void AMDGPUTargetAsmStreamer::emitAMDGPUInfo(
Flags |= AMDGPU::FuncInfoFlags::FUNC_USES_FLAT_SCRATCH;
if (Info->HasDynStack)
Flags |= AMDGPU::FuncInfoFlags::FUNC_HAS_DYN_STACK;
+ if (Info->UsesWgpMode)
+ Flags |= AMDGPU::FuncInfoFlags::FUNC_WGP_MODE;
+ if (Info->UsesWave32)
+ Flags |= AMDGPU::FuncInfoFlags::FUNC_WAVE32;
OS << "\t\t.amdgpu_flags " << llvm::to_underlying(Flags) << '\n';
OS << "\t\t.amdgpu_num_sgpr " << Info->NumSGPR << '\n';
OS << "\t\t.amdgpu_num_vgpr " << Info->NumArchVGPR << '\n';
@@ -773,6 +777,8 @@ void AMDGPUTargetAsmStreamer::emitAMDGPUInfo(
OS << "\t\t.amdgpu_num_agpr " << Info->NumAccVGPR << '\n';
OS << "\t\t.amdgpu_private_segment_size " << Info->PrivateSegmentSize
<< '\n';
+ if (Info->Occupancy)
+ OS << "\t\t.amdgpu_occupancy " << Info->Occupancy << '\n';
}
for (MCSymbol *Res : Uses)
OS << "\t\t.amdgpu_use " << Res->getName() << '\n';
@@ -1238,6 +1244,10 @@ void AMDGPUTargetELFStreamer::emitAMDGPUInfo(
Flags |= AMDGPU::FuncInfoFlags::FUNC_USES_FLAT_SCRATCH;
if (Info->HasDynStack)
Flags |= AMDGPU::FuncInfoFlags::FUNC_HAS_DYN_STACK;
+ if (Info->UsesWgpMode)
+ Flags |= AMDGPU::FuncInfoFlags::FUNC_WGP_MODE;
+ if (Info->UsesWave32)
+ Flags |= AMDGPU::FuncInfoFlags::FUNC_WAVE32;
EmitU32Entry(AMDGPU::InfoKind::INFO_FLAGS, llvm::to_underlying(Flags));
EmitU32Entry(AMDGPU::InfoKind::INFO_NUM_SGPR, Info->NumSGPR);
EmitU32Entry(AMDGPU::InfoKind::INFO_NUM_VGPR, Info->NumArchVGPR);
@@ -1247,6 +1257,8 @@ void AMDGPUTargetELFStreamer::emitAMDGPUInfo(
EmitU32Entry(AMDGPU::InfoKind::INFO_NUM_AGPR, Info->NumAccVGPR);
EmitU32Entry(AMDGPU::InfoKind::INFO_PRIVATE_SEGMENT_SIZE,
Info->PrivateSegmentSize);
+ if (Info->Occupancy)
+ EmitU32Entry(AMDGPU::InfoKind::INFO_OCCUPANCY, Info->Occupancy);
}
for (MCSymbol *Res : Uses)
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h
index d1ef710e9b180..c07a4dd2c5d9b 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h
@@ -23,7 +23,6 @@ class MCSymbol;
class formatted_raw_ostream;
namespace AMDGPU {
-
struct AMDGPUMCKernelCodeT;
struct MCKernelDescriptor;
namespace HSAMD {
@@ -35,6 +34,9 @@ struct FuncInfo {
uint32_t NumArchVGPR = 0;
uint32_t NumAccVGPR = 0;
uint32_t PrivateSegmentSize = 0;
+ uint32_t Occupancy = 0;
+ bool UsesWgpMode = false;
+ bool UsesWave32 = false;
bool UsesVCC = false;
bool UsesFlatScratch = false;
bool HasDynStack = false;
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index a0648da67c06d..08763cf8f4123 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -219,6 +219,14 @@ unsigned getAMDHSACodeObjectVersion(const Module &M) {
return getDefaultAMDHSACodeObjectVersion();
}
+unsigned getAMDGPUABIWavesPerEU(const Module &M) {
+ if (auto *Val = mdconst::extract_or_null<ConstantInt>(
+ M.getModuleFlag("amdgpu_abi_waves_per_eu")))
+ return Val->getZExtValue();
+
+ return 0;
+}
+
unsigned getDefaultAMDHSACodeObjectVersion() {
return DefaultAMDHSACodeObjectVersion;
}
@@ -1541,6 +1549,7 @@ unsigned getAllocatedNumVGPRBlocks(const MCSubtargetInfo &STI,
NumVGPRs,
getVGPRAllocGranule(STI, DynamicVGPRBlockSize, EnableWavefrontSize32));
}
+
} // end namespace IsaInfo
void initDefaultAMDKernelCodeT(AMDGPUMCKernelCodeT &KernelCode,
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index d429584b00999..49b29d7b327aa 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -77,6 +77,9 @@ bool isHsaAbi(const MCSubtargetInfo &STI);
/// \returns Code object version from the IR module flag.
unsigned getAMDHSACodeObjectVersion(const Module &M);
+/// \returns ABI occupancy override in waves per EU from the IR module flag.
+unsigned getAMDGPUABIWavesPerEU(const Module &M);
+
/// \returns Code object version from ELF's e_ident[EI_ABIVERSION].
unsigned getAMDHSACodeObjectVersion(unsigned ABIVersion);
diff --git a/llvm/lib/TargetParser/AMDGPUTargetParser.cpp b/llvm/lib/TargetParser/AMDGPUTargetParser.cpp
index f2ed844347330..e73e2907586a4 100644
--- a/llvm/lib/TargetParser/AMDGPUTargetParser.cpp
+++ b/llvm/lib/TargetParser/AMDGPUTargetParser.cpp
@@ -14,8 +14,10 @@
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringTable.h"
#include "llvm/ADT/Twine.h"
+#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/TargetParser/Triple.h"
+#include <algorithm>
#include <array>
#include <cassert>
@@ -449,6 +451,166 @@ unsigned AMDGPU::getMaxWavesPerEU(Triple::SubArchType SubArch) {
return getMaxWavesPerEU(getGPUKindFromSubArch(SubArch));
}
+static bool hasFeature(const AMDGPU::TargetID &Target,
+ AMDGPU::AMDGPUFeature Feature) {
+ return AMDGPU::getFeatureBitset(Target.getGPUKind()).test(Feature);
+}
+
+static AMDGPU::IsaVersion getVersion(const AMDGPU::TargetID &Target) {
+ return AMDGPU::getIsaVersion(AMDGPU::getArchNameAMDGCN(Target.getGPUKind()));
+}
+
+static bool isGFX10Plus(const AMDGPU::TargetID &Target) {
+ return hasFeature(Target, AMDGPU::FEAT_GFX10_INSTS);
+}
+
+static bool hasAccVGPRs(const AMDGPU::TargetID &Target) {
+ return hasFeature(Target, AMDGPU::FEAT_GFX90A_INSTS);
+}
+
+static bool supportsWGP(const AMDGPU::TargetID &Target) {
+ return hasFeature(Target, AMDGPU::FEAT_SUPPORTS_WGP);
+}
+
+static bool hasArchitectedFlatScratch(const AMDGPU::TargetID &Target) {
+ AMDGPU::IsaVersion Version = getVersion(Target);
+ return Version.Major >= 11 || (Version.Major == 9 && Version.Minor >= 4);
+}
+
+static bool isGFX1250(const AMDGPU::TargetID &Target) {
+ const AMDGPU::AMDGPUFeatureBitset &Features =
+ AMDGPU::getFeatureBitset(Target.getGPUKind());
+ return Features.test(AMDGPU::FEAT_GFX1250_INSTS) &&
+ !Features.test(AMDGPU::FEAT_GFX13_INSTS);
+}
+
+// TargetParser does not expose FeatureHalfAddressablePhysicalLocalMemory. It
+// is set on gfx10/11/12, while gfx12.5 and gfx13 imply FEAT_GFX1250_INSTS.
+static bool
+hasHalfAddressablePhysicalLocalMemory(const AMDGPU::TargetID &Target) {
+ return isGFX10Plus(Target) && !hasFeature(Target, AMDGPU::FEAT_GFX1250_INSTS);
+}
+
+// Keep these byte values in sync with
+// AMDGPU::IsaInfo::getLdsDwGranularity().
+static unsigned getLDSAllocationGranularity(const AMDGPU::TargetID &Target) {
+ switch (AMDGPU::getMaxHWAddressableLocalMemorySize(Target.getGPUKind())) {
+ case 32768:
+ return 256;
+ case 65536:
+ return 512;
+ case 163840:
+ return 1280;
+ case 196608:
+ return 1024;
+ case 327680:
+ return 2048;
+ default:
+ return 256;
+ }
+}
+
+static unsigned getObjectLinkingLocalMemorySize(const AMDGPU::TargetID &Target,
+ bool FullSIMDMode) {
+ unsigned Size =
+ AMDGPU::getMaxHWAddressableLocalMemorySize(Target.getGPUKind());
+ if (hasHalfAddressablePhysicalLocalMemory(Target))
+ Size *= 2;
+ if (!FullSIMDMode)
+ Size /= 2;
+ return Size;
+}
+
+bool AMDGPU::isObjectLinkingWaveSizeSupported(const TargetID &Target,
+ unsigned WaveSize) {
+ if (WaveSize == 32)
+ return hasFeature(Target, FEAT_SUPPORTS_WAVE32);
+ if (WaveSize == 64)
+ return !hasFeature(Target, FEAT_GFX1250_INSTS) ||
+ hasFeature(Target, FEAT_GFX13_INSTS);
+ return false;
+}
+
+static unsigned getVGPREncodingGranule(const AMDGPU::TargetID &Target,
+ unsigned WaveSize) {
+ if (hasAccVGPRs(Target))
+ return 8;
+
+ bool IsWave32 = WaveSize == 32;
+ if (isGFX1250(Target)) {
+ assert(IsWave32 && "gfx12.5 only supports wave32");
+ return 16;
+ }
+
+ return IsWave32 ? 8 : 4;
+}
+
+unsigned AMDGPU::getNumExtraSGPRs(const TargetID &Target, bool VCCUsed,
+ bool FlatScrUsed) {
+ unsigned ExtraSGPRs = VCCUsed ? 2 : 0;
+ if (isGFX10Plus(Target))
+ return ExtraSGPRs;
+
+ if (!hasFeature(Target, FEAT_GFX8_INSTS)) {
+ if (FlatScrUsed)
+ ExtraSGPRs = 4;
+ } else {
+ if (Target.isXnackOnOrAny())
+ ExtraSGPRs = 4;
+
+ if (FlatScrUsed || hasArchitectedFlatScratch(Target))
+ ExtraSGPRs = 6;
+ }
+
+ return ExtraSGPRs;
+}
+
+static unsigned getGranulatedNumRegisterBlocks(unsigned NumRegs,
+ unsigned Granule) {
+ return divideCeil(std::max(1u, NumRegs), Granule);
+}
+
+unsigned AMDGPU::getEncodedNumVGPRBlocks(const TargetID &Target,
+ unsigned NumVGPRs, unsigned WaveSize) {
+ return getGranulatedNumRegisterBlocks(
+ NumVGPRs, getVGPREncodingGranule(Target, WaveSize)) -
+ 1;
+}
+
+unsigned AMDGPU::getNumSGPRBlocks(unsigned NumSGPRs) {
+ // SGPRBlocks is actual number of SGPR blocks minus 1.
+ return getGranulatedNumRegisterBlocks(NumSGPRs, 8) - 1;
+}
+
+bool AMDGPU::isLDSSizeCompatibleWithOccupancy(const TargetID &Target,
+ unsigned WaveSize, bool IsCuMode,
+ uint64_t LDSBytes,
+ unsigned Occupancy) {
+ assert(Occupancy != 0 && Occupancy <= getMaxWavesPerEU(Target.getGPUKind()) &&
+ "invalid occupancy");
+ assert(WaveSize != 0 && "invalid wave size");
+
+ unsigned AddressableLocalMemorySize =
+ getMaxHWAddressableLocalMemorySize(Target.getGPUKind());
+ uint64_t Granularity = getLDSAllocationGranularity(Target);
+ uint64_t AlignedLDSBytes = alignTo(LDSBytes, Granularity);
+ if (AlignedLDSBytes > AddressableLocalMemorySize)
+ return false;
+
+ bool FullSIMDMode = !supportsWGP(Target) || !IsCuMode;
+ unsigned NumSIMDs = getNumWorkGroupSIMDs(FullSIMDMode);
+
+ uint64_t LocalMemorySize =
+ getObjectLinkingLocalMemorySize(Target, FullSIMDMode);
+
+ uint64_t WavesPerWorkgroup = divideCeil(1024u, WaveSize);
+ uint64_t RequiredWavesPerCU = uint64_t(Occupancy) * NumSIMDs;
+ uint64_t WorkGroupsPerCU =
+ std::max<uint64_t>(divideCeil(RequiredWavesPerCU, WavesPerWorkgroup), 1);
+
+ return AlignedLDSBytes <= LocalMemorySize / WorkGroupsPerCU;
+}
+
StringRef AMDGPU::getCanonicalArchName(const Triple &T, StringRef Arch) {
assert(T.isAMDGPU());
auto ProcKind = T.isAMDGCN() ? parseArchAMDGCN(Arch) : parseArchR600(Arch);
diff --git a/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-callgraph-lds-use.ll b/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-callgraph-lds-use.ll
new file mode 100644
index 0000000000000..ed28250f9146a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-callgraph-lds-use.ll
@@ -0,0 +1,56 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-enable-object-linking -filetype=obj < %s | llvm-readobj -r --sections - | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-enable-object-linking -filetype=asm < %s | FileCheck %s --check-prefix=ASM
+
+; Test that the unified .amdgpu.info section includes .amdgpu_use for LDS
+; alongside resource nodes and call edges. The kernel directly uses an external-linkage
+; LDS variable (global-scope), and calls a helper that calls an external.
+;
+; Expected relationships:
+; - Resources: helper and my_kernel (defined functions with resource info)
+; - Calls: my_kernel -> helper, helper -> extern_func
+; - LDS use: my_kernel -> lds_var
+
+ at lds_var = addrspace(3) global [32 x float] poison, align 4
+
+declare void @extern_func()
+
+; CHECK: Section {
+; CHECK: Name: .amdgpu.info
+; CHECK: Type: SHT_PROGBITS
+; CHECK: Flags [
+; CHECK: SHF_EXCLUDE
+; CHECK: ]
+
+; CHECK-DAG: R_AMDGPU_ABS64 my_kernel
+; CHECK-DAG: R_AMDGPU_ABS64 helper
+; CHECK-DAG: R_AMDGPU_ABS64 extern_func
+; CHECK-DAG: R_AMDGPU_ABS64 lds_var
+
+; Assembly: per-function .amdgpu_info blocks (target flags derived from e_flags).
+; ASM-DAG: .amdgpu_info helper
+; ASM-DAG: .amdgpu_flags 1
+; ASM-DAG: .amdgpu_num_vgpr {{[0-9]+}}
+; ASM-DAG: .amdgpu_num_sgpr {{[0-9]+}}
+; ASM-DAG: .amdgpu_private_segment_size {{[0-9]+}}
+; ASM-DAG: .amdgpu_call extern_func
+; ASM-DAG: .end_amdgpu_info
+; ASM-DAG: .amdgpu_info my_kernel
+; ASM-DAG: .amdgpu_flags 3
+; ASM-DAG: .amdgpu_num_vgpr {{[0-9]+}}
+; ASM-DAG: .amdgpu_num_sgpr {{[0-9]+}}
+; ASM-DAG: .amdgpu_private_segment_size {{[0-9]+}}
+; ASM-DAG: .amdgpu_use lds_var
+; ASM-DAG: .amdgpu_call helper
+; ASM-DAG: .end_amdgpu_info
+
+define void @helper() {
+ call void @extern_func()
+ ret void
+}
+
+define amdgpu_kernel void @my_kernel() {
+ %gep = getelementptr [32 x float], ptr addrspace(3) @lds_var, i32 0, i32 0
+ store float 1.0, ptr addrspace(3) %gep
+ call void @helper()
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-comdat.ll b/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-comdat.ll
new file mode 100644
index 0000000000000..421ffe5031387
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-comdat.ll
@@ -0,0 +1,18 @@
+; RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -amdgpu-enable-object-linking < %s | FileCheck %s
+
+; Object-linking lowering externalizes LDS definitions. Verify that it also
+; removes COMDAT membership, which is invalid on a declaration.
+
+$lds = comdat any
+
+ at lds = linkonce_odr addrspace(3) global [4 x i32] undef, comdat($lds), align 16
+
+; CHECK: .amdgpu_info kernel
+; CHECK: .amdgpu_use lds
+; CHECK: .amdgpu_lds lds, 16, 16
+
+define amdgpu_kernel void @kernel() {
+ %lds.ptr = getelementptr [4 x i32], ptr addrspace(3) @lds, i32 0, i32 0
+ store volatile i32 1, ptr addrspace(3) %lds.ptr, align 16
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-cross-tu-addr-taken.ll b/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-cross-tu-addr-taken.ll
new file mode 100644
index 0000000000000..204bfae2aa895
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-cross-tu-addr-taken.ll
@@ -0,0 +1,42 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-enable-object-linking -amdgpu-enable-lower-module-lds=0 -filetype=obj < %s | llvm-readobj -r - | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-enable-object-linking -amdgpu-enable-lower-module-lds=0 -filetype=asm < %s | FileCheck %s --check-prefix=ASM
+
+; Test that address-taken on a declaration (cross-TU scenario) emits a
+; .amdgpu_typeid directive for the external function. The function
+; is defined in another TU but its address is taken here.
+
+declare void @external_func(i32)
+
+define void @taker(ptr %p) {
+ store ptr @external_func, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @kern() {
+ %p = alloca ptr, addrspace(5)
+ call void @taker(ptr addrspace(5) %p)
+ ret void
+}
+
+; CHECK-DAG: R_AMDGPU_ABS64 external_func
+; CHECK-DAG: R_AMDGPU_ABS64 kern
+; CHECK-DAG: R_AMDGPU_ABS64 taker
+
+; Assembly: per-function .amdgpu_info blocks (target flags derived from e_flags).
+; ASM-DAG: .amdgpu_info taker
+; ASM-DAG: .amdgpu_flags 0
+; ASM-DAG: .amdgpu_num_vgpr {{[0-9]+}}
+; ASM-DAG: .amdgpu_num_sgpr {{[0-9]+}}
+; ASM-DAG: .amdgpu_private_segment_size {{[0-9]+}}
+; ASM-DAG: .amdgpu_typeid "vl"
+; ASM-DAG: .end_amdgpu_info
+; ASM-DAG: .amdgpu_info kern
+; ASM-DAG: .amdgpu_flags 2
+; ASM-DAG: .amdgpu_num_vgpr {{[0-9]+}}
+; ASM-DAG: .amdgpu_num_sgpr {{[0-9]+}}
+; ASM-DAG: .amdgpu_private_segment_size {{[0-9]+}}
+; ASM-DAG: .amdgpu_call taker
+; ASM-DAG: .end_amdgpu_info
+; ASM-DAG: .amdgpu_info external_func
+; ASM-DAG: .amdgpu_typeid "vi"
+; ASM-DAG: .end_amdgpu_info
diff --git a/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-dynlds-func.ll b/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-dynlds-func.ll
new file mode 100644
index 0000000000000..83fcd9b987e06
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-dynlds-func.ll
@@ -0,0 +1,37 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-enable-object-linking -filetype=obj < %s | llvm-readobj -r --syms - | FileCheck %s --check-prefix=ELF
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-enable-object-linking -filetype=asm < %s | FileCheck %s --check-prefix=ASM
+
+; Test that dynamic LDS used by a device function (not a kernel) works with
+; object linking. The device function's use generates a relocation, and the
+; AMDGPU info section records the (function, dyn_var) association.
+
+ at dyn_lds = external addrspace(3) global [0 x i8], align 8
+
+declare void @extern_func()
+
+define void @device_func(i32 %idx) {
+ %gep = getelementptr [0 x i8], ptr addrspace(3) @dyn_lds, i32 0, i32 %idx
+ store i8 99, ptr addrspace(3) %gep
+ call void @extern_func()
+ ret void
+}
+
+define amdgpu_kernel void @my_kernel(i32 %idx) {
+ call void @device_func(i32 %idx)
+ ret void
+}
+
+; ELF: R_AMDGPU_ABS32_LO dyn_lds
+
+; ELF: Symbol {
+; ELF: Name: dyn_lds
+; ELF-NEXT: Value: 0x8
+; ELF-NEXT: Size: 0
+; ELF-NEXT: Binding: Global
+; ELF-NEXT: Type: Object
+; ELF-NEXT: Other: 0
+; ELF-NEXT: Section: Processor Specific (0xFF00)
+; ELF-NEXT: }
+
+; ASM-DAG: .amdgpu_lds dyn_lds, 0, 8
+; ASM-DAG: .amdgpu_use dyn_lds
diff --git a/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-dynlds.ll b/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-dynlds.ll
new file mode 100644
index 0000000000000..6a5586b7594fd
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-dynlds.ll
@@ -0,0 +1,49 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-enable-object-linking -filetype=obj < %s | llvm-readobj -r --syms - | FileCheck %s --check-prefix=ELF
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-enable-object-linking -filetype=asm < %s | FileCheck %s --check-prefix=ASM
+
+; Test that dynamic LDS (extern __shared__, represented as zero-sized
+; addrspace(3) globals) works with object linking. Dynamic LDS variables
+; flow through the Global-scope path -- no per-kernel representative is
+; created. The original symbol name is preserved. Uses are recorded in the
+; unified .amdgpu.info section via .amdgpu_use.
+
+ at static_lds = addrspace(3) global [64 x float] poison, align 16
+ at dyn_lds = external addrspace(3) global [0 x i8], align 4
+
+declare void @extern_func()
+
+define void @helper(i32 %idx) {
+ %gep = getelementptr [64 x float], ptr addrspace(3) @static_lds, i32 0, i32 %idx
+ store float 2.0, ptr addrspace(3) %gep
+ call void @extern_func()
+ ret void
+}
+
+define amdgpu_kernel void @test_kernel(i32 %idx) {
+ %gep.static = getelementptr [64 x float], ptr addrspace(3) @static_lds, i32 0, i32 %idx
+ store float 1.0, ptr addrspace(3) %gep.static
+ %gep.dyn = getelementptr [0 x i8], ptr addrspace(3) @dyn_lds, i32 0, i32 %idx
+ store i8 42, ptr addrspace(3) %gep.dyn
+ call void @helper(i32 %idx)
+ ret void
+}
+
+; Relocations appear first in llvm-readobj output.
+; ELF: R_AMDGPU_ABS32_LO dyn_lds
+
+; The resources section should reference the dyn_lds symbol.
+; ELF: R_AMDGPU_ABS64 dyn_lds
+
+; The original dyn_lds symbol should be emitted as size-0 SHN_AMDGPU_LDS.
+; ELF: Symbol {
+; ELF: Name: dyn_lds
+; ELF-NEXT: Value: 0x4
+; ELF-NEXT: Size: 0
+; ELF-NEXT: Binding: Global
+; ELF-NEXT: Type: Object
+; ELF-NEXT: Other: 0
+; ELF-NEXT: Section: Processor Specific (0xFF00)
+; ELF-NEXT: }
+
+; ASM-DAG: .amdgpu_lds dyn_lds, 0, 4
+; ASM-DAG: .amdgpu_use dyn_lds
diff --git a/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-indirect.ll b/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-indirect.ll
new file mode 100644
index 0000000000000..0113dc8186181
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-indirect.ll
@@ -0,0 +1,54 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-enable-object-linking -amdgpu-enable-lower-module-lds=0 -filetype=obj < %s | llvm-readobj -r --sections - | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-enable-object-linking -amdgpu-enable-lower-module-lds=0 -filetype=asm < %s | FileCheck %s --check-prefix=ASM
+
+; Test that the unified .amdgpu.info section includes address-taken metadata and
+; .amdgpu_indirect_call for functions involved in indirect calling.
+;
+; @callee_vi: void(i32) -- address taken, prototype encoding "vi"
+; @caller: has indirect call to void(i32) -- icall encoding "vi"
+; @my_kernel: kernel that passes @callee_vi as a function pointer to @caller
+
+define void @callee_vi(i32 %x) {
+ ret void
+}
+
+define void @caller(ptr %fptr) {
+ call void %fptr(i32 1)
+ ret void
+}
+
+define amdgpu_kernel void @my_kernel() {
+ call void @caller(ptr @callee_vi)
+ ret void
+}
+
+; CHECK: Section {
+; CHECK: Name: .amdgpu.info
+; CHECK: Type: SHT_PROGBITS
+; CHECK: Flags [
+; CHECK: SHF_EXCLUDE
+; CHECK: ]
+
+; CHECK-DAG: R_AMDGPU_ABS64 my_kernel
+; CHECK-DAG: R_AMDGPU_ABS64 caller
+; CHECK-DAG: R_AMDGPU_ABS64 callee_vi
+
+; ASM-DAG: .amdgpu_info callee_vi
+; ASM-DAG: .amdgpu_flags 0
+; ASM-DAG: .amdgpu_num_vgpr {{[0-9]+}}
+; ASM-DAG: .amdgpu_num_sgpr {{[0-9]+}}
+; ASM-DAG: .amdgpu_private_segment_size {{[0-9]+}}
+; ASM-DAG: .amdgpu_typeid "vi"
+; ASM-DAG: .end_amdgpu_info
+; ASM-DAG: .amdgpu_info caller
+; ASM-DAG: .amdgpu_flags 1
+; ASM-DAG: .amdgpu_num_vgpr {{[0-9]+}}
+; ASM-DAG: .amdgpu_indirect_call "vi"
+; ASM-DAG: .end_amdgpu_info
+; ASM-DAG: .amdgpu_info my_kernel
+; ASM-DAG: .amdgpu_flags 3
+; ASM-DAG: .amdgpu_num_vgpr {{[0-9]+}}
+; ASM-DAG: .amdgpu_num_sgpr {{[0-9]+}}
+; ASM-DAG: .amdgpu_private_segment_size {{[0-9]+}}
+; ASM-DAG: .amdgpu_call caller
+; ASM-DAG: .end_amdgpu_info
diff --git a/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-multi.ll b/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-multi.ll
new file mode 100644
index 0000000000000..f061754045a60
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-multi.ll
@@ -0,0 +1,56 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-enable-lower-module-lds=0 -amdgpu-enable-object-linking < %s | FileCheck -check-prefixes=ASM %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-enable-lower-module-lds=0 -amdgpu-enable-object-linking -filetype=obj < %s | llvm-readobj -r --syms - | FileCheck -check-prefixes=ELF %s
+
+; Multiple external LDS declarations with different sizes and alignments,
+; accessed by two kernels. Tests that each LDS decl gets its own
+; SHN_AMDGPU_LDS symbol and .amdgpu_lds directive. The kernel descriptor
+; uses literal 0 for group_segment_fixed_size because the linker patches
+; it via direct binary patching.
+
+ at __amdgpu_lds.func_a = external addrspace(3) global [256 x i8], align 16
+ at __amdgpu_lds.func_b = external addrspace(3) global [128 x i8], align 4
+ at __amdgpu_lds.kernel_one = external addrspace(3) global [64 x i8], align 8
+
+; --- Assembly checks (use DAG for unordered matches) ---
+; Each LDS declaration should produce a .amdgpu_lds directive.
+; ASM-DAG: .amdgpu_lds __amdgpu_lds.func_a, 256, 16
+; ASM-DAG: .amdgpu_lds __amdgpu_lds.func_b, 128, 4
+; ASM-DAG: .amdgpu_lds __amdgpu_lds.kernel_one, 64, 8
+
+; HSA metadata should have group_segment_fixed_size = 0.
+; ASM-DAG: .group_segment_fixed_size: 0
+
+; --- ELF checks ---
+; All three SHN_AMDGPU_LDS symbols should be present.
+; ELF-DAG: Name: __amdgpu_lds.func_a
+; ELF-DAG: Name: __amdgpu_lds.func_b
+; ELF-DAG: Name: __amdgpu_lds.kernel_one
+
+; LDS access relocations in .text.
+; ELF-DAG: R_AMDGPU_ABS32_LO __amdgpu_lds.func_a
+; ELF-DAG: R_AMDGPU_ABS32_LO __amdgpu_lds.func_b
+; ELF-DAG: R_AMDGPU_ABS32_LO __amdgpu_lds.kernel_one
+
+define void @func_a(i32 %idx) {
+ %gep = getelementptr [256 x i8], ptr addrspace(3) @__amdgpu_lds.func_a, i32 0, i32 %idx
+ store i8 1, ptr addrspace(3) %gep
+ ret void
+}
+
+define void @func_b(i32 %idx) {
+ %gep = getelementptr [128 x i8], ptr addrspace(3) @__amdgpu_lds.func_b, i32 0, i32 %idx
+ store i8 2, ptr addrspace(3) %gep
+ ret void
+}
+
+define amdgpu_kernel void @kernel_one(i32 %idx) {
+ %gep = getelementptr [64 x i8], ptr addrspace(3) @__amdgpu_lds.kernel_one, i32 0, i32 %idx
+ store i8 3, ptr addrspace(3) %gep
+ call void @func_a(i32 %idx)
+ ret void
+}
+
+define amdgpu_kernel void @kernel_two(i32 %idx) {
+ call void @func_b(i32 %idx)
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-no-attr.ll b/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-no-attr.ll
new file mode 100644
index 0000000000000..e87f46574fa45
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-no-attr.ll
@@ -0,0 +1,36 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-enable-lower-module-lds=0 -amdgpu-enable-object-linking < %s | FileCheck -check-prefixes=ASM %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-enable-lower-module-lds=0 -amdgpu-enable-object-linking -filetype=obj < %s | llvm-readobj -r --syms - | FileCheck -check-prefixes=ELF %s
+
+; Uses an external LDS declaration with object linking enabled. The LDS access
+; should get a relocation, and the SHN_AMDGPU_LDS symbol should be emitted.
+; However, the kernel descriptor should NOT use a symbolic __amdgpu_lds_size
+; reference.
+
+ at __amdgpu_lds.func = external addrspace(3) global [256 x i8], align 16
+
+; LDS access still generates a relocation.
+; ASM-LABEL: {{^}}test_kernel:
+; ASM: __amdgpu_lds.func at abs32@lo
+
+; Kernel descriptor uses a constant, not a symbolic reference.
+; ASM-NOT: .amdhsa_group_segment_fixed_size __amdgpu_lds_size
+
+; SHN_AMDGPU_LDS symbol is still emitted for the external decl.
+; ASM: .amdgpu_lds __amdgpu_lds.func, 256, 16
+
+; --- ELF checks (relocations before symbols in readobj output) ---
+; LDS relocation in .text.
+; ELF: R_AMDGPU_ABS32_LO __amdgpu_lds.func
+
+; No symbolic KD reference.
+; ELF-NOT: __amdgpu_lds_size
+
+; SHN_AMDGPU_LDS symbol is present.
+; ELF: Name: __amdgpu_lds.func
+; ELF: Section: Processor Specific (0xFF00)
+
+define amdgpu_kernel void @test_kernel(i32 %idx) {
+ %gep = getelementptr [256 x i8], ptr addrspace(3) @__amdgpu_lds.func, i32 0, i32 %idx
+ store i8 42, ptr addrspace(3) %gep
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-prototype.ll b/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-prototype.ll
new file mode 100644
index 0000000000000..20353008af1c6
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-prototype.ll
@@ -0,0 +1,80 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-enable-object-linking -amdgpu-enable-lower-module-lds=0 -filetype=obj < %s | llvm-readobj -r - | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-enable-object-linking -amdgpu-enable-lower-module-lds=0 -filetype=asm < %s | FileCheck %s --check-prefix=ASM
+
+; Test ABI register-size type ID generation for various function signatures.
+; The type ID encodes each parameter/return by bit width: v=void, i=<=32-bit,
+; l=33-64-bit. Types with the same register footprint share an encoding
+; (e.g. float(float) and i32(i32) both produce "ii").
+
+define void @void_void() {
+ ret void
+}
+
+define i32 @i32_i32(i32 %x) {
+ ret i32 %x
+}
+
+define void @void_ptr_i32(ptr %p, i32 %x) {
+ ret void
+}
+
+define i64 @i64_i64_i64(i64 %a, i64 %b) {
+ ret i64 %a
+}
+
+define float @float_float(float %x) {
+ ret float %x
+}
+
+; Take the address of each function so they appear as resource nodes.
+define void @taker() {
+ %p0 = alloca ptr, addrspace(5)
+ store volatile ptr @void_void, ptr addrspace(5) %p0
+ store volatile ptr @i32_i32, ptr addrspace(5) %p0
+ store volatile ptr @void_ptr_i32, ptr addrspace(5) %p0
+ store volatile ptr @i64_i64_i64, ptr addrspace(5) %p0
+ store volatile ptr @float_float, ptr addrspace(5) %p0
+ ret void
+}
+
+define amdgpu_kernel void @kern() {
+ call void @taker()
+ ret void
+}
+
+; CHECK-DAG: R_AMDGPU_ABS64 void_void
+; CHECK-DAG: R_AMDGPU_ABS64 i32_i32
+; CHECK-DAG: R_AMDGPU_ABS64 void_ptr_i32
+; CHECK-DAG: R_AMDGPU_ABS64 i64_i64_i64
+; CHECK-DAG: R_AMDGPU_ABS64 float_float
+; CHECK-DAG: R_AMDGPU_ABS64 taker
+; CHECK-DAG: R_AMDGPU_ABS64 kern
+
+; ASM-DAG: .amdgpu_info void_void
+; ASM-DAG: .amdgpu_flags 0
+; ASM-DAG: .amdgpu_typeid "v"
+; ASM-DAG: .end_amdgpu_info
+; ASM-DAG: .amdgpu_info i32_i32
+; ASM-DAG: .amdgpu_flags 0
+; ASM-DAG: .amdgpu_typeid "ii"
+; ASM-DAG: .end_amdgpu_info
+; ASM-DAG: .amdgpu_info void_ptr_i32
+; ASM-DAG: .amdgpu_flags 0
+; ASM-DAG: .amdgpu_typeid "vli"
+; ASM-DAG: .end_amdgpu_info
+; ASM-DAG: .amdgpu_info i64_i64_i64
+; ASM-DAG: .amdgpu_flags 0
+; ASM-DAG: .amdgpu_typeid "lll"
+; ASM-DAG: .end_amdgpu_info
+; ASM-DAG: .amdgpu_info float_float
+; ASM-DAG: .amdgpu_flags 0
+; ASM-DAG: .amdgpu_typeid "ii"
+; ASM-DAG: .end_amdgpu_info
+; ASM-DAG: .amdgpu_info taker
+; ASM-DAG: .amdgpu_flags 0
+; ASM-DAG: .amdgpu_num_vgpr {{[0-9]+}}
+; ASM-DAG: .end_amdgpu_info
+; ASM-DAG: .amdgpu_info kern
+; ASM-DAG: .amdgpu_flags 2
+; ASM-DAG: .amdgpu_call taker
+; ASM-DAG: .end_amdgpu_info
diff --git a/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-typeid.ll b/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-typeid.ll
index ca13916828443..9745b97898c1c 100644
--- a/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-typeid.ll
+++ b/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen-typeid.ll
@@ -235,7 +235,7 @@ define amdgpu_kernel void @kern() {
; COM: with their plain-pointer counterparts: "vi" (AS 5) and "vl" (AS 4) each
; COM: appear once despite two call sites apiece.
; ASM-DAG: .amdgpu_info icaller
-; ASM-DAG: .amdgpu_flags 1
+; ASM-DAG: .amdgpu_flags 0
; ASM-DAG: .amdgpu_indirect_call "v"
; ASM-DAG: .amdgpu_indirect_call "vlii"
; ASM-DAG: .amdgpu_indirect_call "iiiiliiiiiiii"
@@ -246,7 +246,6 @@ define amdgpu_kernel void @kern() {
; ASM-DAG: .amdgpu_indirect_call "vli"
; ASM-DAG: .end_amdgpu_info
; ASM-DAG: .amdgpu_info taker
-; ASM-DAG: .amdgpu_flags 0
; ASM-DAG: .amdgpu_num_vgpr {{[0-9]+}}
; ASM-DAG: .end_amdgpu_info
; COM: The kernel scope is present but carries no type IDs of its own (kernels
diff --git a/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen.ll b/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen.ll
index e7c0e7a562d75..8bcbe69a48c58 100644
--- a/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen.ll
+++ b/llvm/test/CodeGen/AMDGPU/lds-link-time-codegen.ll
@@ -1,5 +1,9 @@
; RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -amdgpu-enable-object-linking < %s | FileCheck -check-prefixes=ASM %s --implicit-check-not=.amdgpu_num_agpr
; RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -amdgpu-enable-object-linking -filetype=obj < %s | llvm-readobj -r --syms --sections - | FileCheck -check-prefixes=ELF %s
+; RUN: llc -mtriple=amdgpu11.00-amd-amdhsa -amdgpu-enable-object-linking -filetype=asm < %s | FileCheck -check-prefixes=WGP-WAVE32 %s
+; RUN: llc -mtriple=amdgpu11.00-amd-amdhsa -mattr=+wavefrontsize64 -amdgpu-enable-object-linking -filetype=asm < %s | FileCheck -check-prefixes=WGP-WAVE64 %s
+; RUN: llc -mtriple=amdgpu11.00-amd-amdhsa -mattr=+cumode -amdgpu-enable-object-linking -filetype=asm < %s | FileCheck -check-prefixes=CU-WAVE32 %s
+; RUN: llc -mtriple=amdgpu11.00-amd-amdhsa -mattr=+cumode,+wavefrontsize64 -amdgpu-enable-object-linking -filetype=asm < %s | FileCheck -check-prefixes=CU-WAVE64 %s
; Test that with object linking enabled, external LDS declarations produce
; @abs32 at lo relocations, SHN_AMDGPU_LDS symbols, .amdgpu_lds directives,
@@ -38,6 +42,20 @@
; ASM-DAG: .amdgpu_call device_func
; ASM-DAG: .end_amdgpu_info
+; COM: FUNC_WGP_MODE (0x8) and FUNC_WAVE32 (0x10) track the subtarget execution
+; COM: mode and wave size. gfx11 defaults to WGP mode + wave32; +cumode selects
+; COM: CU mode and +wavefrontsize64 selects wave64. The kernel and the device
+; COM: function share the same subtarget-derived flags, so both .amdgpu_flags
+; COM: entries carry the same value in each run.
+; COM: WGP | WAVE32 = 0x8 | 0x10 = 24.
+; WGP-WAVE32-COUNT-2: .amdgpu_flags 24
+; COM: WGP only = 0x8 = 8.
+; WGP-WAVE64-COUNT-2: .amdgpu_flags 8
+; COM: WAVE32 only = 0x10 = 16.
+; CU-WAVE32-COUNT-2: .amdgpu_flags 16
+; COM: neither set = 0.
+; CU-WAVE64-COUNT-2: .amdgpu_flags 0
+
; SHN_AMDGPU_LDS directives.
; ASM-DAG: .amdgpu_lds lds_large, 256, 16
; ASM-DAG: .amdgpu_lds lds_small, 128, 4
diff --git a/llvm/test/CodeGen/AMDGPU/lds-link-time-named-barrier.ll b/llvm/test/CodeGen/AMDGPU/lds-link-time-named-barrier.ll
index 75359d575575e..0ffa88ceb6af9 100644
--- a/llvm/test/CodeGen/AMDGPU/lds-link-time-named-barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/lds-link-time-named-barrier.ll
@@ -6,11 +6,15 @@
; 2. AMDGPULowerModuleLDS does not handle named barriers at all
; 3. amdgpu.lds.uses does NOT contain barrier entries
+$comdat_bar = comdat any
+
@bar = internal addrspace(3) global target("amdgcn.named.barrier", 0) poison
+ at comdat_bar = linkonce_odr addrspace(3) global target("amdgcn.named.barrier", 0) poison, comdat($comdat_bar)
@lds = internal addrspace(3) global [4 x i32] poison, align 4
; Internal named barrier becomes external with a module-unique hash suffix.
; CHECK: @[[BAR:__amdgpu_named_barrier\.bar\.[a-f0-9]+]] = external dso_local addrspace(3) global target("amdgcn.named.barrier", 0)
+; CHECK: @__amdgpu_named_barrier.comdat_bar = external {{(dso_local )?}}addrspace(3) global target("amdgcn.named.barrier", 0)
; CHECK-NOT: !absolute_symbol
; Regular LDS is packed into the per-function struct (external, for linker).
; CHECK: @__amdgpu_lds.kernel = external dso_local addrspace(3) global %__amdgpu_lds.kernel.t, align 16
@@ -19,8 +23,10 @@ define amdgpu_kernel void @kernel(i32 %idx) {
; CHECK-LABEL: define amdgpu_kernel void @kernel(
; CHECK: call void @llvm.amdgcn.s.barrier.join(ptr addrspace(3) @[[BAR]])
; CHECK: call void @llvm.amdgcn.s.barrier.signal.var(ptr addrspace(3) @[[BAR]], i32 3)
+; CHECK: call void @llvm.amdgcn.s.barrier.join(ptr addrspace(3) @__amdgpu_named_barrier.comdat_bar)
call void @llvm.amdgcn.s.barrier.join(ptr addrspace(3) @bar)
call void @llvm.amdgcn.s.barrier.signal.var(ptr addrspace(3) @bar, i32 3)
+ call void @llvm.amdgcn.s.barrier.join(ptr addrspace(3) @comdat_bar)
call void @llvm.amdgcn.s.barrier.wait(i16 1)
%gep = getelementptr [4 x i32], ptr addrspace(3) @lds, i32 0, i32 %idx
store i32 42, ptr addrspace(3) %gep, align 4
@@ -28,8 +34,9 @@ define amdgpu_kernel void @kernel(i32 %idx) {
}
; Named barrier metadata: (barrier_sym, func1, ...) -- emitted by ExecSync.
-; CHECK-DAG: !amdgpu.named_barrier.uses = !{[[BAR_MD:![0-9]+]]}
-; CHECK-DAG: [[BAR_MD]] = !{ptr addrspace(3) @[[BAR]], ptr @kernel}
+; CHECK-DAG: !amdgpu.named_barrier.uses = !{
+; CHECK-DAG: !{ptr addrspace(3) @[[BAR]], ptr @kernel}
+; CHECK-DAG: !{ptr addrspace(3) @__amdgpu_named_barrier.comdat_bar, ptr @kernel}
; LDS metadata must have exactly one entry (the LDS struct), no barrier entries.
; CHECK-DAG: !amdgpu.lds.uses = !{[[LDS_MD:![0-9]+]]}
; CHECK-DAG: [[LDS_MD]] = !{ptr @kernel, ptr addrspace(3) @__amdgpu_lds.kernel}
diff --git a/llvm/test/CodeGen/AMDGPU/object-linking-abi-occupancy-device-no-attr.ll b/llvm/test/CodeGen/AMDGPU/object-linking-abi-occupancy-device-no-attr.ll
new file mode 100644
index 0000000000000..7a9d778fe36da
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/object-linking-abi-occupancy-device-no-attr.ll
@@ -0,0 +1,27 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-enable-object-linking < %s | FileCheck -check-prefix=ABI %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefix=NOABI %s
+
+; Device function with no amdgpu-waves-per-eu attribute. Under object linking
+; the ABI occupancy is used as a register-budget floor without changing the
+; source-level waves-per-EU range. Per-function resource reporting is local-only
+; under object linking, so only the function-level `.num_vgpr` / `.num_agpr`
+; symbols are emitted -- the module-level `amdgpu.max_num_*` symbols are
+; suppressed.
+
+; ABI-LABEL: {{^}}device_fn:
+; ABI: .set .Ldevice_fn.num_vgpr, 3
+; ABI: .set .Ldevice_fn.num_agpr, 0
+
+; ABI-NOT: amdgpu.max_num_
+
+; NOABI-LABEL: {{^}}device_fn:
+; NOABI: .set .Ldevice_fn.num_vgpr, 3
+; NOABI: .set .Ldevice_fn.num_agpr, 0
+
+; NOABI: .set amdgpu.max_num_vgpr, 3
+; NOABI: .set amdgpu.max_num_agpr, 0
+
+define void @device_fn(ptr addrspace(1) %p) {
+ store i32 0, ptr addrspace(1) %p
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/object-linking-abi-occupancy-flat-workgroup-lower.ll b/llvm/test/CodeGen/AMDGPU/object-linking-abi-occupancy-flat-workgroup-lower.ll
new file mode 100644
index 0000000000000..4ac1c35908935
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/object-linking-abi-occupancy-flat-workgroup-lower.ll
@@ -0,0 +1,24 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+wavefrontsize64 -amdgpu-enable-object-linking < %s | FileCheck --check-prefixes=GCN,W64 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+wavefrontsize32 -amdgpu-enable-object-linking < %s | FileCheck --check-prefixes=GCN,W32 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+wavefrontsize64 -amdgpu-enable-object-linking < %s | FileCheck --check-prefixes=GCN,W64 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+wavefrontsize32 -amdgpu-enable-object-linking < %s | FileCheck --check-prefixes=GCN,W32 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+wavefrontsize64 -amdgpu-enable-object-linking < %s | FileCheck --check-prefixes=GCN,W64 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+wavefrontsize32 -amdgpu-enable-object-linking < %s | FileCheck --check-prefixes=GCN,W32 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+wavefrontsize64 -amdgpu-enable-object-linking < %s | FileCheck --check-prefixes=GCN,W64 %s
+
+; amdgpu-flat-work-group-size is ABI-significant and can lower the ABI
+; occupancy implied by the default 1024-workitem workgroup.
+
+; GCN-LABEL: {{^}}fixed_vgpr_flat_lower:
+; GCN: .set .Lfixed_vgpr_flat_lower.num_vgpr, 71
+; W64: .amdgpu_occupancy 2
+; W32: .amdgpu_occupancy 4
+; GCN-NOT: amdgpu.max_num_
+
+define amdgpu_kernel void @fixed_vgpr_flat_lower(ptr addrspace(1) %p) #0 {
+ call void asm sideeffect "; clobber", "~{v70}"()
+ store i32 0, ptr addrspace(1) %p
+ ret void
+}
+
+attributes #0 = { "amdgpu-flat-work-group-size"="1,512" }
diff --git a/llvm/test/CodeGen/AMDGPU/object-linking-abi-occupancy-flat-workgroup-reject-override.ll b/llvm/test/CodeGen/AMDGPU/object-linking-abi-occupancy-flat-workgroup-reject-override.ll
new file mode 100644
index 0000000000000..6812e2b47fe95
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/object-linking-abi-occupancy-flat-workgroup-reject-override.ll
@@ -0,0 +1,28 @@
+; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+wavefrontsize64 -amdgpu-enable-object-linking -filetype=null < %s 2>&1 | FileCheck --check-prefix=GFX9 %s
+; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+wavefrontsize32 -amdgpu-enable-object-linking -filetype=null < %s 2>&1 | FileCheck --check-prefix=GFX10 %s
+; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+wavefrontsize64 -amdgpu-enable-object-linking -filetype=null < %s 2>&1 | FileCheck --check-prefix=GFX10 %s
+; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+wavefrontsize32 -amdgpu-enable-object-linking -filetype=null < %s 2>&1 | FileCheck --check-prefix=GFX11 %s
+; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+wavefrontsize64 -amdgpu-enable-object-linking -filetype=null < %s 2>&1 | FileCheck --check-prefix=GFX11 %s
+; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+wavefrontsize32 -amdgpu-enable-object-linking -filetype=null < %s 2>&1 | FileCheck --check-prefix=GFX12 %s
+; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+wavefrontsize64 -amdgpu-enable-object-linking -filetype=null < %s 2>&1 | FileCheck --check-prefix=GFX12 %s
+
+; amdgpu-flat-work-group-size is ABI-significant. Even when the default ABI
+; occupancy is overridden to 2 waves/EU by a module flag, a flat workgroup size
+; of 1024 requires the per-target occupancy implied by a 1024-workitem
+; workgroup.
+
+; GFX9: error: {{.*}}VGPRs under object-linking ABI (193) exceeds limit (64) in function 'fixed_vgpr_flat'
+; GFX10: error: {{.*}}VGPRs under object-linking ABI (193) exceeds limit (128) in function 'fixed_vgpr_flat'
+; GFX11: error: {{.*}}VGPRs under object-linking ABI (193) exceeds limit (192) in function 'fixed_vgpr_flat'
+; GFX12: error: {{.*}}VGPRs under object-linking ABI (193) exceeds limit (192) in function 'fixed_vgpr_flat'
+
+define amdgpu_kernel void @fixed_vgpr_flat(ptr addrspace(1) %p) #0 {
+ call void asm sideeffect "; clobber", "~{v192}"()
+ store i32 0, ptr addrspace(1) %p
+ ret void
+}
+
+attributes #0 = { "amdgpu-flat-work-group-size"="1,1024" }
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"amdgpu_abi_waves_per_eu", i32 2}
diff --git a/llvm/test/CodeGen/AMDGPU/object-linking-abi-occupancy-kernel-no-attr.ll b/llvm/test/CodeGen/AMDGPU/object-linking-abi-occupancy-kernel-no-attr.ll
new file mode 100644
index 0000000000000..a4f4f2be85c7d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/object-linking-abi-occupancy-kernel-no-attr.ll
@@ -0,0 +1,31 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-enable-object-linking < %s | FileCheck -check-prefix=ABI %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefix=NOABI %s
+
+; Kernel with no amdgpu-waves-per-eu attribute. Under object linking the ABI
+; occupancy is used as a register-budget floor, but it is not modeled as an
+; exact maximum occupancy request. The kernel descriptor therefore still reports
+; the local resource usage and resulting occupancy. Per-function resource
+; reporting is local-only under object linking, so no module-level
+; `amdgpu.max_num_*` symbols are emitted.
+
+; ABI-LABEL: {{^}}kernel_no_attr:
+; ABI: .set .Lkernel_no_attr.num_vgpr, 1
+; ABI: NumVGPRsForWavesPerEU: 1
+; ABI: Occupancy: 10
+; ABI: .amdgpu_occupancy 4
+
+; ABI-NOT: amdgpu.max_num_
+
+; NOABI-LABEL: {{^}}kernel_no_attr:
+; NOABI: .set .Lkernel_no_attr.num_vgpr, 1
+; NOABI: NumVGPRsForWavesPerEU: 1
+; NOABI: Occupancy: 10
+
+; NOABI: .set amdgpu.max_num_vgpr, 0
+; NOABI: .set amdgpu.max_num_agpr, 0
+; NOABI: .set amdgpu.max_num_sgpr, 0
+
+define amdgpu_kernel void @kernel_no_attr(ptr addrspace(1) %p) {
+ store i32 0, ptr addrspace(1) %p
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/object-linking-abi-occupancy-override.ll b/llvm/test/CodeGen/AMDGPU/object-linking-abi-occupancy-override.ll
new file mode 100644
index 0000000000000..5c71064e092f7
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/object-linking-abi-occupancy-override.ll
@@ -0,0 +1,30 @@
+; RUN: split-file %s %t
+; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-enable-object-linking -filetype=null < %t/default.ll 2>&1 | FileCheck -check-prefix=DEFAULT %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-enable-object-linking < %t/override.ll | FileCheck -check-prefix=OVERRIDE %s
+
+; The default object-linking ABI occupancy on gfx900 is 4 waves/EU, which gives
+; a 64 VGPR budget. Overriding the ABI occupancy to 2 waves/EU with the
+; amdgpu_abi_waves_per_eu module flag raises the budget to 128 VGPRs.
+
+; DEFAULT: error: {{.*}}VGPRs under object-linking ABI (71) exceeds limit (64) in function 'fixed_vgpr'
+
+; OVERRIDE-LABEL: {{^}}fixed_vgpr:
+; OVERRIDE: .set .Lfixed_vgpr.num_vgpr, 71
+; OVERRIDE-NOT: amdgpu.max_num_
+
+;--- default.ll
+define amdgpu_kernel void @fixed_vgpr(ptr addrspace(1) %p) {
+ call void asm sideeffect "; clobber", "~{v70}"()
+ store i32 0, ptr addrspace(1) %p
+ ret void
+}
+
+;--- override.ll
+define amdgpu_kernel void @fixed_vgpr(ptr addrspace(1) %p) {
+ call void asm sideeffect "; clobber", "~{v70}"()
+ store i32 0, ptr addrspace(1) %p
+ ret void
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"amdgpu_abi_waves_per_eu", i32 2}
diff --git a/llvm/test/CodeGen/AMDGPU/object-linking-abi-occupancy-preserves-attr.ll b/llvm/test/CodeGen/AMDGPU/object-linking-abi-occupancy-preserves-attr.ll
new file mode 100644
index 0000000000000..8182ce77d3ed1
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/object-linking-abi-occupancy-preserves-attr.ll
@@ -0,0 +1,34 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-enable-object-linking < %s | FileCheck -check-prefix=ABI %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefix=NOABI %s
+
+; Object-linking ABI occupancy is a budget floor, not a synthetic
+; amdgpu-waves-per-eu range. A max-occupancy hint that is weaker than the ABI
+; floor does not force resource inflation below the ABI floor.
+;
+; Here the kernel requests amdgpu-waves-per-eu="1,1", but the effective budget
+; remains compatible with the ABI. The kernel uses 30 VGPRs via inline asm, so
+; the final occupancy is resource-derived. No module-level `amdgpu.max_num_*`
+; symbols are emitted under object linking.
+
+; ABI-LABEL: {{^}}kernel_with_attr:
+; ABI: .set .Lkernel_with_attr.num_vgpr, 30
+; ABI: NumVGPRsForWavesPerEU: 30
+; ABI: Occupancy: 8
+; ABI: .amdgpu_occupancy 4
+
+; ABI-NOT: amdgpu.max_num_
+
+; NOABI-LABEL: {{^}}kernel_with_attr:
+; NOABI: .set .Lkernel_with_attr.num_vgpr, 30
+; NOABI: NumVGPRsForWavesPerEU: 30
+; NOABI: Occupancy: 8
+
+; NOABI: .set amdgpu.max_num_vgpr, 0
+
+define amdgpu_kernel void @kernel_with_attr(ptr addrspace(1) %p) #0 {
+ %r = call { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } asm sideeffect "; clobber", "=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v"()
+ store i32 0, ptr addrspace(1) %p
+ ret void
+}
+
+attributes #0 = { "amdgpu-waves-per-eu"="1,1" }
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-object-linking.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-object-linking.ll
new file mode 100644
index 0000000000000..8f2f12fc1add2
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-object-linking.ll
@@ -0,0 +1,22 @@
+; RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -mcpu=gfx942 -amdgpu-enable-object-linking < %s | FileCheck %s
+; RUN: llc -enable-new-pm -mtriple=amdgpu9.42-amd-amdhsa -mcpu=gfx942 -amdgpu-enable-object-linking < %s | FileCheck %s
+
+; The second object-linking LDS lowering run must process LDS globals created by
+; PromoteAlloca.
+
+; CHECK: .amdgpu_info kernel
+; CHECK: .amdgpu_use kernel.stack
+; CHECK: .amdgpu_lds kernel.stack, 4096, 4
+
+declare ptr @llvm.invariant.start.p5(i64, ptr addrspace(5) nocapture)
+declare void @llvm.invariant.end.p5(ptr, i64, ptr addrspace(5) nocapture)
+
+define amdgpu_kernel void @kernel() {
+ %stack = alloca i32, align 4, addrspace(5)
+ %invariant = call ptr @llvm.invariant.start.p5(i64 0,
+ ptr addrspace(5) %stack)
+ store <2 x i1> zeroinitializer, ptr %invariant, align 1
+ call void @llvm.invariant.end.p5(ptr %invariant, i64 0,
+ ptr addrspace(5) %stack)
+ ret void
+}
diff --git a/llvm/test/Linker/amdgpu-abi-waves-per-eu-module-flag.ll b/llvm/test/Linker/amdgpu-abi-waves-per-eu-module-flag.ll
new file mode 100644
index 0000000000000..3b7d98312611f
--- /dev/null
+++ b/llvm/test/Linker/amdgpu-abi-waves-per-eu-module-flag.ll
@@ -0,0 +1,16 @@
+; RUN: split-file %s %t
+; RUN: not llvm-link %t/a.ll %t/b.ll -S -o /dev/null 2>&1 | FileCheck %s
+
+; CHECK: error: linking module flags 'amdgpu_abi_waves_per_eu': IDs have conflicting values
+
+;--- a.ll
+target triple = "amdgcn-amd-amdhsa"
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"amdgpu_abi_waves_per_eu", i32 2}
+
+;--- b.ll
+target triple = "amdgcn-amd-amdhsa"
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"amdgpu_abi_waves_per_eu", i32 4}
diff --git a/llvm/test/MC/AMDGPU/amdgpu-info-roundtrip.s b/llvm/test/MC/AMDGPU/amdgpu-info-roundtrip.s
index 1769f4cb97629..18931a083b3ef 100644
--- a/llvm/test/MC/AMDGPU/amdgpu-info-roundtrip.s
+++ b/llvm/test/MC/AMDGPU/amdgpu-info-roundtrip.s
@@ -31,15 +31,16 @@ addr_taken_func:
.globl extern_func
-// COM: Kernel: flags=7 (KERNEL|VCC|FLAT_SCRATCH), resources, call edge, use
-// COM: edge, indirect call, and type ID. Non-zero AGPR to verify conditional
-// COM: emission.
+// COM: Kernel: flags=31 (VCC|FLAT_SCRATCH|HAS_DYN_STACK|WGP_MODE|WAVE32),
+// COM: resources, call edge, use edge, indirect call, and type ID. Non-zero AGPR
+// COM: to verify conditional emission.
.amdgpu_info my_kernel
- .amdgpu_flags 7
+ .amdgpu_flags 31
.amdgpu_num_sgpr 33
.amdgpu_num_vgpr 32
.amdgpu_num_agpr 4
.amdgpu_private_segment_size 0
+ .amdgpu_occupancy 4
.amdgpu_use lds_var
.amdgpu_call helper
.amdgpu_indirect_call "vi"
@@ -53,6 +54,7 @@ addr_taken_func:
.amdgpu_num_sgpr 8
.amdgpu_num_vgpr 10
.amdgpu_private_segment_size 16
+ .amdgpu_occupancy 8
.amdgpu_call extern_func
.end_amdgpu_info
@@ -62,15 +64,17 @@ addr_taken_func:
.amdgpu_num_sgpr 2
.amdgpu_num_vgpr 4
.amdgpu_private_segment_size 0
+ .amdgpu_occupancy 10
.amdgpu_typeid "vi"
.end_amdgpu_info
// ASM: .amdgpu_info my_kernel
-// ASM: .amdgpu_flags 7
+// ASM: .amdgpu_flags 31
// ASM: .amdgpu_num_sgpr 33
// ASM: .amdgpu_num_vgpr 32
// ASM: .amdgpu_num_agpr 4
// ASM: .amdgpu_private_segment_size 0
+// ASM: .amdgpu_occupancy 4
// ASM: .amdgpu_use lds_var
// ASM: .amdgpu_call helper
// ASM: .amdgpu_indirect_call "vi"
@@ -82,6 +86,7 @@ addr_taken_func:
// ASM: .amdgpu_num_vgpr 10
// ASM-NOT: .amdgpu_num_agpr
// ASM: .amdgpu_private_segment_size 16
+// ASM: .amdgpu_occupancy 8
// ASM: .amdgpu_call extern_func
// ASM: .end_amdgpu_info
@@ -91,6 +96,7 @@ addr_taken_func:
// ASM: .amdgpu_num_vgpr 4
// ASM-NOT: .amdgpu_num_agpr
// ASM: .amdgpu_private_segment_size 0
+// ASM: .amdgpu_occupancy 10
// ASM: .amdgpu_typeid "vi"
// ASM: .end_amdgpu_info
diff --git a/llvm/test/ThinLTO/AMDGPU/amdgpu-occupancy-merge.ll b/llvm/test/ThinLTO/AMDGPU/amdgpu-occupancy-merge.ll
new file mode 100644
index 0000000000000..9051a411ab749
--- /dev/null
+++ b/llvm/test/ThinLTO/AMDGPU/amdgpu-occupancy-merge.ll
@@ -0,0 +1,59 @@
+; Test that occupancy attributes are merged conservatively when a device
+; function is reachable from multiple kernels with different constraints.
+;
+; kernel1: flat_wg=[64,256], waves=[2,8], max_wg=[16,16,1]
+; kernel2: flat_wg=[32,128], waves=[4,6], max_wg=[8,8,8]
+;
+; Expected merge for device_func:
+; FlatWGSizeMin = min(64,32) = 32
+; FlatWGSizeMax = max(256,128) = 256
+; WavesPerEUMin = max(2,4) = 4
+; WavesPerEUMax = max(8,6) = 8
+; MaxNumWGX = max(16,8) = 16
+; MaxNumWGY = max(16,8) = 16
+; MaxNumWGZ = max(1,8) = 8
+;
+; RUN: split-file %s %t
+; RUN: opt -thinlto-bc %t/kernels.ll -thin-link-bitcode-file=%t1.thinlink.bc -o %t1.bc
+; RUN: opt -thinlto-bc %t/device.ll -thin-link-bitcode-file=%t2.thinlink.bc -o %t2.bc
+; RUN: llvm-lto2 run %t1.bc %t2.bc -o %t.o \
+; RUN: -r %t1.bc,kernel1,px \
+; RUN: -r %t1.bc,kernel2,px \
+; RUN: -r %t1.bc,device_func,l \
+; RUN: -r %t2.bc,device_func,px \
+; RUN: -save-temps
+; RUN: llvm-dis %t.o.2.5.precodegen.bc -o - | FileCheck %s
+
+; CHECK: define void @device_func({{.*}}) {{.*}} #[[ATTR:[0-9]+]]
+; CHECK: attributes #[[ATTR]] = {
+; CHECK-SAME: "amdgpu-flat-work-group-size"="32,256"
+; CHECK-SAME: "amdgpu-max-num-workgroups"="16,16,8"
+; CHECK-SAME: "amdgpu-waves-per-eu"="4,8"
+
+;--- kernels.ll
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+declare void @device_func(ptr)
+
+define amdgpu_kernel void @kernel1(ptr %p) #0 {
+ call void @device_func(ptr %p)
+ ret void
+}
+
+define amdgpu_kernel void @kernel2(ptr %p) #1 {
+ call void @device_func(ptr %p)
+ ret void
+}
+
+attributes #0 = { "amdgpu-flat-work-group-size"="64,256" "amdgpu-waves-per-eu"="2,8" "amdgpu-max-num-workgroups"="16,16,1" }
+attributes #1 = { "amdgpu-flat-work-group-size"="32,128" "amdgpu-waves-per-eu"="4,6" "amdgpu-max-num-workgroups"="8,8,8" }
+
+;--- device.ll
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+define void @device_func(ptr %p) {
+ store i32 42, ptr %p
+ ret void
+}
diff --git a/llvm/test/ThinLTO/AMDGPU/amdgpu-occupancy-propagation.ll b/llvm/test/ThinLTO/AMDGPU/amdgpu-occupancy-propagation.ll
new file mode 100644
index 0000000000000..29a4adbce5e21
--- /dev/null
+++ b/llvm/test/ThinLTO/AMDGPU/amdgpu-occupancy-propagation.ll
@@ -0,0 +1,41 @@
+; Test cross-TU occupancy attribute propagation via ThinLTO.
+; A kernel in one module calls a device function in another module.
+; The device function should receive the kernel's occupancy attributes.
+;
+; RUN: split-file %s %t
+; RUN: opt -thinlto-bc %t/kernel.ll -thin-link-bitcode-file=%t1.thinlink.bc -o %t1.bc
+; RUN: opt -thinlto-bc %t/device.ll -thin-link-bitcode-file=%t2.thinlink.bc -o %t2.bc
+; RUN: llvm-lto2 run %t1.bc %t2.bc -o %t.o \
+; RUN: -r %t1.bc,kernel,px \
+; RUN: -r %t1.bc,device_func,l \
+; RUN: -r %t2.bc,device_func,px \
+; RUN: -save-temps
+; RUN: llvm-dis %t.o.2.5.precodegen.bc -o - | FileCheck %s
+
+; CHECK: define void @device_func({{.*}}) {{.*}} #[[ATTR:[0-9]+]]
+; CHECK: attributes #[[ATTR]] = {
+; CHECK-SAME: "amdgpu-flat-work-group-size"="64,256"
+; CHECK-SAME: "amdgpu-max-num-workgroups"="16,16,1"
+; CHECK-SAME: "amdgpu-waves-per-eu"="2,8"
+
+;--- kernel.ll
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+declare void @device_func(ptr)
+
+define amdgpu_kernel void @kernel(ptr %p) #0 {
+ call void @device_func(ptr %p)
+ ret void
+}
+
+attributes #0 = { "amdgpu-flat-work-group-size"="64,256" "amdgpu-waves-per-eu"="2,8" "amdgpu-max-num-workgroups"="16,16,1" }
+
+;--- device.ll
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+define void @device_func(ptr %p) {
+ store i32 42, ptr %p
+ ret void
+}
diff --git a/llvm/test/ThinLTO/AMDGPU/amdgpu-occupancy-target-default.ll b/llvm/test/ThinLTO/AMDGPU/amdgpu-occupancy-target-default.ll
new file mode 100644
index 0000000000000..ff810238ece2e
--- /dev/null
+++ b/llvm/test/ThinLTO/AMDGPU/amdgpu-occupancy-target-default.ll
@@ -0,0 +1,45 @@
+; Test that a target-dependent waves-per-EU maximum remains absent during
+; ThinLTO propagation. The AMDGPU backend supplies the gfx950 maximum.
+;
+; Use O0 so the pre-codegen IR directly shows the attribute applied by the
+; ThinLTO backend callback.
+;
+; RUN: split-file %s %t
+; RUN: opt -thinlto-bc %t/kernel.ll -thin-link-bitcode-file=%t1.thinlink.bc -o %t1.bc
+; RUN: opt -thinlto-bc %t/device.ll -thin-link-bitcode-file=%t2.thinlink.bc -o %t2.bc
+; RUN: llvm-lto2 run %t1.bc %t2.bc -o %t.s -O0 -mcpu=gfx950 -filetype=asm \
+; RUN: -amdgpu-enable-object-linking \
+; RUN: -r %t1.bc,kernel,px \
+; RUN: -r %t1.bc,device_func,l \
+; RUN: -r %t2.bc,device_func,px \
+; RUN: -save-temps
+; RUN: llvm-dis %t.s.2.5.precodegen.bc -o - | FileCheck %s
+; RUN: FileCheck %s --check-prefix=ASM --input-file=%t.s.2
+
+; CHECK: define void @device_func({{.*}}){{.*}} #[[ATTR:[0-9]+]]
+; CHECK: attributes #[[ATTR]] = {
+; CHECK-SAME: "amdgpu-waves-per-eu"="4"
+
+; ASM: .amdgpu_occupancy 4
+
+;--- kernel.ll
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+declare void @device_func(ptr)
+
+define amdgpu_kernel void @kernel(ptr %p) #0 {
+ call void @device_func(ptr %p)
+ ret void
+}
+
+attributes #0 = { "amdgpu-flat-work-group-size"="64,256" "amdgpu-waves-per-eu"="4" }
+
+;--- device.ll
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+define void @device_func(ptr %p) {
+ store volatile i32 42, ptr %p
+ ret void
+}
diff --git a/llvm/test/ThinLTO/AMDGPU/amdgpu-summary-prevailing.ll b/llvm/test/ThinLTO/AMDGPU/amdgpu-summary-prevailing.ll
new file mode 100644
index 0000000000000..e1b2e00b28376
--- /dev/null
+++ b/llvm/test/ThinLTO/AMDGPU/amdgpu-summary-prevailing.ll
@@ -0,0 +1,32 @@
+; Verify that the AMDGPU summary table keeps the prevailing definition's entry.
+; RUN: split-file %s %t
+; RUN: opt -module-summary < %t/a.ll -o %t/a.bc
+; RUN: opt -module-summary < %t/b.ll -o %t/b.bc
+; RUN: llvm-lto2 run %t/a.bc %t/b.bc -o %t/o \
+; RUN: -thinlto-distributed-indexes -r=%t/a.bc,f, -r=%t/b.bc,f,px
+; RUN: llvm-bcanalyzer -dump %t/a.bc.thinlto.bc | FileCheck %s
+; RUN: llvm-bcanalyzer -dump %t/b.bc.thinlto.bc | FileCheck %s
+
+; CHECK: <GLOBALVAL_SUMMARY_BLOCK
+; CHECK-NOT: <AMDGPU_ENTRY {{.*}} op4=2
+; CHECK: <AMDGPU_ENTRY op0=0 op1=2 op2=0 op3=0 op4=7 op5=0 op6=0 op7=0 op8=0/>
+; CHECK-NOT: <AMDGPU_ENTRY
+; CHECK: </GLOBALVAL_SUMMARY_BLOCK>
+
+;--- a.ll
+target triple = "amdgcn-amd-amdhsa"
+
+define weak_odr void @f() #0 {
+ ret void
+}
+
+attributes #0 = { "amdgpu-waves-per-eu"="2" }
+
+;--- b.ll
+target triple = "amdgcn-amd-amdhsa"
+
+define weak_odr void @f() #0 {
+ ret void
+}
+
+attributes #0 = { "amdgpu-waves-per-eu"="7" }
diff --git a/llvm/test/ThinLTO/AMDGPU/amdgpu-summary-roundtrip.ll b/llvm/test/ThinLTO/AMDGPU/amdgpu-summary-roundtrip.ll
new file mode 100644
index 0000000000000..b5cc3503783b5
--- /dev/null
+++ b/llvm/test/ThinLTO/AMDGPU/amdgpu-summary-roundtrip.ll
@@ -0,0 +1,45 @@
+; Verify that the AMDGPU summary entries round-trip through bitcode
+; as FS_AMDGPU_ENTRY records inside the GLOBALVAL_SUMMARY_BLOCK.
+; Only kernels and functions with explicit occupancy attributes get entries.
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -module-summary < %s -o %t.bc
+; RUN: llvm-bcanalyzer -dump %t.bc | FileCheck %s --check-prefix=PERMODULE
+; RUN: llvm-lto2 run %t.bc -o %t.o -thinlto-distributed-indexes \
+; RUN: -r=%t.bc,kernel,px -r=%t.bc,device_func,px -r=%t.bc,plain,px
+; RUN: llvm-bcanalyzer -dump %t.bc.thinlto.bc \
+; RUN: | FileCheck %s --check-prefix=COMBINED
+
+; PERMODULE: <GLOBALVAL_SUMMARY_BLOCK
+; PERMODULE-NOT: <AMDGPU_ENTRY
+; PERMODULE: <AMDGPU_ENTRY op0=91 op1=15 op2=64 op3=256 op4=2 op5=8 op6=16 op7=16 op8=1/>
+; PERMODULE-NOT: <AMDGPU_ENTRY
+; PERMODULE: <AMDGPU_ENTRY op0=0 op1=2 op2=0 op3=0 op4=4 op5=0 op6=0 op7=0 op8=0/>
+; PERMODULE-NOT: <AMDGPU_ENTRY
+; PERMODULE: </GLOBALVAL_SUMMARY_BLOCK>
+
+; COMBINED: <GLOBALVAL_SUMMARY_BLOCK
+; COMBINED-NOT: <AMDGPU_ENTRY
+; COMBINED: <AMDGPU_ENTRY op0=0 op1=2 op2=0 op3=0 op4=4 op5=0 op6=0 op7=0 op8=0/>
+; COMBINED-NOT: <AMDGPU_ENTRY
+; COMBINED: <AMDGPU_ENTRY op0=91 op1=15 op2=64 op3=256 op4=2 op5=8 op6=16 op7=16 op8=1/>
+; COMBINED-NOT: <AMDGPU_ENTRY
+; COMBINED: </GLOBALVAL_SUMMARY_BLOCK>
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+define amdgpu_kernel void @kernel(ptr %p) #0 {
+ call void @device_func(ptr %p)
+ ret void
+}
+
+define void @device_func(ptr %p) #1 {
+ store i32 42, ptr %p
+ ret void
+}
+
+define void @plain() {
+ ret void
+}
+
+attributes #0 = { "amdgpu-flat-work-group-size"="64,256" "amdgpu-waves-per-eu"="2,8" "amdgpu-max-num-workgroups"="16,16,1" }
+attributes #1 = { "amdgpu-waves-per-eu"="4" }
diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp
index 81cd1ea09ddb8..6f2e21bc44b70 100644
--- a/llvm/unittests/TargetParser/TargetParserTest.cpp
+++ b/llvm/unittests/TargetParser/TargetParserTest.cpp
@@ -3100,6 +3100,8 @@ TEST(TargetParserTest, testAMDGPUgetMaxHWAddressableLocalMemorySize) {
163840u);
EXPECT_EQ(AMDGPU::getMaxHWAddressableLocalMemorySize(AMDGPU::GK_GFX1250),
327680u);
+ EXPECT_EQ(AMDGPU::getMaxHWAddressableLocalMemorySize(AMDGPU::GK_GFX1310),
+ 196608u);
}
TEST(TargetParserTest, testAMDGPUgetNumWorkGroupSIMDs) {
@@ -3127,6 +3129,75 @@ TEST(TargetParserTest, testAMDGPUgetMaxWavesPerEU) {
EXPECT_EQ(AMDGPU::getMaxWavesPerEU(AMDGPU::GK_GFX1030), 16u);
}
+TEST(TargetParserTest, testAMDGPUObjectLinkingWaveSize) {
+ const Triple AMDHSA("amdgcn-amd-amdhsa");
+ const AMDGPU::TargetID GFX900(AMDHSA, "gfx900");
+ const AMDGPU::TargetID GFX1030(AMDHSA, "gfx1030");
+ const AMDGPU::TargetID GFX1250(AMDHSA, "gfx1250");
+ const AMDGPU::TargetID GFX1310(AMDHSA, "gfx1310");
+
+ EXPECT_FALSE(AMDGPU::isObjectLinkingWaveSizeSupported(GFX900, 32));
+ EXPECT_TRUE(AMDGPU::isObjectLinkingWaveSizeSupported(GFX900, 64));
+ EXPECT_TRUE(AMDGPU::isObjectLinkingWaveSizeSupported(GFX1030, 32));
+ EXPECT_TRUE(AMDGPU::isObjectLinkingWaveSizeSupported(GFX1030, 64));
+ EXPECT_TRUE(AMDGPU::isObjectLinkingWaveSizeSupported(GFX1250, 32));
+ EXPECT_FALSE(AMDGPU::isObjectLinkingWaveSizeSupported(GFX1250, 64));
+ EXPECT_TRUE(AMDGPU::isObjectLinkingWaveSizeSupported(GFX1310, 32));
+ EXPECT_TRUE(AMDGPU::isObjectLinkingWaveSizeSupported(GFX1310, 64));
+}
+
+TEST(TargetParserTest, testAMDGPUObjectLinkingRegisterEncoding) {
+ const Triple AMDHSA("amdgcn-amd-amdhsa");
+ const AMDGPU::TargetID GFX600(AMDHSA, "gfx600");
+ const AMDGPU::TargetID GFX900XnackOn(AMDHSA, "gfx900:xnack+");
+ const AMDGPU::TargetID GFX942XnackOff(AMDHSA, "gfx942:xnack-");
+ const AMDGPU::TargetID GFX1030(AMDHSA, "gfx1030");
+ const AMDGPU::TargetID GFX1250(AMDHSA, "gfx1250");
+
+ EXPECT_EQ(AMDGPU::getNumExtraSGPRs(GFX600, false, false), 0u);
+ EXPECT_EQ(AMDGPU::getNumExtraSGPRs(GFX600, true, false), 2u);
+ EXPECT_EQ(AMDGPU::getNumExtraSGPRs(GFX600, false, true), 4u);
+ EXPECT_EQ(AMDGPU::getNumExtraSGPRs(GFX900XnackOn, false, false), 4u);
+ EXPECT_EQ(AMDGPU::getNumExtraSGPRs(GFX900XnackOn, false, true), 6u);
+ EXPECT_EQ(AMDGPU::getNumExtraSGPRs(GFX942XnackOff, false, false), 6u);
+ EXPECT_EQ(AMDGPU::getNumExtraSGPRs(GFX1030, true, true), 2u);
+
+ EXPECT_EQ(AMDGPU::getEncodedNumVGPRBlocks(GFX600, 32, 64), 7u);
+ EXPECT_EQ(AMDGPU::getEncodedNumVGPRBlocks(GFX1030, 32, 32), 3u);
+ EXPECT_EQ(AMDGPU::getEncodedNumVGPRBlocks(GFX1250, 32, 32), 1u);
+
+ EXPECT_EQ(AMDGPU::getNumSGPRBlocks(0), 0u);
+ EXPECT_EQ(AMDGPU::getNumSGPRBlocks(8), 0u);
+ EXPECT_EQ(AMDGPU::getNumSGPRBlocks(9), 1u);
+}
+
+TEST(TargetParserTest, testAMDGPUObjectLinkingLDSOccupancy) {
+ const Triple AMDHSA("amdgcn-amd-amdhsa");
+ const AMDGPU::TargetID GFX900(AMDHSA, "gfx900");
+ const AMDGPU::TargetID GFX1250(AMDHSA, "gfx1250");
+ const AMDGPU::TargetID GFX1310(AMDHSA, "gfx1310");
+
+ EXPECT_TRUE(
+ AMDGPU::isLDSSizeCompatibleWithOccupancy(GFX900, 64, true, 21504, 10));
+ EXPECT_FALSE(
+ AMDGPU::isLDSSizeCompatibleWithOccupancy(GFX900, 64, true, 21505, 10));
+
+ EXPECT_TRUE(
+ AMDGPU::isLDSSizeCompatibleWithOccupancy(GFX1250, 32, true, 327680, 1));
+ EXPECT_FALSE(
+ AMDGPU::isLDSSizeCompatibleWithOccupancy(GFX1250, 32, true, 327681, 1));
+
+ // gfx13 has 192 KiB in full-SIMD mode and 96 KiB in half-SIMD mode.
+ EXPECT_TRUE(
+ AMDGPU::isLDSSizeCompatibleWithOccupancy(GFX1310, 64, false, 65000, 9));
+ EXPECT_FALSE(
+ AMDGPU::isLDSSizeCompatibleWithOccupancy(GFX1310, 64, false, 65537, 9));
+ EXPECT_TRUE(
+ AMDGPU::isLDSSizeCompatibleWithOccupancy(GFX1310, 64, true, 49000, 9));
+ EXPECT_FALSE(
+ AMDGPU::isLDSSizeCompatibleWithOccupancy(GFX1310, 64, true, 49153, 9));
+}
+
TEST(TargetParserTest, testAMDGPUParseTargetIDString) {
using AMDGPU::TargetID;
using AMDGPU::TargetIDSetting;
More information about the flang-commits
mailing list