[Mlir-commits] [llvm] [mlir] Claude/bold davinci 46xi9q (PR #233342)
Jianwen Ding
llvmlistbot at llvm.org
Sun Oct 18 22:31:54 PDT 2026
https://github.com/Jianwen-Ding created https://github.com/llvm/llvm-project/pull/233342
None
>From aae06b4d67993652e272d058d69c5e4d04aafc9b Mon Sep 17 00:00:00 2001
From: Claude <noreply at anthropic.com>
Date: Sun, 18 Oct 2026 20:00:42 +0000
Subject: [PATCH 01/22] [AGX] Add the agx architecture, agx_kernel calling
convention and intrinsics
Core support for the experimental Apple AGX GPU target: the agx triple
architecture and its data layout, the EM_AGX ELF machine (unregistered,
0x4147), the agx_kernel calling convention and the llvm.agx.* intrinsics
(special registers, barriers, SIMD-group operations, structured control
flow).
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
Claude-Session: https://claude.ai/code/session_01E56KEecg6edNaS23myRWER
---
llvm/CMakeLists.txt | 1 +
llvm/include/llvm/AsmParser/LLToken.h | 1 +
llvm/include/llvm/BinaryFormat/ELF.h | 3 +
llvm/include/llvm/IR/CMakeLists.txt | 1 +
llvm/include/llvm/IR/CallingConv.h | 4 +
llvm/include/llvm/IR/Intrinsics.td | 1 +
llvm/include/llvm/IR/IntrinsicsAGX.td | 177 +++++++++++++++++++++
llvm/include/llvm/Object/ELFObjectFile.h | 5 +
llvm/include/llvm/TargetParser/Triple.h | 10 +-
llvm/lib/AsmParser/LLLexer.cpp | 1 +
llvm/lib/AsmParser/LLParser.cpp | 2 +
llvm/lib/BinaryFormat/ELF.cpp | 3 +
llvm/lib/IR/AsmWriter.cpp | 1 +
llvm/lib/IR/Function.cpp | 1 +
llvm/lib/IR/Verifier.cpp | 5 +
llvm/lib/TargetParser/TargetDataLayout.cpp | 6 +
llvm/lib/TargetParser/Triple.cpp | 13 ++
17 files changed, 233 insertions(+), 2 deletions(-)
create mode 100644 llvm/include/llvm/IR/IntrinsicsAGX.td
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
index 5dd8797b50679..384e231165cc2 100644
--- a/llvm/CMakeLists.txt
+++ b/llvm/CMakeLists.txt
@@ -586,6 +586,7 @@ set(LLVM_ALL_TARGETS
)
set(LLVM_ALL_EXPERIMENTAL_TARGETS
+ AGX
ARC
CSKY
M68k
diff --git a/llvm/include/llvm/AsmParser/LLToken.h b/llvm/include/llvm/AsmParser/LLToken.h
index 5624410497ac8..3578b5fd830ea 100644
--- a/llvm/include/llvm/AsmParser/LLToken.h
+++ b/llvm/include/llvm/AsmParser/LLToken.h
@@ -184,6 +184,7 @@ enum Kind {
kw_amdgpu_kernel,
kw_amdgpu_gfx,
kw_amdgpu_gfx_whole_wave,
+ kw_agx_kernel,
kw_tailcc,
kw_m68k_rtdcc,
kw_graalcc,
diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h
index 2bc076a87cd1e..dbab8bf104dbf 100644
--- a/llvm/include/llvm/BinaryFormat/ELF.h
+++ b/llvm/include/llvm/BinaryFormat/ELF.h
@@ -325,6 +325,9 @@ enum {
EM_VE = 251, // NEC SX-Aurora VE
EM_CSKY = 252, // C-SKY 32-bit processor
EM_LOONGARCH = 258, // LoongArch
+ // Apple AGX GPU. This value is not assigned by the ELF ABI registry; it is
+ // used only for objects produced by the LLVM AGX backend (see AGXUsage.rst).
+ EM_AGX = 0x4147,
};
// Object file classes.
diff --git a/llvm/include/llvm/IR/CMakeLists.txt b/llvm/include/llvm/IR/CMakeLists.txt
index 5a7676b968589..33bf18ef2ce8e 100644
--- a/llvm/include/llvm/IR/CMakeLists.txt
+++ b/llvm/include/llvm/IR/CMakeLists.txt
@@ -8,6 +8,7 @@ set(LLVM_TARGET_DEFINITIONS Intrinsics.td)
tablegen(LLVM IntrinsicImpl.inc -gen-intrinsic-impl)
tablegen(LLVM IntrinsicEnums.inc -gen-intrinsic-enums)
tablegen(LLVM IntrinsicsAArch64.h -gen-intrinsic-enums -intrinsic-prefix=aarch64)
+tablegen(LLVM IntrinsicsAGX.h -gen-intrinsic-enums -intrinsic-prefix=agx)
tablegen(LLVM IntrinsicsAMDGPU.h -gen-intrinsic-enums -intrinsic-prefix=amdgcn)
tablegen(LLVM IntrinsicsARM.h -gen-intrinsic-enums -intrinsic-prefix=arm)
tablegen(LLVM IntrinsicsAVR.h -gen-intrinsic-enums -intrinsic-prefix=avr)
diff --git a/llvm/include/llvm/IR/CallingConv.h b/llvm/include/llvm/IR/CallingConv.h
index 249c512dda532..807a41b3199c4 100644
--- a/llvm/include/llvm/IR/CallingConv.h
+++ b/llvm/include/llvm/IR/CallingConv.h
@@ -297,6 +297,10 @@ namespace CallingConv {
/// stateless compartment.
CHERIoT_LibraryCall = 127,
+ /// Used for Apple AGX GPU compute kernels. Kernel arguments are delivered
+ /// in uniform registers, see AGXUsage.rst.
+ AGX_KERNEL = 128,
+
/// The highest possible ID. Must be some 2^k - 1.
MaxID = 1023
};
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 4494bf1e2a204..f14f3e6fd9ec7 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -2949,6 +2949,7 @@ include "llvm/IR/IntrinsicsHexagon.td"
include "llvm/IR/IntrinsicsNVVM.td"
include "llvm/IR/IntrinsicsMips.td"
include "llvm/IR/IntrinsicsAVR.td"
+include "llvm/IR/IntrinsicsAGX.td"
include "llvm/IR/IntrinsicsAMDGPU.td"
include "llvm/IR/IntrinsicsBPF.td"
include "llvm/IR/IntrinsicsSystemZ.td"
diff --git a/llvm/include/llvm/IR/IntrinsicsAGX.td b/llvm/include/llvm/IR/IntrinsicsAGX.td
new file mode 100644
index 0000000000000..c55b1915e39c8
--- /dev/null
+++ b/llvm/include/llvm/IR/IntrinsicsAGX.td
@@ -0,0 +1,177 @@
+//===- IntrinsicsAGX.td - Defines Apple AGX intrinsics -----*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines all of the Apple AGX GPU specific intrinsics. See
+// AGXUsage.rst for the semantics of each intrinsic.
+//
+//===----------------------------------------------------------------------===//
+
+let TargetPrefix = "agx" in {
+
+//===----------------------------------------------------------------------===//
+// Special registers and dispatch geometry.
+//===----------------------------------------------------------------------===//
+
+// Values that are fixed for the lifetime of a thread. They are read with
+// get_sr (or from implicit uniforms) and can be freely speculated.
+class AGXReadSRIntrinsic
+ : DefaultAttrsIntrinsic<[llvm_i32_ty], [],
+ [NoUndef<RetIndex>, IntrNoMem, IntrSpeculatable]>;
+
+multiclass AGXReadSRIntrinsic_xyz {
+ def _x : AGXReadSRIntrinsic;
+ def _y : AGXReadSRIntrinsic;
+ def _z : AGXReadSRIntrinsic;
+}
+
+defm int_agx_thread_position_in_grid : AGXReadSRIntrinsic_xyz;
+defm int_agx_threadgroup_position_in_grid : AGXReadSRIntrinsic_xyz;
+defm int_agx_threads_per_threadgroup : AGXReadSRIntrinsic_xyz;
+defm int_agx_thread_position_in_threadgroup : AGXReadSRIntrinsic_xyz;
+
+// The number of threadgroups in the grid. There is no special register for
+// this; the value is an implicit kernel argument (see AGXUsage.rst).
+defm int_agx_threadgroups_per_grid : AGXReadSRIntrinsic_xyz;
+
+def int_agx_thread_index_in_threadgroup : AGXReadSRIntrinsic;
+def int_agx_thread_index_in_simdgroup : AGXReadSRIntrinsic;
+def int_agx_simdgroup_index_in_threadgroup : AGXReadSRIntrinsic;
+def int_agx_core_id : AGXReadSRIntrinsic;
+
+// Values that depend on which threads of the SIMD-group are active.
+class AGXReadActiveSRIntrinsic
+ : DefaultAttrsIntrinsic<[llvm_i32_ty], [],
+ [NoUndef<RetIndex>, IntrNoMem, IntrConvergent]>;
+
+def int_agx_active_thread_index_in_simdgroup : AGXReadActiveSRIntrinsic;
+def int_agx_total_active_threads_in_simdgroup : AGXReadActiveSRIntrinsic;
+def int_agx_active_thread_index_in_quadgroup : AGXReadActiveSRIntrinsic;
+def int_agx_total_active_threads_in_quadgroup : AGXReadActiveSRIntrinsic;
+
+// Read an arbitrary special register by number. Prefer the named intrinsics.
+def int_agx_read_sr
+ : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty],
+ [IntrNoMem, IntrConvergent, ImmArg<ArgIndex<0>>]>;
+
+//===----------------------------------------------------------------------===//
+// Synchronization.
+//===----------------------------------------------------------------------===//
+
+// Execution barrier across the threadgroup (threadgroup_barrier).
+def int_agx_threadgroup_barrier
+ : Intrinsic<[], [],
+ [IntrNoMem, IntrHasSideEffects, IntrConvergent,
+ IntrWillReturn, IntrNoCallback, IntrNoFree]>;
+
+// Raw memory_barrier with its three hardware fields (a, b, c). The memory
+// model lowering of `fence` picks the encodings documented in AGXUsage.rst.
+def int_agx_memory_barrier
+ : Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
+ [IntrHasSideEffects, IntrWillReturn, IntrNoCallback,
+ IntrNoFree, ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
+ ImmArg<ArgIndex<2>>]>;
+
+// Wait for all outstanding asynchronous operations on a scoreboard slot.
+def int_agx_wait
+ : Intrinsic<[], [llvm_i32_ty],
+ [IntrHasSideEffects, IntrWillReturn, IntrNoCallback,
+ IntrNoFree, ImmArg<ArgIndex<0>>]>;
+
+//===----------------------------------------------------------------------===//
+// SIMD-group and quad-group operations.
+//===----------------------------------------------------------------------===//
+
+class AGXSIMDShuffleIntrinsic
+ : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>, llvm_i32_ty],
+ [IntrNoMem, IntrConvergent]>;
+
+def int_agx_simd_shuffle : AGXSIMDShuffleIntrinsic;
+def int_agx_simd_shuffle_xor : AGXSIMDShuffleIntrinsic;
+def int_agx_simd_shuffle_up : AGXSIMDShuffleIntrinsic;
+def int_agx_simd_shuffle_down : AGXSIMDShuffleIntrinsic;
+def int_agx_quad_shuffle : AGXSIMDShuffleIntrinsic;
+def int_agx_quad_shuffle_xor : AGXSIMDShuffleIntrinsic;
+def int_agx_quad_shuffle_up : AGXSIMDShuffleIntrinsic;
+def int_agx_quad_shuffle_down : AGXSIMDShuffleIntrinsic;
+
+class AGXBallotIntrinsic
+ : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i1_ty],
+ [IntrNoMem, IntrConvergent]>;
+
+def int_agx_simd_ballot : AGXBallotIntrinsic;
+def int_agx_quad_ballot : AGXBallotIntrinsic;
+
+// Reductions and exclusive scans. The second operand is the hardware
+// operation number (AGX::SIMDOp): and = 0, fadd = 1, or = 2, fmul = 3,
+// xor = 4, fmin = 5, fmax = 7, iadd = 16, smin = 20, smax = 22, umin = 28,
+// umax = 30.
+class AGXSIMDReduceIntrinsic
+ : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>, llvm_i32_ty],
+ [IntrNoMem, IntrConvergent, ImmArg<ArgIndex<1>>]>;
+
+def int_agx_simd_reduce : AGXSIMDReduceIntrinsic;
+def int_agx_simd_prefix : AGXSIMDReduceIntrinsic;
+def int_agx_quad_reduce : AGXSIMDReduceIntrinsic;
+def int_agx_quad_prefix : AGXSIMDReduceIntrinsic;
+
+// simdgroup 8x8 matrix multiply-accumulate. Each thread holds two elements
+// of each matrix: D = A * B + C.
+def int_agx_simdgroup_matrix_fma
+ : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+ [LLVMMatchType<0>, LLVMMatchType<0>,
+ LLVMMatchType<0>],
+ [IntrNoMem, IntrConvergent]>;
+
+//===----------------------------------------------------------------------===//
+// Arithmetic.
+//===----------------------------------------------------------------------===//
+
+class AGXFloatUnaryIntrinsic
+ : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>],
+ [IntrNoMem, IntrSpeculatable]>;
+
+// Hardware approximations, without any special case handling beyond what
+// the instruction does.
+def int_agx_rcp : AGXFloatUnaryIntrinsic;
+def int_agx_rsqrt : AGXFloatUnaryIntrinsic;
+// rsqrt with special cases chosen so that rsqrt_special(x) * x == sqrt(x).
+def int_agx_rsqrt_special : AGXFloatUnaryIntrinsic;
+// Sine building blocks, the argument is in quadrants: see AGXUsage.rst.
+def int_agx_sin_pt_1 : AGXFloatUnaryIntrinsic;
+def int_agx_sin_pt_2 : AGXFloatUnaryIntrinsic;
+
+// Screen space derivatives, only meaningful in fragment shaders.
+class AGXDerivativeIntrinsic
+ : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>],
+ [IntrNoMem, IntrConvergent]>;
+def int_agx_dfdx : AGXDerivativeIntrinsic;
+def int_agx_dfdy : AGXDerivativeIntrinsic;
+
+// Interleave the low 16 bits of the operands (Morton encode).
+def int_agx_interleave
+ : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
+ [IntrNoMem, IntrSpeculatable]>;
+
+//===----------------------------------------------------------------------===//
+// Structured control flow. These are inserted by AGXAnnotateControlFlow and
+// are not meant to be used by frontends.
+//===----------------------------------------------------------------------===//
+
+def int_agx_if : Intrinsic<[llvm_i1_ty], [llvm_i1_ty],
+ [IntrWillReturn, IntrNoCallback, IntrNoFree]>;
+def int_agx_else : Intrinsic<[llvm_i1_ty], [],
+ [IntrWillReturn, IntrNoCallback, IntrNoFree]>;
+def int_agx_end_cf : Intrinsic<[], [],
+ [IntrWillReturn, IntrNoCallback, IntrNoFree]>;
+def int_agx_loop_begin : Intrinsic<[], [],
+ [IntrWillReturn, IntrNoCallback,
+ IntrNoFree]>;
+def int_agx_loop : Intrinsic<[llvm_i1_ty], [llvm_i1_ty],
+ [IntrWillReturn, IntrNoCallback, IntrNoFree]>;
+
+} // TargetPrefix = "agx"
diff --git a/llvm/include/llvm/Object/ELFObjectFile.h b/llvm/include/llvm/Object/ELFObjectFile.h
index 6d72741c3bde0..fb7628353dcc2 100644
--- a/llvm/include/llvm/Object/ELFObjectFile.h
+++ b/llvm/include/llvm/Object/ELFObjectFile.h
@@ -1346,6 +1346,8 @@ StringRef ELFObjectFile<ELFT>::getFileFormatName() const {
return "elf64-mips";
case ELF::EM_AMDGPU:
return "elf64-amdgpu";
+ case ELF::EM_AGX:
+ return "elf64-agx";
case ELF::EM_BPF:
return "elf64-bpf";
case ELF::EM_VE:
@@ -1435,6 +1437,9 @@ template <class ELFT> Triple::ArchType ELFObjectFile<ELFT>::getArch() const {
return Triple::nvptx64;
}
+ case ELF::EM_AGX:
+ return IsLittleEndian ? Triple::agx : Triple::UnknownArch;
+
case ELF::EM_BPF:
return IsLittleEndian ? Triple::bpfel : Triple::bpfeb;
diff --git a/llvm/include/llvm/TargetParser/Triple.h b/llvm/include/llvm/TargetParser/Triple.h
index 6fb2904def2e5..15587e1554a4a 100644
--- a/llvm/include/llvm/TargetParser/Triple.h
+++ b/llvm/include/llvm/TargetParser/Triple.h
@@ -115,7 +115,8 @@ class Triple {
renderscript32, // 32-bit RenderScript
renderscript64, // 64-bit RenderScript
ve, // NEC SX-Aurora Vector Engine
- LastArchType = ve
+ agx, // AGX: Apple GPUs (G13 and later)
+ LastArchType = agx
};
enum SubArchType {
NoSubArch,
@@ -997,6 +998,9 @@ class Triple {
bool isAMDGPU() const { return isAMDGCN() || getArch() == Triple::r600; }
+ /// Tests whether the target is an Apple AGX GPU.
+ bool isAGX() const { return getArch() == Triple::agx; }
+
/// Tests whether the target is Thumb (little and big endian).
bool isThumb() const {
return getArch() == Triple::thumb || getArch() == Triple::thumbeb;
@@ -1411,7 +1415,9 @@ class Triple {
LLVM_ABI bool isCompatibleWith(const Triple &Other) const;
/// Test whether the target triple is for a GPU.
- bool isGPU() const { return isSPIROrSPIRV() || isNVPTX() || isAMDGPU(); }
+ bool isGPU() const {
+ return isSPIROrSPIRV() || isNVPTX() || isAMDGPU() || isAGX();
+ }
/// Merge target triples.
LLVM_ABI std::string merge(const Triple &Other) const;
diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp
index 97362c614091c..b3fb5e5412b63 100644
--- a/llvm/lib/AsmParser/LLLexer.cpp
+++ b/llvm/lib/AsmParser/LLLexer.cpp
@@ -697,6 +697,7 @@ lltok::Kind LLLexer::LexIdentifier() {
KEYWORD(amdgpu_kernel);
KEYWORD(amdgpu_gfx);
KEYWORD(amdgpu_gfx_whole_wave);
+ KEYWORD(agx_kernel);
KEYWORD(tailcc);
KEYWORD(m68k_rtdcc);
KEYWORD(graalcc);
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 30e8aeea508ab..7e06fb0322e24 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -2374,6 +2374,7 @@ void LLParser::parseOptionalDLLStorageClass(unsigned &Res) {
/// ::= 'amdgpu_cs_chain'
/// ::= 'amdgpu_cs_chain_preserve'
/// ::= 'amdgpu_kernel'
+/// ::= 'agx_kernel'
/// ::= 'tailcc'
/// ::= 'm68k_rtdcc'
/// ::= 'graalcc'
@@ -2449,6 +2450,7 @@ bool LLParser::parseOptionalCallingConv(unsigned &CC) {
CC = CallingConv::AMDGPU_CS_ChainPreserve;
break;
case lltok::kw_amdgpu_kernel: CC = CallingConv::AMDGPU_KERNEL; break;
+ case lltok::kw_agx_kernel: CC = CallingConv::AGX_KERNEL; break;
case lltok::kw_amdgpu_gfx_whole_wave:
CC = CallingConv::AMDGPU_Gfx_WholeWave;
break;
diff --git a/llvm/lib/BinaryFormat/ELF.cpp b/llvm/lib/BinaryFormat/ELF.cpp
index 4b430bc287ff9..0ecaa0e952218 100644
--- a/llvm/lib/BinaryFormat/ELF.cpp
+++ b/llvm/lib/BinaryFormat/ELF.cpp
@@ -191,6 +191,7 @@ uint16_t ELF::convertArchNameToEMachine(StringRef Arch) {
.Case("norc", EM_NORC)
.Case("csr_kalimba", EM_CSR_KALIMBA)
.Case("amdgpu", EM_AMDGPU)
+ .Case("agx", EM_AGX)
.Case("riscv", EM_RISCV)
.Case("lanai", EM_LANAI)
.Case("bpf", EM_BPF)
@@ -616,6 +617,8 @@ StringRef ELF::convertEMachineToArchName(uint16_t EMachine) {
return "csr_kalimba";
case EM_AMDGPU:
return "amdgpu";
+ case EM_AGX:
+ return "agx";
case EM_RISCV:
return "riscv";
case EM_LANAI:
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index 7593a4f8e8833..5be6825838e22 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -414,6 +414,7 @@ static void printCallingConv(unsigned cc, raw_ostream &Out) {
Out << "amdgpu_cs_chain_preserve";
break;
case CallingConv::AMDGPU_KERNEL: Out << "amdgpu_kernel"; break;
+ case CallingConv::AGX_KERNEL: Out << "agx_kernel"; break;
case CallingConv::AMDGPU_Gfx: Out << "amdgpu_gfx"; break;
case CallingConv::AMDGPU_Gfx_WholeWave:
Out << "amdgpu_gfx_whole_wave";
diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp
index c1fdfe1544c03..521a086916597 100644
--- a/llvm/lib/IR/Function.cpp
+++ b/llvm/lib/IR/Function.cpp
@@ -1195,6 +1195,7 @@ bool llvm::CallingConv::supportsNonVoidReturnType(CallingConv::ID CC) {
case CallingConv::SPIR_KERNEL:
case CallingConv::AMDGPU_CS_Chain:
case CallingConv::AMDGPU_CS_ChainPreserve:
+ case CallingConv::AGX_KERNEL:
return false;
case CallingConv::AMDGPU_VS:
case CallingConv::AMDGPU_HS:
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index ed7c68b24df1b..20db650d476b0 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -3299,6 +3299,11 @@ void Verifier::visitFunction(const Function &F) {
"Calling convention parameter requires byval", &F);
break;
}
+ case CallingConv::AGX_KERNEL:
+ Check(F.getReturnType()->isVoidTy(),
+ "Calling convention requires void return type", &F);
+ Check(!F.isVarArg(), "Calling convention does not support varargs", &F);
+ break;
case CallingConv::AMDGPU_KERNEL:
case CallingConv::SPIR_KERNEL:
case CallingConv::AMDGPU_CS_Chain:
diff --git a/llvm/lib/TargetParser/TargetDataLayout.cpp b/llvm/lib/TargetParser/TargetDataLayout.cpp
index 83bc4a624bf99..cd6567a796a0a 100644
--- a/llvm/lib/TargetParser/TargetDataLayout.cpp
+++ b/llvm/lib/TargetParser/TargetDataLayout.cpp
@@ -634,6 +634,12 @@ std::string Triple::computeDataLayout(StringRef ABIName) const {
case Triple::ppc64:
case Triple::ppc64le:
return computePowerDataLayout(*this, ABIName);
+ case Triple::agx:
+ // Device and constant pointers are 64-bit; threadgroup (3) and private
+ // (5) pointers are 32-bit offsets. Allocas are private.
+ return "e-m:e-p:64:64-p1:64:64-p3:32:32-p4:64:64-p5:32:32-i1:8-i8:8-"
+ "i16:16-i32:32-i64:64-f16:16-f32:32-f64:64-v16:16-v32:32-v64:64-"
+ "v96:32-v128:128-n16:32:64-S32-A5-G1";
case Triple::amdgpu:
case Triple::r600:
return computeAMDDataLayout(*this);
diff --git a/llvm/lib/TargetParser/Triple.cpp b/llvm/lib/TargetParser/Triple.cpp
index 4d17c36917f16..3cdc68e3e1581 100644
--- a/llvm/lib/TargetParser/Triple.cpp
+++ b/llvm/lib/TargetParser/Triple.cpp
@@ -93,6 +93,8 @@ StringRef Triple::getArchTypeName(ArchType Kind) {
return "mipsel";
case msp430:
return "msp430";
+ case agx:
+ return "agx";
case nvptx64:
return "nvptx64";
case nvptx:
@@ -320,6 +322,9 @@ StringRef Triple::getArchTypePrefix(ArchType Kind) {
case xcore:
return "xcore";
+ case agx:
+ return "agx";
+
// NVPTX intrinsics are namespaced under nvvm.
case nvptx:
return "nvvm";
@@ -503,6 +508,7 @@ Triple::ArchType Triple::getArchTypeForLLVMName(StringRef Name) {
.Case("xcore", xcore)
.Case("nvptx", nvptx)
.Case("nvptx64", nvptx64)
+ .Case("agx", agx)
.Case("amdil", amdil)
.Case("amdil64", amdil64)
.Case("hsail", hsail)
@@ -651,6 +657,7 @@ Triple::ArchType Triple::parseArch(StringRef ArchName) {
.Case("xcore", Triple::xcore)
.Case("nvptx", Triple::nvptx)
.Case("nvptx64", Triple::nvptx64)
+ .Case("agx", Triple::agx)
.Case("amdil", Triple::amdil)
.Case("amdil64", Triple::amdil64)
.Case("hsail", Triple::hsail)
@@ -986,6 +993,7 @@ static Triple::ObjectFormatType getDefaultFormat(const Triple &T) {
case Triple::mips64el:
case Triple::mips:
case Triple::msp430:
+ case Triple::agx:
case Triple::nvptx64:
case Triple::nvptx:
case Triple::ppc64le:
@@ -1788,6 +1796,7 @@ unsigned Triple::getArchPointerBitWidth(llvm::Triple::ArchType Arch) {
case llvm::Triple::loongarch64:
case llvm::Triple::mips64:
case llvm::Triple::mips64el:
+ case llvm::Triple::agx:
case llvm::Triple::nvptx64:
case llvm::Triple::ppc64:
case llvm::Triple::ppc64le:
@@ -1842,6 +1851,7 @@ Triple Triple::get32BitArchVariant() const {
Triple T(*this);
switch (getArch()) {
case Triple::UnknownArch:
+ case Triple::agx:
case Triple::amdgpu:
case Triple::avr:
case Triple::bpfeb:
@@ -1984,6 +1994,7 @@ Triple Triple::get64BitArchVariant() const {
case Triple::loongarch64:
case Triple::mips64:
case Triple::mips64el:
+ case Triple::agx:
case Triple::nvptx64:
case Triple::ppc64:
case Triple::ppc64le:
@@ -2091,6 +2102,7 @@ Triple Triple::getBigEndianArchVariant() const {
case Triple::loongarch32:
case Triple::loongarch64:
case Triple::msp430:
+ case Triple::agx:
case Triple::nvptx64:
case Triple::nvptx:
case Triple::r600:
@@ -2231,6 +2243,7 @@ bool Triple::isLittleEndian() const {
case Triple::mips64el:
case Triple::mipsel:
case Triple::msp430:
+ case Triple::agx:
case Triple::nvptx64:
case Triple::nvptx:
case Triple::ppcle:
>From dabcf22b50b043f645508ec93e08ec7be394446a Mon Sep 17 00:00:00 2001
From: Claude <noreply at anthropic.com>
Date: Sun, 18 Oct 2026 20:00:43 +0000
Subject: [PATCH 02/22] [AGX] Add the experimental AGX backend
MC layer for the G13/G14 (AGX2) encoding family, validated against the
applegpu reference assembler, and an initial code generator modelled on
AMDGPU: pseudo instructions mapped to real instructions per encoding
family, structured control flow on the execution mask nesting counter,
kernel arguments in uniform registers, uniform operand folding and
scoreboard wait insertion.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
Claude-Session: https://claude.ai/code/session_01E56KEecg6edNaS23myRWER
---
llvm/lib/Target/AGX/AGX.h | 49 +
llvm/lib/Target/AGX/AGX.td | 51 +
.../lib/Target/AGX/AGXAnnotateControlFlow.cpp | 252 ++++
llvm/lib/Target/AGX/AGXAsmPrinter.cpp | 130 ++
llvm/lib/Target/AGX/AGXCallingConv.td | 17 +
llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp | 233 +++
llvm/lib/Target/AGX/AGXFeatures.td | 196 +++
llvm/lib/Target/AGX/AGXFoldOperands.cpp | 112 ++
llvm/lib/Target/AGX/AGXFrameLowering.cpp | 38 +
llvm/lib/Target/AGX/AGXFrameLowering.h | 39 +
llvm/lib/Target/AGX/AGXISelDAGToDAG.cpp | 360 +++++
llvm/lib/Target/AGX/AGXISelLowering.cpp | 813 +++++++++++
llvm/lib/Target/AGX/AGXISelLowering.h | 101 ++
llvm/lib/Target/AGX/AGXInsertWaits.cpp | 109 ++
llvm/lib/Target/AGX/AGXInstrFormats.td | 116 ++
llvm/lib/Target/AGX/AGXInstrInfo.cpp | 307 ++++
llvm/lib/Target/AGX/AGXInstrInfo.h | 101 ++
llvm/lib/Target/AGX/AGXInstrInfo.td | 109 ++
llvm/lib/Target/AGX/AGXInstructions.td | 1270 +++++++++++++++++
llvm/lib/Target/AGX/AGXLowerControlFlow.cpp | 151 ++
llvm/lib/Target/AGX/AGXMCInstLower.cpp | 82 ++
llvm/lib/Target/AGX/AGXMCInstLower.h | 39 +
.../lib/Target/AGX/AGXMachineFunctionInfo.cpp | 27 +
llvm/lib/Target/AGX/AGXMachineFunctionInfo.h | 63 +
llvm/lib/Target/AGX/AGXOperands.td | 312 ++++
llvm/lib/Target/AGX/AGXProcessors.td | 50 +
llvm/lib/Target/AGX/AGXRegisterInfo.cpp | 98 ++
llvm/lib/Target/AGX/AGXRegisterInfo.h | 51 +
llvm/lib/Target/AGX/AGXRegisterInfo.td | 138 ++
llvm/lib/Target/AGX/AGXSchedule.td | 108 ++
llvm/lib/Target/AGX/AGXSearchableTables.td | 18 +
llvm/lib/Target/AGX/AGXSelectionDAGInfo.cpp | 17 +
llvm/lib/Target/AGX/AGXSelectionDAGInfo.h | 26 +
llvm/lib/Target/AGX/AGXSubtarget.cpp | 33 +
llvm/lib/Target/AGX/AGXSubtarget.h | 89 ++
llvm/lib/Target/AGX/AGXTargetMachine.cpp | 178 +++
llvm/lib/Target/AGX/AGXTargetMachine.h | 51 +
.../lib/Target/AGX/AGXTargetTransformInfo.cpp | 84 ++
llvm/lib/Target/AGX/AGXTargetTransformInfo.h | 56 +
.../lib/Target/AGX/AsmParser/AGXAsmParser.cpp | 864 +++++++++++
llvm/lib/Target/AGX/AsmParser/CMakeLists.txt | 15 +
llvm/lib/Target/AGX/CMakeLists.txt | 65 +
.../AGX/Disassembler/AGXDisassembler.cpp | 423 ++++++
.../Target/AGX/Disassembler/CMakeLists.txt | 14 +
llvm/lib/Target/AGX/G13Instructions.td | 1043 ++++++++++++++
.../Target/AGX/MCTargetDesc/AGXAsmBackend.cpp | 110 ++
.../AGX/MCTargetDesc/AGXELFObjectWriter.cpp | 42 +
.../Target/AGX/MCTargetDesc/AGXFixupKinds.h | 27 +
.../AGX/MCTargetDesc/AGXInstPrinter.cpp | 346 +++++
.../Target/AGX/MCTargetDesc/AGXInstPrinter.h | 89 ++
.../Target/AGX/MCTargetDesc/AGXMCAsmInfo.cpp | 30 +
.../Target/AGX/MCTargetDesc/AGXMCAsmInfo.h | 26 +
.../AGX/MCTargetDesc/AGXMCCodeEmitter.cpp | 386 +++++
.../AGX/MCTargetDesc/AGXMCTargetDesc.cpp | 114 ++
.../Target/AGX/MCTargetDesc/AGXMCTargetDesc.h | 52 +
.../Target/AGX/MCTargetDesc/CMakeLists.txt | 21 +
.../Target/AGX/TargetInfo/AGXTargetInfo.cpp | 23 +
.../lib/Target/AGX/TargetInfo/AGXTargetInfo.h | 18 +
llvm/lib/Target/AGX/TargetInfo/CMakeLists.txt | 10 +
llvm/lib/Target/AGX/Utils/AGXBaseInfo.cpp | 340 +++++
llvm/lib/Target/AGX/Utils/AGXBaseInfo.h | 185 +++
llvm/lib/Target/AGX/Utils/CMakeLists.txt | 12 +
llvm/test/MC/AGX/g13-alu.s | 178 +++
llvm/test/MC/AGX/g13-control.s | 117 ++
llvm/test/MC/AGX/g13-fp.s | 162 +++
llvm/test/MC/AGX/g13-memory.s | 44 +
llvm/test/MC/AGX/g13-simd.s | 466 ++++++
llvm/test/MC/AGX/lit.local.cfg | 2 +
llvm/test/MC/Disassembler/AGX/g13-alu.txt | 176 +++
llvm/test/MC/Disassembler/AGX/g13-control.txt | 115 ++
llvm/test/MC/Disassembler/AGX/g13-fp.txt | 160 +++
llvm/test/MC/Disassembler/AGX/g13-memory.txt | 42 +
llvm/test/MC/Disassembler/AGX/g13-simd.txt | 464 ++++++
llvm/test/MC/Disassembler/AGX/lit.local.cfg | 2 +
74 files changed, 12257 insertions(+)
create mode 100644 llvm/lib/Target/AGX/AGX.h
create mode 100644 llvm/lib/Target/AGX/AGX.td
create mode 100644 llvm/lib/Target/AGX/AGXAnnotateControlFlow.cpp
create mode 100644 llvm/lib/Target/AGX/AGXAsmPrinter.cpp
create mode 100644 llvm/lib/Target/AGX/AGXCallingConv.td
create mode 100644 llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp
create mode 100644 llvm/lib/Target/AGX/AGXFeatures.td
create mode 100644 llvm/lib/Target/AGX/AGXFoldOperands.cpp
create mode 100644 llvm/lib/Target/AGX/AGXFrameLowering.cpp
create mode 100644 llvm/lib/Target/AGX/AGXFrameLowering.h
create mode 100644 llvm/lib/Target/AGX/AGXISelDAGToDAG.cpp
create mode 100644 llvm/lib/Target/AGX/AGXISelLowering.cpp
create mode 100644 llvm/lib/Target/AGX/AGXISelLowering.h
create mode 100644 llvm/lib/Target/AGX/AGXInsertWaits.cpp
create mode 100644 llvm/lib/Target/AGX/AGXInstrFormats.td
create mode 100644 llvm/lib/Target/AGX/AGXInstrInfo.cpp
create mode 100644 llvm/lib/Target/AGX/AGXInstrInfo.h
create mode 100644 llvm/lib/Target/AGX/AGXInstrInfo.td
create mode 100644 llvm/lib/Target/AGX/AGXInstructions.td
create mode 100644 llvm/lib/Target/AGX/AGXLowerControlFlow.cpp
create mode 100644 llvm/lib/Target/AGX/AGXMCInstLower.cpp
create mode 100644 llvm/lib/Target/AGX/AGXMCInstLower.h
create mode 100644 llvm/lib/Target/AGX/AGXMachineFunctionInfo.cpp
create mode 100644 llvm/lib/Target/AGX/AGXMachineFunctionInfo.h
create mode 100644 llvm/lib/Target/AGX/AGXOperands.td
create mode 100644 llvm/lib/Target/AGX/AGXProcessors.td
create mode 100644 llvm/lib/Target/AGX/AGXRegisterInfo.cpp
create mode 100644 llvm/lib/Target/AGX/AGXRegisterInfo.h
create mode 100644 llvm/lib/Target/AGX/AGXRegisterInfo.td
create mode 100644 llvm/lib/Target/AGX/AGXSchedule.td
create mode 100644 llvm/lib/Target/AGX/AGXSearchableTables.td
create mode 100644 llvm/lib/Target/AGX/AGXSelectionDAGInfo.cpp
create mode 100644 llvm/lib/Target/AGX/AGXSelectionDAGInfo.h
create mode 100644 llvm/lib/Target/AGX/AGXSubtarget.cpp
create mode 100644 llvm/lib/Target/AGX/AGXSubtarget.h
create mode 100644 llvm/lib/Target/AGX/AGXTargetMachine.cpp
create mode 100644 llvm/lib/Target/AGX/AGXTargetMachine.h
create mode 100644 llvm/lib/Target/AGX/AGXTargetTransformInfo.cpp
create mode 100644 llvm/lib/Target/AGX/AGXTargetTransformInfo.h
create mode 100644 llvm/lib/Target/AGX/AsmParser/AGXAsmParser.cpp
create mode 100644 llvm/lib/Target/AGX/AsmParser/CMakeLists.txt
create mode 100644 llvm/lib/Target/AGX/CMakeLists.txt
create mode 100644 llvm/lib/Target/AGX/Disassembler/AGXDisassembler.cpp
create mode 100644 llvm/lib/Target/AGX/Disassembler/CMakeLists.txt
create mode 100644 llvm/lib/Target/AGX/G13Instructions.td
create mode 100644 llvm/lib/Target/AGX/MCTargetDesc/AGXAsmBackend.cpp
create mode 100644 llvm/lib/Target/AGX/MCTargetDesc/AGXELFObjectWriter.cpp
create mode 100644 llvm/lib/Target/AGX/MCTargetDesc/AGXFixupKinds.h
create mode 100644 llvm/lib/Target/AGX/MCTargetDesc/AGXInstPrinter.cpp
create mode 100644 llvm/lib/Target/AGX/MCTargetDesc/AGXInstPrinter.h
create mode 100644 llvm/lib/Target/AGX/MCTargetDesc/AGXMCAsmInfo.cpp
create mode 100644 llvm/lib/Target/AGX/MCTargetDesc/AGXMCAsmInfo.h
create mode 100644 llvm/lib/Target/AGX/MCTargetDesc/AGXMCCodeEmitter.cpp
create mode 100644 llvm/lib/Target/AGX/MCTargetDesc/AGXMCTargetDesc.cpp
create mode 100644 llvm/lib/Target/AGX/MCTargetDesc/AGXMCTargetDesc.h
create mode 100644 llvm/lib/Target/AGX/MCTargetDesc/CMakeLists.txt
create mode 100644 llvm/lib/Target/AGX/TargetInfo/AGXTargetInfo.cpp
create mode 100644 llvm/lib/Target/AGX/TargetInfo/AGXTargetInfo.h
create mode 100644 llvm/lib/Target/AGX/TargetInfo/CMakeLists.txt
create mode 100644 llvm/lib/Target/AGX/Utils/AGXBaseInfo.cpp
create mode 100644 llvm/lib/Target/AGX/Utils/AGXBaseInfo.h
create mode 100644 llvm/lib/Target/AGX/Utils/CMakeLists.txt
create mode 100644 llvm/test/MC/AGX/g13-alu.s
create mode 100644 llvm/test/MC/AGX/g13-control.s
create mode 100644 llvm/test/MC/AGX/g13-fp.s
create mode 100644 llvm/test/MC/AGX/g13-memory.s
create mode 100644 llvm/test/MC/AGX/g13-simd.s
create mode 100644 llvm/test/MC/AGX/lit.local.cfg
create mode 100644 llvm/test/MC/Disassembler/AGX/g13-alu.txt
create mode 100644 llvm/test/MC/Disassembler/AGX/g13-control.txt
create mode 100644 llvm/test/MC/Disassembler/AGX/g13-fp.txt
create mode 100644 llvm/test/MC/Disassembler/AGX/g13-memory.txt
create mode 100644 llvm/test/MC/Disassembler/AGX/g13-simd.txt
create mode 100644 llvm/test/MC/Disassembler/AGX/lit.local.cfg
diff --git a/llvm/lib/Target/AGX/AGX.h b/llvm/lib/Target/AGX/AGX.h
new file mode 100644
index 0000000000000..8bea7ed34e48d
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGX.h
@@ -0,0 +1,49 @@
+//===-- AGX.h - Top-level interface for AGX ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Entry points of the AGX code generator passes.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AGX_AGX_H
+#define LLVM_LIB_TARGET_AGX_AGX_H
+
+#include "llvm/Pass.h"
+
+namespace llvm {
+
+class AGXTargetMachine;
+class FunctionPass;
+class ModulePass;
+class PassRegistry;
+
+// IR passes.
+ModulePass *createAGXAlwaysInlinePass();
+FunctionPass *createAGXCodeGenPreparePass();
+FunctionPass *createAGXAnnotateControlFlowPass();
+
+// Instruction selection.
+FunctionPass *createAGXISelDag(AGXTargetMachine &TM);
+
+// Machine passes.
+FunctionPass *createAGXFoldOperandsPass();
+FunctionPass *createAGXLowerControlFlowPass();
+FunctionPass *createAGXInsertWaitsPass();
+
+void initializeAGXAlwaysInlinePass(PassRegistry &);
+void initializeAGXCodeGenPreparePass(PassRegistry &);
+void initializeAGXAnnotateControlFlowPass(PassRegistry &);
+void initializeAGXDAGToDAGISelLegacyPass(PassRegistry &);
+void initializeAGXFoldOperandsPass(PassRegistry &);
+void initializeAGXLowerControlFlowPass(PassRegistry &);
+void initializeAGXInsertWaitsPass(PassRegistry &);
+void initializeAGXAsmPrinterPass(PassRegistry &);
+
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AGX_AGX_H
diff --git a/llvm/lib/Target/AGX/AGX.td b/llvm/lib/Target/AGX/AGX.td
new file mode 100644
index 0000000000000..afcdd5976b5bd
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGX.td
@@ -0,0 +1,51 @@
+//===-- AGX.td - Apple AGX GPU target definition -----------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Top level target description of the Apple AGX GPUs (G13 and later). See
+// llvm/docs/AGXUsage.rst.
+//
+//===----------------------------------------------------------------------===//
+
+include "llvm/Target/Target.td"
+
+include "AGXFeatures.td"
+include "AGXRegisterInfo.td"
+include "AGXSchedule.td"
+include "AGXProcessors.td"
+include "AGXInstrInfo.td"
+include "AGXCallingConv.td"
+
+// Standard pseudo instructions that take a pointer use 64-bit register pairs.
+defm : RemapAllTargetPseudoPointerOperands<GPR64>;
+
+def AGXInstrInfo : InstrInfo {
+ let guessInstructionProperties = 1;
+}
+
+def AGXAsmParser : AsmParser {
+ let ShouldEmitMatchRegisterName = 0;
+}
+
+def AGXAsmParserVariant : AsmParserVariant {
+ let Variant = 0;
+ let Name = "";
+ let TokenizingCharacters = "";
+ let SeparatorCharacters = " \t,";
+}
+
+def AGXAsmWriter : AsmWriter {
+ int PassSubtarget = 1;
+}
+
+def AGX : Target {
+ let InstructionSet = AGXInstrInfo;
+ let AssemblyParsers = [AGXAsmParser];
+ let AssemblyParserVariants = [AGXAsmParserVariant];
+ let AssemblyWriters = [AGXAsmWriter];
+ let AllowRegisterRenaming = 1;
+}
diff --git a/llvm/lib/Target/AGX/AGXAnnotateControlFlow.cpp b/llvm/lib/Target/AGX/AGXAnnotateControlFlow.cpp
new file mode 100644
index 0000000000000..6c53385af1221
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXAnnotateControlFlow.cpp
@@ -0,0 +1,252 @@
+//===-- AGXAnnotateControlFlow.cpp - Annotate structured control flow -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Annotates the structurized control flow graph with the execution mask
+// intrinsics llvm.agx.if, llvm.agx.else, llvm.agx.end.cf, llvm.agx.loop.begin
+// and llvm.agx.loop, following SIAnnotateControlFlow.
+//
+// AGX keeps the execution mask in the per-thread nesting counter r0l: a
+// thread is active when its counter is zero. Unlike AMDGPU no mask needs to
+// be saved: "if" deactivates the threads where the condition is false
+// (counter 1) and pushes the inactive ones one level deeper, "else" swaps
+// the threads at the innermost level and "end.cf" pops a level. A loop
+// pushes a level on entry (llvm.agx.loop.begin, in a dedicated preheader);
+// at the latch, llvm.agx.loop deactivates the threads leaving the loop and
+// branches back while any thread remains; the exit pops the level.
+//
+// All branches are annotated, uniform or not: there are no scalar branches.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AGX.h"
+#include "llvm/ADT/DepthFirstIterator.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicsAGX.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/Local.h"
+#include "llvm/Transforms/Utils/LoopUtils.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "agx-annotate-control-flow"
+
+namespace {
+
+class AGXAnnotateControlFlowImpl {
+ Function &F;
+ DominatorTree &DT;
+ LoopInfo &LI;
+
+ /// Blocks where the innermost open construct reconverges.
+ SmallVector<BasicBlock *, 16> Stack;
+
+ ConstantInt *BoolTrue;
+ ConstantInt *BoolFalse;
+
+ bool isTopOfStack(BasicBlock *BB) const {
+ return !Stack.empty() && Stack.back() == BB;
+ }
+ bool isElse(PHINode *Phi) const;
+ void prepareLoops();
+
+ bool openIf(CondBrInst *Term);
+ bool insertElse(CondBrInst *Term);
+ bool handleLoop(CondBrInst *Term);
+ bool closeControlFlow(BasicBlock *BB);
+
+ CallInst *createCall(IRBuilder<> &IRB, Intrinsic::ID ID,
+ ArrayRef<Value *> Args = {}) {
+ return IRB.CreateCall(
+ Intrinsic::getOrInsertDeclaration(F.getParent(), ID), Args);
+ }
+
+public:
+ AGXAnnotateControlFlowImpl(Function &F, DominatorTree &DT, LoopInfo &LI)
+ : F(F), DT(DT), LI(LI) {
+ BoolTrue = ConstantInt::getTrue(F.getContext());
+ BoolFalse = ConstantInt::getFalse(F.getContext());
+ }
+
+ bool run();
+};
+
+class AGXAnnotateControlFlow : public FunctionPass {
+public:
+ static char ID;
+ AGXAnnotateControlFlow() : FunctionPass(ID) {}
+
+ StringRef getPassName() const override {
+ return "AGX annotate control flow";
+ }
+
+ bool runOnFunction(Function &F) override {
+ auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+ auto &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
+ return AGXAnnotateControlFlowImpl(F, DT, LI).run();
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<DominatorTreeWrapperPass>();
+ AU.addRequired<LoopInfoWrapperPass>();
+ AU.addPreserved<DominatorTreeWrapperPass>();
+ AU.addPreserved<LoopInfoWrapperPass>();
+ }
+};
+
+} // namespace
+
+char AGXAnnotateControlFlow::ID = 0;
+
+INITIALIZE_PASS_BEGIN(AGXAnnotateControlFlow, DEBUG_TYPE,
+ "Annotate AGX control flow", false, false)
+INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
+INITIALIZE_PASS_END(AGXAnnotateControlFlow, DEBUG_TYPE,
+ "Annotate AGX control flow", false, false)
+
+FunctionPass *llvm::createAGXAnnotateControlFlowPass() {
+ return new AGXAnnotateControlFlow();
+}
+
+/// Is \p Phi the condition of the else part of an if-then-else produced by
+/// the structurizer: true coming from the if block, false from the then
+/// block?
+bool AGXAnnotateControlFlowImpl::isElse(PHINode *Phi) const {
+ BasicBlock *IDom = DT.getNode(Phi->getParent())->getIDom()->getBlock();
+ for (unsigned I = 0, E = Phi->getNumIncomingValues(); I != E; ++I) {
+ Value *Expected = Phi->getIncomingBlock(I) == IDom ? BoolTrue : BoolFalse;
+ if (Phi->getIncomingValue(I) != Expected)
+ return false;
+ }
+ return true;
+}
+
+/// Give every loop a dedicated preheader that only branches to the header,
+/// so that the loop entry push is executed once, with the execution mask of
+/// the loop.
+void AGXAnnotateControlFlowImpl::prepareLoops() {
+ SmallVector<Loop *, 8> Loops(LI.getLoopsInPreorder());
+ for (Loop *L : Loops) {
+ BasicBlock *Pre = L->getLoopPreheader();
+ if (!Pre || !isa<UncondBrInst>(Pre->getTerminator()) ||
+ Pre->getSingleSuccessor() != L->getHeader())
+ Pre = InsertPreheaderForLoop(L, &DT, &LI, nullptr,
+ /*PreserveLCSSA=*/false);
+ if (!Pre)
+ report_fatal_error("AGX: cannot create a loop preheader");
+ }
+}
+
+bool AGXAnnotateControlFlowImpl::openIf(CondBrInst *Term) {
+ IRBuilder<> IRB(Term);
+ Term->setCondition(createCall(IRB, Intrinsic::agx_if, Term->getCondition()));
+ Stack.push_back(Term->getSuccessor(1));
+ return true;
+}
+
+bool AGXAnnotateControlFlowImpl::insertElse(CondBrInst *Term) {
+ Stack.pop_back();
+ IRBuilder<> IRB(Term);
+ Term->setCondition(createCall(IRB, Intrinsic::agx_else));
+ Stack.push_back(Term->getSuccessor(1));
+ return true;
+}
+
+bool AGXAnnotateControlFlowImpl::handleLoop(CondBrInst *Term) {
+ BasicBlock *BB = Term->getParent();
+ Loop *L = LI.getLoopFor(BB);
+ if (!L)
+ report_fatal_error("AGX: backward branch outside of a loop");
+
+ // The structurizer leaves loops with a single latch that branches to the
+ // exit when its condition is true.
+ BasicBlock *Header = Term->getSuccessor(1);
+ BasicBlock *Pre = L->getLoopPreheader();
+ assert(Pre && "loop preheaders are created by prepareLoops");
+ IRBuilder<> PreIRB(Pre->getTerminator());
+ createCall(PreIRB, Intrinsic::agx_loop_begin);
+
+ IRBuilder<> IRB(Term);
+ Term->setCondition(
+ createCall(IRB, Intrinsic::agx_loop, Term->getCondition()));
+ (void)Header;
+ Stack.push_back(Term->getSuccessor(0));
+ return true;
+}
+
+bool AGXAnnotateControlFlowImpl::closeControlFlow(BasicBlock *BB) {
+ assert(isTopOfStack(BB));
+ Loop *L = LI.getLoopFor(BB);
+ if (L && L->getHeader() == BB) {
+ // end.cf must not run on every iteration: move it to a block before the
+ // header, on the entry edges.
+ SmallVector<BasicBlock *, 2> Preds;
+ for (BasicBlock *Pred : predecessors(BB))
+ if (!L->contains(Pred))
+ Preds.push_back(Pred);
+ BB = SplitBlockPredecessors(BB, Preds, "endcf.split", &DT, &LI, nullptr,
+ false);
+ }
+
+ Stack.pop_back();
+ BasicBlock::iterator InsertPt = BB->getFirstInsertionPt();
+ IRBuilder<> IRB(BB, InsertPt);
+ IRB.SetCurrentDebugLocation(DebugLoc());
+ createCall(IRB, Intrinsic::agx_end_cf);
+ return true;
+}
+
+bool AGXAnnotateControlFlowImpl::run() {
+ prepareLoops();
+
+ bool Changed = false;
+ for (df_iterator<BasicBlock *> I = df_begin(&F.getEntryBlock()),
+ E = df_end(&F.getEntryBlock());
+ I != E; ++I) {
+ BasicBlock *BB = *I;
+ auto *Term = dyn_cast<CondBrInst>(BB->getTerminator());
+
+ if (!Term) {
+ if (isTopOfStack(BB))
+ Changed |= closeControlFlow(BB);
+ continue;
+ }
+
+ if (I.nodeVisited(Term->getSuccessor(1))) {
+ if (isTopOfStack(BB))
+ Changed |= closeControlFlow(BB);
+ if (DT.dominates(Term->getSuccessor(1), BB))
+ Changed |= handleLoop(Term);
+ continue;
+ }
+
+ if (isTopOfStack(BB)) {
+ auto *Phi = dyn_cast<PHINode>(Term->getCondition());
+ if (Phi && Phi->getParent() == BB && isElse(Phi)) {
+ Changed |= insertElse(Term);
+ RecursivelyDeleteDeadPHINode(Phi);
+ continue;
+ }
+ Changed |= closeControlFlow(BB);
+ }
+
+ Changed |= openIf(Term);
+ }
+
+ if (!Stack.empty())
+ report_fatal_error("AGX: failed to annotate the control flow graph (it "
+ "is not structured)");
+ return Changed;
+}
diff --git a/llvm/lib/Target/AGX/AGXAsmPrinter.cpp b/llvm/lib/Target/AGX/AGXAsmPrinter.cpp
new file mode 100644
index 0000000000000..9c067049c8cb4
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXAsmPrinter.cpp
@@ -0,0 +1,130 @@
+//===-- AGXAsmPrinter.cpp - AGX assembly printer --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Emits AGX machine code. Each kernel is preceded by comments describing the
+// resources the driver must allocate for it (uniform registers, general
+// purpose registers, threadgroup memory and stack), and followed by trap
+// padding, since the instruction prefetcher reads past the end of the
+// program.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AGX.h"
+#include "AGXFrameLowering.h"
+#include "AGXMCInstLower.h"
+#include "AGXMachineFunctionInfo.h"
+#include "AGXSubtarget.h"
+#include "MCTargetDesc/AGXMCTargetDesc.h"
+#include "TargetInfo/AGXTargetInfo.h"
+#include "llvm/CodeGen/AsmPrinter.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Support/Compiler.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "agx-asm-printer"
+
+namespace {
+
+class AGXAsmPrinter : public AsmPrinter {
+ bool ReportedMissingEncoding = false;
+
+public:
+ static char ID;
+
+ explicit AGXAsmPrinter(TargetMachine &TM,
+ std::unique_ptr<MCStreamer> Streamer)
+ : AsmPrinter(TM, std::move(Streamer), ID) {}
+
+ StringRef getPassName() const override { return "AGX Assembly Printer"; }
+
+ bool runOnMachineFunction(MachineFunction &MF) override {
+ ReportedMissingEncoding = false;
+ return AsmPrinter::runOnMachineFunction(MF);
+ }
+
+ void emitFunctionBodyStart() override;
+ void emitFunctionBodyEnd() override;
+ void emitInstruction(const MachineInstr *MI) override;
+};
+
+} // namespace
+
+char AGXAsmPrinter::ID = 0;
+
+INITIALIZE_PASS(AGXAsmPrinter, "agx-asm-printer", "AGX Assembly Printer",
+ false, false)
+
+/// The number of 32-bit general purpose registers used, counting from r0.
+static unsigned getGPRCount(const MachineFunction &MF) {
+ const MachineRegisterInfo &MRI = MF.getRegInfo();
+ unsigned Count = 1; // r0l is always used for the execution mask.
+ for (unsigned I = 0; I < 128; ++I)
+ if (MRI.isPhysRegUsed(AGX::GPR32RegClass.getRegister(I)))
+ Count = I + 1;
+ return Count;
+}
+
+void AGXAsmPrinter::emitFunctionBodyStart() {
+ if (!isVerbose())
+ return;
+ const auto *MFI = MF->getInfo<AGXMachineFunctionInfo>();
+ OutStreamer->emitRawComment(" uniform registers: " +
+ Twine(MFI->getUniformHalfCount()) +
+ " x 16-bit (threadgroups_per_grid at u" +
+ Twine(MFI->getImplicitArgHalfIndex() / 2) + ")");
+ OutStreamer->emitRawComment(" general purpose registers: " +
+ Twine(getGPRCount(*MF)) + " x 32-bit");
+ OutStreamer->emitRawComment(" threadgroup memory: " +
+ Twine(MFI->getThreadgroupSize()) + " bytes");
+ OutStreamer->emitRawComment(" stack: " +
+ Twine(AGXFrameLowering::getStackAdjustUnits(*MF) *
+ 4) +
+ " bytes");
+}
+
+void AGXAsmPrinter::emitFunctionBodyEnd() {
+ const auto &ST = MF->getSubtarget<AGXSubtarget>();
+ if (ST.getEncodingFamily() != AGXSubtarget::AGX2)
+ return;
+ for (unsigned I = 0; I < 8; ++I) {
+ MCInst Trap;
+ Trap.setOpcode(AGX::TRAP_g13);
+ EmitToStreamer(*OutStreamer, Trap);
+ }
+}
+
+void AGXAsmPrinter::emitInstruction(const MachineInstr *MI) {
+ const auto &ST = MF->getSubtarget<AGXSubtarget>();
+ AGXMCInstLower Lower(OutContext, ST);
+ MCInst Inst;
+ if (!Lower.lower(MI, Inst)) {
+ if (!ReportedMissingEncoding) {
+ OutContext.reportError(
+ SMLoc(), "no AGX instruction encoding for '" +
+ Twine(ST.getInstrInfo()->getName(MI->getOpcode())) +
+ "' on " + ST.getCPU() +
+ (ST.getEncodingFamily() == AGXSubtarget::Apple9
+ ? " (the Apple9 encoding is not yet supported)"
+ : ""));
+ ReportedMissingEncoding = true;
+ }
+ return;
+ }
+ EmitToStreamer(*OutStreamer, Inst);
+}
+
+extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
+LLVMInitializeAGXAsmPrinter() {
+ RegisterAsmPrinter<AGXAsmPrinter> X(getTheAGXTarget());
+}
diff --git a/llvm/lib/Target/AGX/AGXCallingConv.td b/llvm/lib/Target/AGX/AGXCallingConv.td
new file mode 100644
index 0000000000000..515ce18c17b13
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXCallingConv.td
@@ -0,0 +1,17 @@
+//===-- AGXCallingConv.td - AGX calling conventions --------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// AGX kernels receive their arguments in uniform registers. The layout is
+// computed by AGXTargetLowering::LowerFormalArguments (natural alignment in
+// 16-bit units, see AGXUsage.rst), not by a CCState, so only the callee saved
+// register list is described here. There are no callable functions: all calls
+// are inlined before code generation.
+//
+//===----------------------------------------------------------------------===//
+
+def CSR_AGX_NoRegs : CalleeSavedRegs<(add)>;
diff --git a/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp b/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp
new file mode 100644
index 0000000000000..ccabfe5ce075b
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp
@@ -0,0 +1,233 @@
+//===-- AGXCodeGenPrepare.cpp - AGX IR preparation ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// IR transformations that AGX code generation relies on:
+//
+// * AGXAlwaysInline marks every non-kernel function always_inline (and
+// internal, so it disappears once inlined), since there are no calls.
+// * AGXCodeGenPrepare expands what the hardware has no instruction or
+// library for: integer division, and memory intrinsics that are not
+// small enough to be expanded by instruction selection. It also unifies
+// the function exits, which the control flow structurizer expects.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AGX.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstIterator.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Module.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Transforms/Utils/IntegerDivision.h"
+#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "agx-codegenprepare"
+
+//===----------------------------------------------------------------------===//
+// AGXAlwaysInline
+//===----------------------------------------------------------------------===//
+
+namespace {
+
+class AGXAlwaysInline : public ModulePass {
+public:
+ static char ID;
+ AGXAlwaysInline() : ModulePass(ID) {}
+
+ StringRef getPassName() const override { return "AGX Always Inline"; }
+ bool runOnModule(Module &M) override;
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesAll();
+ }
+};
+
+} // namespace
+
+char AGXAlwaysInline::ID = 0;
+
+INITIALIZE_PASS(AGXAlwaysInline, "agx-always-inline", "AGX Always Inline",
+ false, false)
+
+ModulePass *llvm::createAGXAlwaysInlinePass() { return new AGXAlwaysInline(); }
+
+bool AGXAlwaysInline::runOnModule(Module &M) {
+ // Modules without any agx_kernel function (for example tests of single
+ // functions) compile every external function as an entry point.
+ bool HasKernels = any_of(M, [](const Function &F) {
+ return F.getCallingConv() == CallingConv::AGX_KERNEL;
+ });
+
+ bool Changed = false;
+ for (Function &F : M) {
+ if (F.isDeclaration() || F.getCallingConv() == CallingConv::AGX_KERNEL)
+ continue;
+ if (!HasKernels && !F.hasLocalLinkage())
+ continue;
+ if (F.hasFnAttribute(Attribute::NoInline))
+ continue;
+ F.addFnAttr(Attribute::AlwaysInline);
+ if (HasKernels && !F.hasLocalLinkage() && !F.use_empty()) {
+ F.setLinkage(GlobalValue::InternalLinkage);
+ F.setVisibility(GlobalValue::DefaultVisibility);
+ }
+ Changed = true;
+ }
+ return Changed;
+}
+
+//===----------------------------------------------------------------------===//
+// AGXCodeGenPrepare
+//===----------------------------------------------------------------------===//
+
+namespace {
+
+class AGXCodeGenPrepare : public FunctionPass {
+ bool expandDivisions(Function &F);
+ bool expandMemIntrinsics(Function &F, const TargetTransformInfo &TTI);
+ bool unifyExits(Function &F);
+
+public:
+ static char ID;
+ AGXCodeGenPrepare() : FunctionPass(ID) {}
+
+ StringRef getPassName() const override { return "AGX CodeGen Prepare"; }
+ bool runOnFunction(Function &F) override;
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<TargetTransformInfoWrapperPass>();
+ }
+};
+
+} // namespace
+
+char AGXCodeGenPrepare::ID = 0;
+
+INITIALIZE_PASS_BEGIN(AGXCodeGenPrepare, DEBUG_TYPE, "AGX CodeGen Prepare",
+ false, false)
+INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
+INITIALIZE_PASS_END(AGXCodeGenPrepare, DEBUG_TYPE, "AGX CodeGen Prepare",
+ false, false)
+
+FunctionPass *llvm::createAGXCodeGenPreparePass() {
+ return new AGXCodeGenPrepare();
+}
+
+/// Division by a power of two is a shift, and 32-bit division by another
+/// constant is a multiply-high; both are left to instruction selection.
+static bool isCheapDivision(const BinaryOperator &BO) {
+ auto *C = dyn_cast<ConstantInt>(BO.getOperand(1));
+ if (!C)
+ return false;
+ if (C->getValue().isPowerOf2())
+ return true;
+ return BO.getType()->isIntegerTy(32);
+}
+
+bool AGXCodeGenPrepare::expandDivisions(Function &F) {
+ SmallVector<BinaryOperator *, 8> Worklist;
+ for (Instruction &I : instructions(F)) {
+ auto *BO = dyn_cast<BinaryOperator>(&I);
+ if (!BO || !BO->getType()->isIntegerTy())
+ continue;
+ switch (BO->getOpcode()) {
+ case Instruction::UDiv:
+ case Instruction::SDiv:
+ case Instruction::URem:
+ case Instruction::SRem:
+ if (!isCheapDivision(*BO) &&
+ BO->getType()->getIntegerBitWidth() <= 64)
+ Worklist.push_back(BO);
+ break;
+ default:
+ break;
+ }
+ }
+
+ for (BinaryOperator *BO : Worklist) {
+ bool IsRem = BO->getOpcode() == Instruction::URem ||
+ BO->getOpcode() == Instruction::SRem;
+ bool Is64 = BO->getType()->getIntegerBitWidth() > 32;
+ if (IsRem)
+ Is64 ? expandRemainderUpTo64Bits(BO) : expandRemainderUpTo32Bits(BO);
+ else
+ Is64 ? expandDivisionUpTo64Bits(BO) : expandDivisionUpTo32Bits(BO);
+ }
+ return !Worklist.empty();
+}
+
+bool AGXCodeGenPrepare::expandMemIntrinsics(Function &F,
+ const TargetTransformInfo &TTI) {
+ // Instruction selection expands memory intrinsics of a known, small size
+ // into loads and stores; everything else becomes a loop.
+ constexpr uint64_t MaxInlineSize = 128;
+ SmallVector<MemIntrinsic *, 4> Worklist;
+ for (Instruction &I : instructions(F)) {
+ auto *MI = dyn_cast<MemIntrinsic>(&I);
+ if (!MI)
+ continue;
+ auto *Len = dyn_cast<ConstantInt>(MI->getLength());
+ if (Len && Len->getZExtValue() <= MaxInlineSize)
+ continue;
+ Worklist.push_back(MI);
+ }
+
+ for (MemIntrinsic *MI : Worklist) {
+ if (auto *Cpy = dyn_cast<MemCpyInst>(MI)) {
+ expandMemCpyAsLoop(Cpy, TTI);
+ } else if (auto *Move = dyn_cast<MemMoveInst>(MI)) {
+ if (!expandMemMoveAsLoop(Move, TTI))
+ continue;
+ } else if (auto *Set = dyn_cast<MemSetInst>(MI)) {
+ expandMemSetAsLoop(Set, TTI);
+ } else {
+ continue;
+ }
+ MI->eraseFromParent();
+ }
+ return !Worklist.empty();
+}
+
+/// Replace the returns, and unreachable terminators, by branches to a single
+/// return block.
+bool AGXCodeGenPrepare::unifyExits(Function &F) {
+ SmallVector<Instruction *, 4> Exits;
+ for (BasicBlock &BB : F) {
+ Instruction *T = BB.getTerminator();
+ if (isa<ReturnInst>(T) || isa<UnreachableInst>(T))
+ Exits.push_back(T);
+ }
+ if (Exits.size() <= 1)
+ return false;
+ if (!F.getReturnType()->isVoidTy())
+ return false;
+
+ BasicBlock *Ret = BasicBlock::Create(F.getContext(), "unified.return", &F);
+ ReturnInst::Create(F.getContext(), Ret);
+ for (Instruction *T : Exits) {
+ UncondBrInst::Create(Ret, T->getIterator());
+ T->eraseFromParent();
+ }
+ return true;
+}
+
+bool AGXCodeGenPrepare::runOnFunction(Function &F) {
+ if (skipFunction(F))
+ return false;
+ const TargetTransformInfo &TTI =
+ getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
+ bool Changed = expandDivisions(F);
+ Changed |= expandMemIntrinsics(F, TTI);
+ Changed |= unifyExits(F);
+ return Changed;
+}
diff --git a/llvm/lib/Target/AGX/AGXFeatures.td b/llvm/lib/Target/AGX/AGXFeatures.td
new file mode 100644
index 0000000000000..d692d160b143a
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXFeatures.td
@@ -0,0 +1,196 @@
+//===-- AGXFeatures.td - AGX feature definitions -----------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Subtarget features of the Apple AGX GPUs. Features fall in three groups:
+//
+// * Generation features (FeatureG13 ... FeatureG17). Exactly one is set by
+// each processor. They select defaults, not legality.
+// * Encoding family features. They select which real (MC) instruction a
+// pseudo instruction is lowered to.
+// * Capability features. Every capability can be enabled or disabled on any
+// processor with -mattr, so code paths can be tried on generations where
+// the capability is not known to exist.
+//
+// The provenance of each capability is recorded in AGXUsage.rst.
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// Encoding families
+//===----------------------------------------------------------------------===//
+
+def FeatureAGX2Encoding : SubtargetFeature<"agx2-encoding",
+ "HasAGX2Encoding", "true",
+ "Uses the G13/G14 (\"AGX2\") instruction encoding"
+>;
+
+def FeatureApple9Encoding : SubtargetFeature<"apple9-encoding",
+ "HasApple9Encoding", "true",
+ "Uses the Apple9 (G16/G17) instruction encoding. Code generation for this "
+ "encoding is experimental"
+>;
+
+//===----------------------------------------------------------------------===//
+// Capability features
+//===----------------------------------------------------------------------===//
+
+def FeatureFP16Arith : SubtargetFeature<"fp16-arith",
+ "HasFP16Arith", "true",
+ "Has native half precision fadd/fmul/fmadd"
+>;
+
+def FeatureInt64Add : SubtargetFeature<"int64-add",
+ "HasInt64Add", "true",
+ "iadd and imadd accept 64-bit destinations and sources"
+>;
+
+def FeatureSIMDShuffle : SubtargetFeature<"simd-shuffle",
+ "HasSIMDShuffle", "true",
+ "Has simd_shuffle and quad_shuffle instructions"
+>;
+
+def FeatureSIMDReduce : SubtargetFeature<"simd-reduce",
+ "HasSIMDReduce", "true",
+ "Has simd/quad reduction and prefix instructions"
+>;
+
+def FeatureSIMDBallot : SubtargetFeature<"simd-ballot",
+ "HasSIMDBallot", "true",
+ "Has icmp_ballot and fcmp_ballot"
+>;
+
+def FeatureSIMDMatrix : SubtargetFeature<"simd-matrix",
+ "HasSIMDMatrix", "true",
+ "Has simdgroup 8x8 matrix multiply-accumulate"
+>;
+
+def FeatureQuadShuffleIndexMerge : SubtargetFeature<"quad-shuffle-index-merge",
+ "HasQuadShuffleIndexMerge", "true",
+ "simd_shuffle ORs the low two index bits across each quad, so divergent "
+ "shuffle indices must be lowered with a loop"
+>;
+
+def FeatureMultiDieCoherency : SubtargetFeature<"multi-die-coherency",
+ "HasMultiDieCoherency", "true",
+ "Device atomics must set the coherency bit (multi-die G13 parts)"
+>;
+
+def FeatureThreadgroupAtomics : SubtargetFeature<"threadgroup-atomics",
+ "HasThreadgroupAtomics", "true",
+ "Has threadgroup memory atomics"
+>;
+
+def FeatureDeviceAtomics : SubtargetFeature<"device-atomics",
+ "HasDeviceAtomics", "true",
+ "Has device memory atomics"
+>;
+
+def FeatureStackAccess : SubtargetFeature<"stack-access",
+ "HasStackAccess", "true",
+ "Has stack_load/stack_store for register spilling"
+>;
+
+def FeatureDynamicCaching : SubtargetFeature<"dynamic-caching",
+ "HasDynamicCaching", "true",
+ "Register file and threadgroup memory are allocated dynamically "
+ "(no effect on code generation, reported for occupancy)"
+>;
+
+def FeatureRayTracing : SubtargetFeature<"ray-tracing",
+ "HasRayTracing", "true",
+ "Has hardware ray tracing instructions (not used by code generation)"
+>;
+
+def FeatureMeshShading : SubtargetFeature<"mesh-shading",
+ "HasMeshShading", "true",
+ "Has hardware mesh shading (not used by code generation)"
+>;
+
+def FeatureSourceLifetime : SubtargetFeature<"source-lifetime",
+ "HasSourceLifetime", "true",
+ "ALU sources carry release flags that free the register after the read; "
+ "a released register reads as zero (Apple9)"
+>;
+
+//===----------------------------------------------------------------------===//
+// Register file size
+//===----------------------------------------------------------------------===//
+
+class FeatureMaxGPRs<int Value> : SubtargetFeature<
+ "max-gprs-"#Value,
+ "MaxGPRs",
+ !cast<string>(Value),
+ "Up to "#Value#" addressable 32-bit general purpose registers"
+>;
+
+def FeatureMaxGPRs128 : FeatureMaxGPRs<128>;
+def FeatureMaxGPRs64 : FeatureMaxGPRs<64>;
+
+//===----------------------------------------------------------------------===//
+// Generations
+//===----------------------------------------------------------------------===//
+
+class AGXSubtargetFeatureGeneration<string Value, string FeatureName,
+ list<SubtargetFeature> Implies>
+ : SubtargetFeature<FeatureName, "Gen", "AGXSubtarget::"#Value,
+ Value#" GPU generation", Implies>;
+
+def FeatureG13 : AGXSubtargetFeatureGeneration<"G13", "g13",
+ [FeatureAGX2Encoding, FeatureFP16Arith, FeatureInt64Add, FeatureSIMDShuffle,
+ FeatureSIMDReduce, FeatureSIMDBallot, FeatureSIMDMatrix,
+ FeatureQuadShuffleIndexMerge, FeatureThreadgroupAtomics,
+ FeatureDeviceAtomics, FeatureStackAccess, FeatureMaxGPRs128]
+>;
+
+def FeatureG14 : AGXSubtargetFeatureGeneration<"G14", "g14",
+ [FeatureAGX2Encoding, FeatureFP16Arith, FeatureInt64Add, FeatureSIMDShuffle,
+ FeatureSIMDReduce, FeatureSIMDBallot, FeatureSIMDMatrix,
+ FeatureQuadShuffleIndexMerge, FeatureThreadgroupAtomics,
+ FeatureDeviceAtomics, FeatureStackAccess, FeatureMaxGPRs128]
+>;
+
+// No public instruction set documentation exists for G15. It is assumed to
+// keep the G13/G14 encoding (see AGXUsage.rst, "Provenance").
+def FeatureG15 : AGXSubtargetFeatureGeneration<"G15", "g15",
+ [FeatureAGX2Encoding, FeatureFP16Arith, FeatureInt64Add, FeatureSIMDShuffle,
+ FeatureSIMDReduce, FeatureSIMDBallot, FeatureSIMDMatrix,
+ FeatureQuadShuffleIndexMerge, FeatureThreadgroupAtomics,
+ FeatureDeviceAtomics, FeatureStackAccess, FeatureMaxGPRs128,
+ FeatureDynamicCaching, FeatureRayTracing, FeatureMeshShading]
+>;
+
+def FeatureG16 : AGXSubtargetFeatureGeneration<"G16", "g16",
+ [FeatureApple9Encoding, FeatureFP16Arith, FeatureSIMDShuffle,
+ FeatureSIMDReduce, FeatureSIMDBallot, FeatureSIMDMatrix,
+ FeatureThreadgroupAtomics, FeatureDeviceAtomics, FeatureMaxGPRs64,
+ FeatureDynamicCaching, FeatureRayTracing, FeatureMeshShading,
+ FeatureSourceLifetime]
+>;
+
+def FeatureG17 : AGXSubtargetFeatureGeneration<"G17", "g17",
+ [FeatureApple9Encoding, FeatureFP16Arith, FeatureSIMDShuffle,
+ FeatureSIMDReduce, FeatureSIMDBallot, FeatureSIMDMatrix,
+ FeatureThreadgroupAtomics, FeatureDeviceAtomics, FeatureMaxGPRs64,
+ FeatureDynamicCaching, FeatureRayTracing, FeatureMeshShading,
+ FeatureSourceLifetime]
+>;
+
+//===----------------------------------------------------------------------===//
+// Predicates
+//===----------------------------------------------------------------------===//
+
+def HasAGX2Encoding : Predicate<"Subtarget->hasAGX2Encoding()">,
+ AssemblerPredicate<(all_of FeatureAGX2Encoding)>;
+def HasSIMDShuffle : Predicate<"Subtarget->hasSIMDShuffle()">;
+def HasSIMDReduce : Predicate<"Subtarget->hasSIMDReduce()">;
+def HasSIMDBallot : Predicate<"Subtarget->hasSIMDBallot()">;
+def HasSIMDMatrix : Predicate<"Subtarget->hasSIMDMatrix()">;
+def HasFP16Arith : Predicate<"Subtarget->hasFP16Arith()">;
+def HasInt64Add : Predicate<"Subtarget->hasInt64Add()">;
+def HasThreadgroupAtomics : Predicate<"Subtarget->hasThreadgroupAtomics()">;
+def HasDeviceAtomics : Predicate<"Subtarget->hasDeviceAtomics()">;
diff --git a/llvm/lib/Target/AGX/AGXFoldOperands.cpp b/llvm/lib/Target/AGX/AGXFoldOperands.cpp
new file mode 100644
index 0000000000000..e2baf7442091e
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXFoldOperands.cpp
@@ -0,0 +1,112 @@
+//===-- AGXFoldOperands.cpp - Fold uniform registers into operands --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Kernel arguments live in uniform registers, which are never allocated:
+// instruction selection copies them into virtual general purpose registers.
+// Like SIFoldOperands, this pass folds such copies into the operands that
+// can read uniform registers directly (ALU sources and memory bases), which
+// saves both the moves and the general purpose registers.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AGX.h"
+#include "AGXInstrInfo.h"
+#include "AGXRegisterInfo.h"
+#include "AGXSubtarget.h"
+#include "MCTargetDesc/AGXMCTargetDesc.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "agx-fold-operands"
+
+namespace {
+
+class AGXFoldOperands : public MachineFunctionPass {
+public:
+ static char ID;
+ AGXFoldOperands() : MachineFunctionPass(ID) {}
+
+ StringRef getPassName() const override { return "AGX Fold Operands"; }
+ bool runOnMachineFunction(MachineFunction &MF) override;
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesCFG();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+};
+
+} // namespace
+
+char AGXFoldOperands::ID = 0;
+
+INITIALIZE_PASS(AGXFoldOperands, DEBUG_TYPE, "AGX Fold Operands", false,
+ false)
+
+FunctionPass *llvm::createAGXFoldOperandsPass() {
+ return new AGXFoldOperands();
+}
+
+/// Memory base registers encode uniform pairs below u128 only.
+static constexpr unsigned MaxUniformBaseHalf = 0x100;
+
+bool AGXFoldOperands::runOnMachineFunction(MachineFunction &MF) {
+ if (skipFunction(MF.getFunction()))
+ return false;
+ const AGXSubtarget &ST = MF.getSubtarget<AGXSubtarget>();
+ const AGXInstrInfo *TII = ST.getInstrInfo();
+ const AGXRegisterInfo *TRI = ST.getRegisterInfo();
+ MachineRegisterInfo &MRI = MF.getRegInfo();
+
+ bool Changed = false;
+ for (MachineBasicBlock &MBB : MF) {
+ for (MachineInstr &Copy : make_early_inc_range(MBB)) {
+ if (!Copy.isCopy())
+ continue;
+ Register Dst = Copy.getOperand(0).getReg();
+ Register Src = Copy.getOperand(1).getReg();
+ if (!Dst.isVirtual() || !Src.isPhysical() ||
+ Copy.getOperand(1).getSubReg())
+ continue;
+ AGX::RegInfo SrcInfo = AGX::getRegInfo(Src, *TRI);
+ if (!SrcInfo.isUniform())
+ continue;
+
+ for (MachineOperand &Use :
+ make_early_inc_range(MRI.use_nodbg_operands(Dst))) {
+ MachineInstr &UseMI = *Use.getParent();
+ if (UseMI.isCopy() || UseMI.isPHI() || UseMI.isRegSequence() ||
+ UseMI.isInlineAsm())
+ continue;
+ unsigned OpIdx = UseMI.getOperandNo(&Use);
+ if (OpIdx >= UseMI.getDesc().getNumOperands())
+ continue;
+ const TargetRegisterClass *RC = TII->getRegClass(UseMI.getDesc(),
+ OpIdx);
+ MCRegister Folded = Src;
+ if (Use.getSubReg())
+ Folded = TRI->getSubReg(Src, Use.getSubReg());
+ if (!Folded || !RC || !RC->contains(Folded))
+ continue;
+ if (RC == &AGX::BASE64RegClass &&
+ AGX::getRegInfo(Folded, *TRI).HalfIndex >= MaxUniformBaseHalf)
+ continue;
+ Use.setReg(Folded);
+ Use.setSubReg(0);
+ Use.setIsKill(false);
+ Changed = true;
+ }
+
+ if (MRI.use_nodbg_empty(Dst)) {
+ Copy.eraseFromParent();
+ Changed = true;
+ }
+ }
+ }
+ return Changed;
+}
diff --git a/llvm/lib/Target/AGX/AGXFrameLowering.cpp b/llvm/lib/Target/AGX/AGXFrameLowering.cpp
new file mode 100644
index 0000000000000..af5435ff4bcb1
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXFrameLowering.cpp
@@ -0,0 +1,38 @@
+//===-- AGXFrameLowering.cpp - AGX frame lowering -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AGXFrameLowering.h"
+#include "AGXSubtarget.h"
+#include "MCTargetDesc/AGXMCTargetDesc.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/Support/MathExtras.h"
+
+using namespace llvm;
+
+unsigned AGXFrameLowering::getStackAdjustUnits(const MachineFunction &MF) {
+ uint64_t Bytes = MF.getFrameInfo().getStackSize();
+ if (!Bytes)
+ return 0;
+ // Like the vendor compiler, allocate 40 more bytes and round up to 16
+ // bytes, in 32-bit units.
+ return alignTo(divideCeil(Bytes, 4) + 10, 4);
+}
+
+void AGXFrameLowering::emitPrologue(MachineFunction &MF,
+ MachineBasicBlock &MBB) const {
+ unsigned Units = getStackAdjustUnits(MF);
+ if (!Units)
+ return;
+ if (Units > 0x7fff)
+ report_fatal_error("AGX stack frame too large");
+ const AGXInstrInfo *TII = MF.getSubtarget<AGXSubtarget>().getInstrInfo();
+ BuildMI(MBB, MBB.begin(), DebugLoc(), TII->get(AGX::STACK_ADJUST))
+ .addImm(Units);
+}
diff --git a/llvm/lib/Target/AGX/AGXFrameLowering.h b/llvm/lib/Target/AGX/AGXFrameLowering.h
new file mode 100644
index 0000000000000..bf3d199080033
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXFrameLowering.h
@@ -0,0 +1,39 @@
+//===-- AGXFrameLowering.h - Define frame lowering for AGX ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AGX_AGXFRAMELOWERING_H
+#define LLVM_LIB_TARGET_AGX_AGXFRAMELOWERING_H
+
+#include "llvm/CodeGen/TargetFrameLowering.h"
+
+namespace llvm {
+
+class AGXSubtarget;
+
+/// Each thread has a private stack addressed by byte offsets. Objects are
+/// laid out upwards from offset zero; the stack is allocated with
+/// stack_adjust at kernel entry.
+class AGXFrameLowering : public TargetFrameLowering {
+public:
+ explicit AGXFrameLowering(const AGXSubtarget &ST)
+ : TargetFrameLowering(TargetFrameLowering::StackGrowsUp, Align(4), 0,
+ Align(4)) {}
+
+ void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
+ void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override {}
+
+ /// Size of the stack in the 32-bit units of stack_adjust, or zero.
+ static unsigned getStackAdjustUnits(const MachineFunction &MF);
+
+protected:
+ bool hasFPImpl(const MachineFunction &MF) const override { return false; }
+};
+
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AGX_AGXFRAMELOWERING_H
diff --git a/llvm/lib/Target/AGX/AGXISelDAGToDAG.cpp b/llvm/lib/Target/AGX/AGXISelDAGToDAG.cpp
new file mode 100644
index 0000000000000..0ed8d58b88dfa
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXISelDAGToDAG.cpp
@@ -0,0 +1,360 @@
+//===-- AGXISelDAGToDAG.cpp - A DAG to DAG instruction selector for AGX ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Instruction selection for AGX: the TableGen generated matcher plus the
+// complex patterns for ALU sources and memory addresses.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AGX.h"
+#include "AGXISelLowering.h"
+#include "AGXSubtarget.h"
+#include "AGXTargetMachine.h"
+#include "MCTargetDesc/AGXMCTargetDesc.h"
+#include "llvm/CodeGen/SelectionDAGISel.h"
+#include "llvm/IR/IntrinsicsAGX.h"
+#include "llvm/Support/MathExtras.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "agx-isel"
+#define PASS_NAME "AGX DAG->DAG Pattern Instruction Selection"
+
+namespace {
+
+class AGXDAGToDAGISel : public SelectionDAGISel {
+ const AGXSubtarget *Subtarget = nullptr;
+
+public:
+ AGXDAGToDAGISel() = delete;
+ explicit AGXDAGToDAGISel(AGXTargetMachine &TM) : SelectionDAGISel(TM) {}
+
+ bool runOnMachineFunction(MachineFunction &MF) override {
+ Subtarget = &MF.getSubtarget<AGXSubtarget>();
+ return SelectionDAGISel::runOnMachineFunction(MF);
+ }
+
+private:
+#include "AGXGenDAGISel.inc"
+
+ void Select(SDNode *N) override;
+ bool trySelectShift64(SDNode *N);
+
+ SDValue getI32Imm(uint64_t Imm, const SDLoc &DL) {
+ return CurDAG->getTargetConstant(Imm, DL, MVT::i32);
+ }
+
+ // Complex patterns.
+ bool SelectIntSrc(SDValue N, SDValue &Src);
+ bool SelectFPSrc(SDValue N, SDValue &Src, SDValue &Mods);
+ bool SelectDeviceAddr(SDValue Addr, unsigned ElemLog2, SDValue &Base,
+ SDValue &Index, SDValue &Signed, SDValue &Shift);
+ bool SelectDeviceAddr8(SDValue Addr, SDValue &Base, SDValue &Index,
+ SDValue &Signed, SDValue &Shift) {
+ return SelectDeviceAddr(Addr, 0, Base, Index, Signed, Shift);
+ }
+ bool SelectDeviceAddr16(SDValue Addr, SDValue &Base, SDValue &Index,
+ SDValue &Signed, SDValue &Shift) {
+ return SelectDeviceAddr(Addr, 1, Base, Index, Signed, Shift);
+ }
+ bool SelectDeviceAddr32(SDValue Addr, SDValue &Base, SDValue &Index,
+ SDValue &Signed, SDValue &Shift) {
+ return SelectDeviceAddr(Addr, 2, Base, Index, Signed, Shift);
+ }
+ bool SelectLocalAddr(SDValue Addr, SDValue &Base, SDValue &Index);
+ bool SelectLocalAddr8(SDValue Addr, SDValue &Base, SDValue &Index) {
+ return SelectLocalAddr(Addr, Base, Index);
+ }
+ bool SelectLocalAddr16(SDValue Addr, SDValue &Base, SDValue &Index) {
+ return SelectLocalAddr(Addr, Base, Index);
+ }
+ bool SelectLocalAddr32(SDValue Addr, SDValue &Base, SDValue &Index) {
+ return SelectLocalAddr(Addr, Base, Index);
+ }
+ bool SelectStackAddr(SDValue Addr, SDValue &Offset);
+};
+
+class AGXDAGToDAGISelLegacy : public SelectionDAGISelLegacy {
+public:
+ static char ID;
+ explicit AGXDAGToDAGISelLegacy(AGXTargetMachine &TM)
+ : SelectionDAGISelLegacy(ID, std::make_unique<AGXDAGToDAGISel>(TM)) {}
+};
+
+} // namespace
+
+char AGXDAGToDAGISelLegacy::ID = 0;
+
+INITIALIZE_PASS(AGXDAGToDAGISelLegacy, DEBUG_TYPE, PASS_NAME, false, false)
+
+FunctionPass *llvm::createAGXISelDag(AGXTargetMachine &TM) {
+ return new AGXDAGToDAGISelLegacy(TM);
+}
+
+void AGXDAGToDAGISel::Select(SDNode *N) {
+ if (N->isMachineOpcode()) {
+ N->setNodeId(-1);
+ return;
+ }
+ switch (N->getOpcode()) {
+ case ISD::SHL:
+ case ISD::SRL:
+ case ISD::SRA:
+ // Shifts of extended 32-bit values by up to 4 are selected as iadd by
+ // the generated matcher.
+ if (N->getOpcode() == ISD::SHL &&
+ (N->getOperand(0).getOpcode() == ISD::ZERO_EXTEND ||
+ N->getOperand(0).getOpcode() == ISD::SIGN_EXTEND) &&
+ isa<ConstantSDNode>(N->getOperand(1)) &&
+ N->getConstantOperandVal(1) <= 4)
+ break;
+ if (trySelectShift64(N))
+ return;
+ break;
+ default:
+ break;
+ }
+ SelectCode(N);
+}
+
+/// Integer immediates 0-255 are encoded in the instruction.
+bool AGXDAGToDAGISel::SelectIntSrc(SDValue N, SDValue &Src) {
+ if (auto *C = dyn_cast<ConstantSDNode>(N)) {
+ if (C->getZExtValue() <= 0xff) {
+ Src = CurDAG->getTargetConstant(C->getZExtValue(), SDLoc(N),
+ N.getValueType());
+ return true;
+ }
+ }
+ Src = N;
+ return true;
+}
+
+/// Floating point sources fold neg and abs into modifiers, and constants
+/// representable as 8-bit floats into the instruction.
+bool AGXDAGToDAGISel::SelectFPSrc(SDValue N, SDValue &Src, SDValue &Mods) {
+ SDLoc DL(N);
+ unsigned M = 0;
+ while (true) {
+ if (N.getOpcode() == ISD::FNEG) {
+ M ^= AGX::SRC_MOD_NEG;
+ N = N.getOperand(0);
+ } else if (N.getOpcode() == ISD::FABS) {
+ // |-x| = |x|, and an outer negation applies after the absolute value.
+ M |= AGX::SRC_MOD_ABS;
+ N = N.getOperand(0);
+ } else {
+ break;
+ }
+ }
+
+ if (auto *C = dyn_cast<ConstantFPSDNode>(N)) {
+ APFloat V = C->getValueAPF();
+ if (M & AGX::SRC_MOD_ABS)
+ V.clearSign();
+ if (M & AGX::SRC_MOD_NEG)
+ V.changeSign();
+ bool LosesInfo = false;
+ V.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
+ &LosesInfo);
+ if (!LosesInfo) {
+ if (std::optional<unsigned> Enc = AGX::encodeFPImm8(V.convertToDouble())) {
+ Src = CurDAG->getTargetConstant(*Enc, DL, MVT::i32);
+ Mods = getI32Imm(0, DL);
+ return true;
+ }
+ }
+ }
+
+ Src = N;
+ Mods = getI32Imm(M, DL);
+ return true;
+}
+
+/// 64-bit shifts by a constant: shift both halves, combining them with a
+/// funnel shift (extr).
+bool AGXDAGToDAGISel::trySelectShift64(SDNode *N) {
+ auto *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
+ if (!C || N->getValueType(0) != MVT::i64)
+ return false;
+ SDLoc DL(N);
+ unsigned Opc = N->getOpcode();
+ uint64_t S = C->getZExtValue() & 63;
+ SDValue X = N->getOperand(0);
+ SDValue Lo = CurDAG->getTargetExtractSubreg(AGX::sub0, DL, MVT::i32, X);
+ SDValue Hi = CurDAG->getTargetExtractSubreg(AGX::sub1, DL, MVT::i32, X);
+ SDValue Zero = getI32Imm(0, DL);
+
+ // Machine node builders mirroring the 32-bit shift patterns.
+ auto Shl = [&](SDValue V, uint64_t A) {
+ return SDValue(CurDAG->getMachineNode(AGX::BFI_I32, DL, MVT::i32,
+ {Zero, Zero, Zero, V, Zero,
+ getI32Imm(A, DL), Zero}),
+ 0);
+ };
+ auto Lsr = [&](SDValue V, uint64_t A) {
+ return SDValue(CurDAG->getMachineNode(AGX::BFEIL_I32, DL, MVT::i32,
+ {Zero, Zero, Zero, V, Zero,
+ getI32Imm(A, DL), Zero}),
+ 0);
+ };
+ auto Asr = [&](SDValue V, uint64_t A) {
+ return SDValue(CurDAG->getMachineNode(AGX::ASR_I32, DL, MVT::i32,
+ {Zero, V, Zero, getI32Imm(A, DL)}),
+ 0);
+ };
+ // (B:A) >> A, the low 32 bits.
+ auto Extr = [&](SDValue A, SDValue B, uint64_t Amt) {
+ return SDValue(CurDAG->getMachineNode(AGX::EXTR_I32, DL, MVT::i32,
+ {Zero, A, Zero, B, Zero,
+ getI32Imm(Amt, DL), Zero}),
+ 0);
+ };
+ auto Imm0 = [&]() {
+ return SDValue(
+ CurDAG->getMachineNode(AGX::MOV_IMM32, DL, MVT::i32, Zero), 0);
+ };
+
+ SDValue NewLo, NewHi;
+ if (S == 0) {
+ ReplaceNode(N, X.getNode());
+ return true;
+ }
+ switch (Opc) {
+ case ISD::SHL:
+ if (S >= 32) {
+ NewLo = Imm0();
+ NewHi = Shl(Lo, S - 32);
+ } else {
+ NewLo = Shl(Lo, S);
+ NewHi = Extr(Lo, Hi, 32 - S);
+ }
+ break;
+ case ISD::SRL:
+ if (S >= 32) {
+ NewLo = Lsr(Hi, S - 32);
+ NewHi = Imm0();
+ } else {
+ NewLo = Extr(Lo, Hi, S);
+ NewHi = Lsr(Hi, S);
+ }
+ break;
+ case ISD::SRA:
+ if (S >= 32) {
+ NewLo = Asr(Hi, S - 32);
+ NewHi = Asr(Hi, 31);
+ } else {
+ NewLo = Extr(Lo, Hi, S);
+ NewHi = Asr(Hi, S);
+ }
+ break;
+ default:
+ return false;
+ }
+ SDValue Ops[] = {
+ CurDAG->getTargetConstant(AGX::GPR64RegClassID, DL, MVT::i32), NewLo,
+ CurDAG->getTargetConstant(AGX::sub0, DL, MVT::i32), NewHi,
+ CurDAG->getTargetConstant(AGX::sub1, DL, MVT::i32)};
+ ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL,
+ MVT::i64, Ops));
+ return true;
+}
+
+/// Device addresses are base + (index << (log2(element size) + shift)), with
+/// a 32-bit index that is zero or sign extended, or a signed 16-bit
+/// immediate index.
+bool AGXDAGToDAGISel::SelectDeviceAddr(SDValue Addr, unsigned ElemLog2,
+ SDValue &Base, SDValue &Index,
+ SDValue &Signed, SDValue &Shift) {
+ SDLoc DL(Addr);
+ Signed = getI32Imm(0, DL);
+ Shift = getI32Imm(0, DL);
+
+ if (Addr.getOpcode() == ISD::ADD ||
+ (Addr.getOpcode() == ISD::OR && Addr->getFlags().hasDisjoint())) {
+ for (unsigned I = 0; I < 2; ++I) {
+ SDValue B = Addr.getOperand(I);
+ SDValue O = Addr.getOperand(1 - I);
+
+ // Constant byte offset.
+ if (auto *C = dyn_cast<ConstantSDNode>(O)) {
+ int64_t Off = C->getSExtValue();
+ if (Off % (int64_t(1) << ElemLog2) == 0 &&
+ isInt<16>(Off >> ElemLog2)) {
+ Base = B;
+ Index = CurDAG->getTargetConstant(Off >> ElemLog2, DL, MVT::i32);
+ return true;
+ }
+ continue;
+ }
+
+ // Scaled, extended 32-bit index.
+ unsigned Scale = 0;
+ if (O.getOpcode() == ISD::SHL) {
+ auto *C = dyn_cast<ConstantSDNode>(O.getOperand(1));
+ if (!C)
+ continue;
+ Scale = C->getZExtValue();
+ O = O.getOperand(0);
+ } else if (O.getOpcode() == ISD::MUL) {
+ auto *C = dyn_cast<ConstantSDNode>(O.getOperand(1));
+ if (!C || !isPowerOf2_64(C->getZExtValue()))
+ continue;
+ Scale = Log2_64(C->getZExtValue());
+ O = O.getOperand(0);
+ }
+ if (Scale < ElemLog2 || Scale > ElemLog2 + 3)
+ continue;
+ bool IsSExt = O.getOpcode() == ISD::SIGN_EXTEND;
+ if (!IsSExt && O.getOpcode() != ISD::ZERO_EXTEND)
+ continue;
+ if (O.getOperand(0).getValueType() != MVT::i32)
+ continue;
+ Base = B;
+ Index = O.getOperand(0);
+ // The index signedness field is 1 for zero extension.
+ Signed = getI32Imm(!IsSExt, DL);
+ Shift = getI32Imm(Scale - ElemLog2, DL);
+ return true;
+ }
+ }
+
+ Base = Addr;
+ Index = CurDAG->getTargetConstant(0, DL, MVT::i32);
+ return true;
+}
+
+/// Threadgroup addresses are 16-bit byte offsets in the base register.
+bool AGXDAGToDAGISel::SelectLocalAddr(SDValue Addr, SDValue &Base,
+ SDValue &Index) {
+ SDLoc DL(Addr);
+ if (isNullConstant(Addr))
+ Base = CurDAG->getTargetConstant(0, DL, MVT::i16);
+ else
+ Base = CurDAG->getTargetExtractSubreg(AGX::lo16, DL, MVT::i16, Addr);
+ Index = CurDAG->getTargetConstant(0, DL, MVT::i16);
+ return true;
+}
+
+/// Stack addresses are byte offsets: a frame index (replaced by its offset
+/// after frame lowering), a 16-bit immediate or a register.
+bool AGXDAGToDAGISel::SelectStackAddr(SDValue Addr, SDValue &Offset) {
+ SDLoc DL(Addr);
+ if (auto *FI = dyn_cast<FrameIndexSDNode>(Addr)) {
+ Offset = CurDAG->getTargetFrameIndex(FI->getIndex(), MVT::i32);
+ return true;
+ }
+ if (auto *C = dyn_cast<ConstantSDNode>(Addr)) {
+ if (isInt<16>(C->getSExtValue())) {
+ Offset = CurDAG->getTargetConstant(C->getSExtValue(), DL, MVT::i32);
+ return true;
+ }
+ }
+ Offset = Addr;
+ return true;
+}
diff --git a/llvm/lib/Target/AGX/AGXISelLowering.cpp b/llvm/lib/Target/AGX/AGXISelLowering.cpp
new file mode 100644
index 0000000000000..e307cf95c204f
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXISelLowering.cpp
@@ -0,0 +1,813 @@
+//===-- AGXISelLowering.cpp - AGX DAG lowering implementation -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Lowering of LLVM IR to the AGX selection DAG.
+//
+// Legal types are i16, i32, i64, f16 and f32. There is no f64 support and no
+// vector types: vectors are scalarized. Booleans are 0/1 in 32-bit registers.
+// 64-bit integers are native for add, sub, multiply and bitwise operations;
+// shifts, compares and bit counts are split into 32-bit halves here.
+//
+// Kernel arguments arrive in uniform registers, see LowerFormalArguments and
+// llvm/docs/AGXUsage.rst. Calls are not supported: everything is inlined.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AGXISelLowering.h"
+#include "AGXMachineFunctionInfo.h"
+#include "AGXRegisterInfo.h"
+#include "AGXSubtarget.h"
+#include "AGXTargetMachine.h"
+#include "MCTargetDesc/AGXMCTargetDesc.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/SelectionDAG.h"
+#include "llvm/IR/DiagnosticInfo.h"
+#include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicsAGX.h"
+#include "llvm/Support/MathExtras.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "agx-lower"
+
+static void diagnose(SelectionDAG &DAG, const SDLoc &DL, const Twine &Msg) {
+ const Function &F = DAG.getMachineFunction().getFunction();
+ DAG.getContext()->diagnose(
+ DiagnosticInfoUnsupported(F, Msg, DL.getDebugLoc()));
+}
+
+AGXTargetLowering::AGXTargetLowering(const TargetMachine &TM,
+ const AGXSubtarget &ST)
+ : TargetLowering(TM, ST), ST(ST) {
+ addRegisterClass(MVT::i16, &AGX::GPR16RegClass);
+ addRegisterClass(MVT::f16, &AGX::GPR16RegClass);
+ addRegisterClass(MVT::i32, &AGX::GPR32RegClass);
+ addRegisterClass(MVT::f32, &AGX::GPR32RegClass);
+ addRegisterClass(MVT::i64, &AGX::GPR64RegClass);
+ computeRegisterProperties(ST.getRegisterInfo());
+
+ setBooleanContents(ZeroOrOneBooleanContent);
+ setSchedulingPreference(Sched::RegPressure);
+ setMinFunctionAlignment(Align(2));
+ setPrefFunctionAlignment(Align(2));
+ setJumpIsExpensive(true);
+ setMaxAtomicSizeInBitsSupported(32);
+ setMinCmpXchgSizeInBits(32);
+
+ // There are no calls and no scalar branches: all control flow is
+ // structured by AGXAnnotateControlFlow.
+ setOperationAction(ISD::BR_CC, {MVT::i16, MVT::i32, MVT::i64, MVT::f16,
+ MVT::f32}, Expand);
+ setOperationAction(ISD::BRCOND, MVT::Other, Custom);
+ setOperationAction(ISD::BR_JT, MVT::Other, Expand);
+ setOperationAction(ISD::BRIND, MVT::Other, Expand);
+ setOperationAction({ISD::STACKSAVE, ISD::STACKRESTORE,
+ ISD::DYNAMIC_STACKALLOC},
+ {MVT::i32, MVT::Other}, Expand);
+ setOperationAction(ISD::GlobalAddress, {MVT::i32, MVT::i64}, Custom);
+ setOperationAction({ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_VOID},
+ MVT::Other, Custom);
+ // llvm.agx.simd.ballot takes an i1, which is promoted by custom lowering.
+ setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i1, Custom);
+ setTargetDAGCombine(ISD::BRCOND);
+
+ //===--------------------------------------------------------------------===//
+ // Integer operations
+ //===--------------------------------------------------------------------===//
+
+ for (MVT VT : {MVT::i16, MVT::i32, MVT::i64}) {
+ setOperationAction({ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM,
+ ISD::SDIVREM, ISD::UDIVREM, ISD::SMUL_LOHI,
+ ISD::UMUL_LOHI, ISD::BSWAP, ISD::SELECT_CC,
+ ISD::SHL_PARTS, ISD::SRL_PARTS, ISD::SRA_PARTS},
+ VT, Expand);
+ setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, VT,
+ VT == MVT::i64 ? Expand : Legal);
+ }
+ setOperationAction({ISD::MULHU, ISD::MULHS}, MVT::i32, Legal);
+ setOperationAction({ISD::MULHU, ISD::MULHS}, {MVT::i16, MVT::i64}, Expand);
+ setOperationAction({ISD::ROTL, ISD::ROTR, ISD::FSHL}, {MVT::i16, MVT::i64},
+ Expand);
+ setOperationAction(ISD::ROTL, MVT::i32, Expand);
+ setOperationAction(ISD::FSHL, MVT::i32, Expand);
+ setOperationAction(ISD::FSHR, MVT::i32, Legal);
+ setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
+ setOperationAction(ISD::BITREVERSE, MVT::i16, Promote);
+ setOperationAction({ISD::CTPOP, ISD::CTLZ, ISD::CTTZ}, MVT::i16, Promote);
+ setOperationAction({ISD::CTLZ, ISD::CTTZ}, MVT::i64, Custom);
+ setOperationAction({ISD::SHL, ISD::SRL, ISD::SRA}, MVT::i64, Custom);
+ setOperationAction(ISD::SETCC, MVT::i64, Custom);
+ setOperationAction({ISD::UADDSAT, ISD::USUBSAT, ISD::SADDSAT,
+ ISD::SSUBSAT},
+ MVT::i32, Legal);
+
+ //===--------------------------------------------------------------------===//
+ // Floating point operations
+ //===--------------------------------------------------------------------===//
+
+ for (MVT VT : {MVT::f16, MVT::f32}) {
+ setOperationAction({ISD::FFLOOR, ISD::FCEIL, ISD::FTRUNC, ISD::FRINT,
+ ISD::FNEARBYINT, ISD::FROUNDEVEN, ISD::FMINNUM,
+ ISD::FMAXNUM, ISD::FEXP2, ISD::FLOG2},
+ VT, Legal);
+ setOperationAction({ISD::FSUB, ISD::SELECT_CC, ISD::FCOPYSIGN,
+ ISD::FMINIMUM, ISD::FMAXIMUM, ISD::FMINNUM_IEEE,
+ ISD::FMAXNUM_IEEE, ISD::FSINCOS, ISD::FROUND},
+ VT, Expand);
+ // Conditions without a direct hardware condition.
+ setCondCodeAction({ISD::SETONE, ISD::SETUEQ, ISD::SETO, ISD::SETUO}, VT,
+ Expand);
+ }
+ setOperationAction({ISD::FDIV, ISD::FREM, ISD::FSIN, ISD::FCOS, ISD::FEXP,
+ ISD::FEXP10, ISD::FLOG, ISD::FLOG10, ISD::FPOW},
+ MVT::f32, Custom);
+ setOperationAction(ISD::FDIV, MVT::f16, Custom);
+ setOperationAction({ISD::FREM, ISD::FSIN, ISD::FCOS, ISD::FEXP, ISD::FEXP10,
+ ISD::FLOG, ISD::FLOG10, ISD::FPOW},
+ MVT::f16, Promote);
+ setOperationAction({ISD::FP16_TO_FP, ISD::FP_TO_FP16}, MVT::f32, Expand);
+
+ // 64-bit conversions are expanded with 32-bit conversions.
+ setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::SINT_TO_FP,
+ ISD::UINT_TO_FP},
+ MVT::i64, Custom);
+
+ //===--------------------------------------------------------------------===//
+ // Memory
+ //===--------------------------------------------------------------------===//
+
+ for (MVT VT : {MVT::i16, MVT::i32, MVT::i64}) {
+ setLoadExtAction({ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD}, VT,
+ MVT::i1, Promote);
+ setLoadExtAction({ISD::SEXTLOAD}, VT, {MVT::i8, MVT::i16}, Expand);
+ }
+ setLoadExtAction({ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD}, MVT::i64,
+ {MVT::i8, MVT::i16, MVT::i32}, Expand);
+ for (MVT MemVT : {MVT::i8, MVT::i16, MVT::i32})
+ setTruncStoreAction(MVT::i64, MemVT, Expand);
+ setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
+ setTruncStoreAction(MVT::f32, MVT::f16, Expand);
+
+ setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i32, Expand);
+}
+
+EVT AGXTargetLowering::getSetCCResultType(const DataLayout &DL,
+ LLVMContext &Context,
+ EVT VT) const {
+ if (VT.isVector())
+ return EVT::getVectorVT(Context, MVT::i32, VT.getVectorElementCount());
+ return MVT::i32;
+}
+
+MVT AGXTargetLowering::getScalarShiftAmountTy(const DataLayout &DL,
+ EVT VT) const {
+ return VT == MVT::i16 ? MVT::i16 : MVT::i32;
+}
+
+bool AGXTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
+ EVT VT) const {
+ VT = VT.getScalarType();
+ return VT == MVT::f32 || VT == MVT::f16;
+}
+
+bool AGXTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
+ // Truncation reads a subregister.
+ return SrcVT.isScalarInteger() && DstVT.isScalarInteger() &&
+ SrcVT.getSizeInBits() > DstVT.getSizeInBits() &&
+ DstVT.getSizeInBits() >= 16;
+}
+
+TargetLowering::AtomicExpansionKind
+AGXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
+ if (RMW->getType()->isIntegerTy(32)) {
+ switch (RMW->getOperation()) {
+ case AtomicRMWInst::Add:
+ case AtomicRMWInst::Sub:
+ case AtomicRMWInst::Xchg:
+ case AtomicRMWInst::Min:
+ case AtomicRMWInst::Max:
+ case AtomicRMWInst::UMin:
+ case AtomicRMWInst::UMax:
+ case AtomicRMWInst::And:
+ case AtomicRMWInst::Or:
+ case AtomicRMWInst::Xor:
+ return AtomicExpansionKind::None;
+ default:
+ break;
+ }
+ }
+ return AtomicExpansionKind::CmpXChg;
+}
+
+TargetLowering::AtomicExpansionKind
+AGXTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
+ // There are no atomic loads: use a compare and swap that never succeeds in
+ // changing memory.
+ return AtomicExpansionKind::CmpXChg;
+}
+
+TargetLowering::AtomicExpansionKind
+AGXTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
+ // Atomic stores become atomic exchanges.
+ return AtomicExpansionKind::Expand;
+}
+
+//===----------------------------------------------------------------------===//
+// Calling convention
+//===----------------------------------------------------------------------===//
+
+SDValue AGXTargetLowering::LowerFormalArguments(
+ SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
+ const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
+ SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
+ MachineFunction &MF = DAG.getMachineFunction();
+ auto *MFI = MF.getInfo<AGXMachineFunctionInfo>();
+ const MCRegisterInfo &MRI = *ST.getRegisterInfo();
+
+ if (IsVarArg)
+ diagnose(DAG, DL, "variadic AGX kernels are not supported");
+
+ // Arguments are packed into consecutive uniform registers in 16-bit units,
+ // each aligned to its size (64-bit values to 32 bits).
+ unsigned Half = 0;
+ bool Overflow = false;
+ auto Allocate = [&](MVT VT) -> MCRegister {
+ unsigned Bits = VT.getSizeInBits();
+ AGX::RegKind Kind = Bits == 16 ? AGX::RegKind::UGPR16
+ : Bits == 32 ? AGX::RegKind::UGPR32
+ : AGX::RegKind::UGPR64;
+ Half = alignTo(Half, Bits == 16 ? 1 : 2);
+ MCRegister Reg = AGX::getRegister(Kind, Half, MRI);
+ Half += Bits / 16;
+ if (!Reg)
+ Overflow = true;
+ return Reg;
+ };
+
+ for (const ISD::InputArg &In : Ins) {
+ MVT VT = In.VT;
+ if (In.Flags.isByVal() || !(VT == MVT::i16 || VT == MVT::f16 ||
+ VT == MVT::i32 || VT == MVT::f32 ||
+ VT == MVT::i64)) {
+ diagnose(DAG, DL, "unsupported AGX kernel argument type");
+ InVals.push_back(DAG.getUNDEF(VT));
+ continue;
+ }
+ MCRegister PReg = Allocate(VT);
+ if (!PReg) {
+ InVals.push_back(DAG.getUNDEF(VT));
+ continue;
+ }
+ Register VReg = MF.addLiveIn(PReg, getRegClassFor(VT));
+ InVals.push_back(DAG.getCopyFromReg(Chain, DL, VReg, VT));
+ }
+
+ // Implicit arguments: threadgroups_per_grid.xyz.
+ Half = alignTo(Half, 2);
+ MFI->setImplicitArgHalfIndex(Half);
+ for (unsigned Dim = 0; Dim < 3; ++Dim) {
+ MCRegister PReg = Allocate(MVT::i32);
+ if (PReg)
+ MFI->setImplicitArgVReg(Dim,
+ MF.addLiveIn(PReg, &AGX::GPR32RegClass));
+ }
+ MFI->setUniformHalfCount(Half);
+
+ if (Overflow)
+ diagnose(DAG, DL, "AGX kernel arguments exceed the uniform registers");
+ return Chain;
+}
+
+SDValue
+AGXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
+ bool IsVarArg,
+ const SmallVectorImpl<ISD::OutputArg> &Outs,
+ const SmallVectorImpl<SDValue> &OutVals,
+ const SDLoc &DL, SelectionDAG &DAG) const {
+ if (!Outs.empty())
+ diagnose(DAG, DL, "AGX kernels cannot return values");
+ return DAG.getNode(AGXISD::STOP, DL, MVT::Other, Chain);
+}
+
+SDValue AGXTargetLowering::LowerCall(CallLoweringInfo &CLI,
+ SmallVectorImpl<SDValue> &InVals) const {
+ SelectionDAG &DAG = CLI.DAG;
+ StringRef Name = "<indirect>";
+ if (auto *G = dyn_cast<GlobalAddressSDNode>(CLI.Callee))
+ Name = G->getGlobal()->getName();
+ else if (auto *E = dyn_cast<ExternalSymbolSDNode>(CLI.Callee))
+ Name = E->getSymbol();
+ diagnose(DAG, CLI.DL, "unsupported call to '" + Name + "' (AGX has no calls)");
+ for (const ISD::InputArg &In : CLI.Ins)
+ InVals.push_back(DAG.getUNDEF(In.VT));
+ return CLI.Chain;
+}
+
+//===----------------------------------------------------------------------===//
+// Custom lowering
+//===----------------------------------------------------------------------===//
+
+/// Low and high halves of a 64-bit value.
+static std::pair<SDValue, SDValue> split64(SelectionDAG &DAG, const SDLoc &DL,
+ SDValue V) {
+ return {DAG.getTargetExtractSubreg(AGX::sub0, DL, MVT::i32, V),
+ DAG.getTargetExtractSubreg(AGX::sub1, DL, MVT::i32, V)};
+}
+
+static SDValue join64(SelectionDAG &DAG, const SDLoc &DL, SDValue Lo,
+ SDValue Hi) {
+ SDValue Ops[] = {
+ DAG.getTargetConstant(AGX::GPR64RegClassID, DL, MVT::i32), Lo,
+ DAG.getTargetConstant(AGX::sub0, DL, MVT::i32), Hi,
+ DAG.getTargetConstant(AGX::sub1, DL, MVT::i32)};
+ return SDValue(
+ DAG.getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::i64, Ops), 0);
+}
+
+SDValue AGXTargetLowering::LowerOperation(SDValue Op,
+ SelectionDAG &DAG) const {
+ switch (Op.getOpcode()) {
+ case ISD::GlobalAddress:
+ return lowerGlobalAddress(Op, DAG);
+ case ISD::INTRINSIC_WO_CHAIN:
+ return lowerINTRINSIC_WO_CHAIN(Op, DAG);
+ case ISD::INTRINSIC_VOID:
+ return lowerINTRINSIC_VOID(Op, DAG);
+ case ISD::BRCOND:
+ return lowerBRCOND(Op, DAG);
+ case ISD::SETCC:
+ return lowerSETCC64(Op, DAG);
+ case ISD::SHL:
+ case ISD::SRL:
+ case ISD::SRA:
+ return lowerShift64(Op, DAG);
+ case ISD::CTLZ:
+ case ISD::CTTZ:
+ return lowerCTLZ_CTTZ64(Op, DAG);
+ case ISD::FDIV:
+ return lowerFDIV(Op, DAG);
+ case ISD::FREM:
+ return lowerFREM(Op, DAG);
+ case ISD::FSIN:
+ case ISD::FCOS:
+ return lowerTrig(Op, DAG);
+ case ISD::FEXP:
+ case ISD::FEXP10:
+ case ISD::FLOG:
+ case ISD::FLOG10:
+ case ISD::FPOW:
+ return lowerExpLog(Op, DAG);
+ case ISD::SINT_TO_FP:
+ case ISD::UINT_TO_FP:
+ return lowerINT_TO_FP64(Op, DAG);
+ case ISD::FP_TO_SINT:
+ case ISD::FP_TO_UINT:
+ return lowerFP_TO_INT64(Op, DAG);
+ default:
+ llvm_unreachable("unexpected custom lowering");
+ }
+}
+
+void AGXTargetLowering::ReplaceNodeResults(SDNode *N,
+ SmallVectorImpl<SDValue> &Results,
+ SelectionDAG &DAG) const {
+ // The custom operations do not have illegal result types.
+}
+
+SDValue AGXTargetLowering::lowerGlobalAddress(SDValue Op,
+ SelectionDAG &DAG) const {
+ SDLoc DL(Op);
+ auto *GA = cast<GlobalAddressSDNode>(Op);
+ const auto *GV = dyn_cast<GlobalVariable>(GA->getGlobal());
+ if (!GV || GA->getAddressSpace() != AGX::THREADGROUP_ADDRESS) {
+ diagnose(DAG, DL,
+ "unsupported global '" + GA->getGlobal()->getName() +
+ "' (only threadgroup variables are supported)");
+ return DAG.getUNDEF(Op.getValueType());
+ }
+ if (GV->hasInitializer() && !isa<UndefValue>(GV->getInitializer()))
+ diagnose(DAG, DL, "threadgroup variables cannot have initializers");
+
+ // Threadgroup variables are allocated statically per kernel; their
+ // address is a constant offset into threadgroup memory.
+ MachineFunction &MF = DAG.getMachineFunction();
+ auto *MFI = MF.getInfo<AGXMachineFunctionInfo>();
+ unsigned Offset =
+ MFI->allocateThreadgroupVariable(DAG.getDataLayout(), *GV);
+ return DAG.getConstant(Offset + GA->getOffset(), DL, Op.getValueType());
+}
+
+SDValue AGXTargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
+ SelectionDAG &DAG) const {
+ SDLoc DL(Op);
+ unsigned IID = Op.getConstantOperandVal(0);
+ switch (IID) {
+ case Intrinsic::agx_threadgroups_per_grid_x:
+ case Intrinsic::agx_threadgroups_per_grid_y:
+ case Intrinsic::agx_threadgroups_per_grid_z: {
+ auto *MFI = DAG.getMachineFunction().getInfo<AGXMachineFunctionInfo>();
+ Register VReg =
+ MFI->getImplicitArgVReg(IID - Intrinsic::agx_threadgroups_per_grid_x);
+ if (!VReg)
+ return DAG.getUNDEF(MVT::i32);
+ return DAG.getCopyFromReg(DAG.getEntryNode(), DL, VReg, MVT::i32);
+ }
+ case Intrinsic::agx_simd_ballot:
+ case Intrinsic::agx_quad_ballot: {
+ SDValue Cond = DAG.getZExtOrTrunc(Op.getOperand(1), DL, MVT::i32);
+ return DAG.getNode(IID == Intrinsic::agx_simd_ballot
+ ? AGXISD::BALLOT
+ : AGXISD::QUAD_BALLOT,
+ DL, MVT::i32, Cond);
+ }
+ default:
+ return SDValue();
+ }
+}
+
+SDValue AGXTargetLowering::lowerINTRINSIC_VOID(SDValue Op,
+ SelectionDAG &DAG) const {
+ SDLoc DL(Op);
+ switch (Op.getConstantOperandVal(1)) {
+ case Intrinsic::agx_end_cf:
+ return DAG.getNode(AGXISD::END_CF, DL, MVT::Other, Op.getOperand(0));
+ case Intrinsic::agx_loop_begin:
+ return DAG.getNode(AGXISD::LOOP_BEGIN, DL, MVT::Other, Op.getOperand(0));
+ default:
+ return SDValue();
+ }
+}
+
+SDValue AGXTargetLowering::lowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
+ // Every conditional branch is rewritten into execution mask operations by
+ // performBRCONDCombine; anything else is unstructured control flow.
+ diagnose(DAG, SDLoc(Op),
+ "unstructured conditional branch (AGXAnnotateControlFlow did not "
+ "run)");
+ return Op.getOperand(0);
+}
+
+/// Rewrite
+///
+/// brcond (llvm.agx.if/else/loop ...), %a
+/// br %b
+///
+/// to the target node, branching to the block executed when no thread
+/// remains active, and an unconditional branch to the other block. This is
+/// done before type legalization so that the i1 results of the control flow
+/// intrinsics never need to be legalized.
+SDValue AGXTargetLowering::performBRCONDCombine(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ SelectionDAG &DAG = DCI.DAG;
+ SDLoc DL(N);
+ SDValue Cond = N->getOperand(1);
+ SDValue Target = N->getOperand(2);
+
+ bool Negated = false;
+ if (Cond.getOpcode() == ISD::XOR && isOneConstant(Cond.getOperand(1))) {
+ Negated = true;
+ Cond = Cond.getOperand(0);
+ } else if (Cond.getOpcode() == ISD::SETCC &&
+ isa<ConstantSDNode>(Cond.getOperand(1))) {
+ ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
+ bool IsOne = isOneConstant(Cond.getOperand(1));
+ bool IsZero = isNullConstant(Cond.getOperand(1));
+ if ((CC == ISD::SETEQ && IsZero) || (CC == ISD::SETNE && IsOne))
+ Negated = true;
+ else if (!((CC == ISD::SETNE && IsZero) || (CC == ISD::SETEQ && IsOne)))
+ return SDValue();
+ Cond = Cond.getOperand(0);
+ }
+
+ if (Cond.getOpcode() != ISD::INTRINSIC_W_CHAIN)
+ return SDValue();
+ unsigned IID = Cond.getConstantOperandVal(1);
+ unsigned Opc;
+ switch (IID) {
+ case Intrinsic::agx_if:
+ Opc = AGXISD::IF;
+ break;
+ case Intrinsic::agx_else:
+ Opc = AGXISD::ELSE;
+ break;
+ case Intrinsic::agx_loop:
+ Opc = AGXISD::LOOP;
+ break;
+ default:
+ return SDValue();
+ }
+ SDNode *Intr = Cond.getNode();
+
+ // The block to skip to. Without negation, it is the destination of the
+ // unconditional branch that follows, which is redirected to the original
+ // destination.
+ SDNode *BR = nullptr;
+ if (!Negated) {
+ for (SDNode *U : N->users())
+ if (U->getOpcode() == ISD::BR)
+ BR = U;
+ if (!BR)
+ return SDValue();
+ Target = BR->getOperand(1);
+ }
+
+ SmallVector<SDValue, 3> Ops = {N->getOperand(0)};
+ if (Opc != AGXISD::ELSE)
+ Ops.push_back(DAG.getZExtOrTrunc(Intr->getOperand(2), DL, MVT::i32));
+ Ops.push_back(Target);
+ SDValue Result = DAG.getNode(Opc, DL, MVT::Other, Ops);
+
+ if (BR) {
+ SDValue NewBR = DAG.getNode(ISD::BR, SDLoc(BR), MVT::Other,
+ BR->getOperand(0), N->getOperand(2));
+ DAG.ReplaceAllUsesWith(BR, NewBR.getNode());
+ }
+
+ // Remove the intrinsic from the chain.
+ DAG.ReplaceAllUsesOfValueWith(SDValue(Intr, Intr->getNumValues() - 1),
+ Intr->getOperand(0));
+ return Result;
+}
+
+SDValue AGXTargetLowering::PerformDAGCombine(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ switch (N->getOpcode()) {
+ case ISD::BRCOND:
+ return performBRCONDCombine(N, DCI);
+ default:
+ return SDValue();
+ }
+}
+
+static ISD::CondCode getUnsignedCondCode(ISD::CondCode CC) {
+ switch (CC) {
+ case ISD::SETLT:
+ return ISD::SETULT;
+ case ISD::SETLE:
+ return ISD::SETULE;
+ case ISD::SETGT:
+ return ISD::SETUGT;
+ case ISD::SETGE:
+ return ISD::SETUGE;
+ default:
+ return CC;
+ }
+}
+
+SDValue AGXTargetLowering::lowerSETCC64(SDValue Op, SelectionDAG &DAG) const {
+ SDLoc DL(Op);
+ EVT VT = Op.getValueType();
+ ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
+ auto [ALo, AHi] = split64(DAG, DL, Op.getOperand(0));
+ auto [BLo, BHi] = split64(DAG, DL, Op.getOperand(1));
+
+ if (CC == ISD::SETEQ || CC == ISD::SETNE) {
+ SDValue Lo = DAG.getSetCC(DL, VT, ALo, BLo, CC);
+ SDValue Hi = DAG.getSetCC(DL, VT, AHi, BHi, CC);
+ return DAG.getNode(CC == ISD::SETEQ ? ISD::AND : ISD::OR, DL, VT, Lo, Hi);
+ }
+
+ // The high halves decide unless they are equal; the low halves always
+ // compare unsigned.
+ SDValue HiEq = DAG.getSetCC(DL, VT, AHi, BHi, ISD::SETEQ);
+ SDValue HiCmp = DAG.getSetCC(DL, VT, AHi, BHi, CC);
+ SDValue LoCmp = DAG.getSetCC(DL, VT, ALo, BLo, getUnsignedCondCode(CC));
+ return DAG.getSelect(DL, VT, HiEq, LoCmp, HiCmp);
+}
+
+SDValue AGXTargetLowering::lowerShift64(SDValue Op, SelectionDAG &DAG) const {
+ SDLoc DL(Op);
+ unsigned Opc = Op.getOpcode();
+ auto [Lo, Hi] = split64(DAG, DL, Op.getOperand(0));
+ SDValue Amt = DAG.getZExtOrTrunc(Op.getOperand(1), DL, MVT::i32);
+ SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
+ auto C32 = [&](uint64_t V) { return DAG.getConstant(V, DL, MVT::i32); };
+
+ // Shifts by a constant stay legal for address mode matching; they are
+ // expanded during instruction selection.
+ if (isa<ConstantSDNode>(Op.getOperand(1)))
+ return Op;
+
+ // Variable shifts: shift each half by the amount modulo 32, then select
+ // on bit 5 of the amount.
+ SDValue A5 = DAG.getNode(ISD::AND, DL, MVT::i32, Amt, C32(31));
+ SDValue Big = DAG.getSetCC(
+ DL, MVT::i32, DAG.getNode(ISD::AND, DL, MVT::i32, Amt, C32(32)), Zero,
+ ISD::SETNE);
+ SDValue NewLo, NewHi;
+ if (Opc == ISD::SHL) {
+ // hi = (hi << s) | ((lo >> 1) >> (31 - s)), avoiding a shift by 32.
+ SDValue LoS = DAG.getNode(ISD::SHL, DL, MVT::i32, Lo, A5);
+ SDValue Carry = DAG.getNode(
+ ISD::SRL, DL, MVT::i32,
+ DAG.getNode(ISD::SRL, DL, MVT::i32, Lo, C32(1)),
+ DAG.getNode(ISD::XOR, DL, MVT::i32, A5, C32(31)));
+ SDValue HiS = DAG.getNode(ISD::OR, DL, MVT::i32,
+ DAG.getNode(ISD::SHL, DL, MVT::i32, Hi, A5),
+ Carry);
+ NewLo = DAG.getSelect(DL, MVT::i32, Big, Zero, LoS);
+ NewHi = DAG.getSelect(DL, MVT::i32, Big, LoS, HiS);
+ } else {
+ SDValue HiS = DAG.getNode(Opc, DL, MVT::i32, Hi, A5);
+ SDValue LoS = DAG.getNode(ISD::FSHR, DL, MVT::i32, Hi, Lo, A5);
+ SDValue Fill = Opc == ISD::SRA
+ ? DAG.getNode(ISD::SRA, DL, MVT::i32, Hi, C32(31))
+ : Zero;
+ NewLo = DAG.getSelect(DL, MVT::i32, Big, HiS, LoS);
+ NewHi = DAG.getSelect(DL, MVT::i32, Big, Fill, HiS);
+ }
+ return join64(DAG, DL, NewLo, NewHi);
+}
+
+SDValue AGXTargetLowering::lowerCTLZ_CTTZ64(SDValue Op,
+ SelectionDAG &DAG) const {
+ SDLoc DL(Op);
+ bool IsCTLZ = Op.getOpcode() == ISD::CTLZ;
+ auto [Lo, Hi] = split64(DAG, DL, Op.getOperand(0));
+ // Count in the half that decides; fall back to the other half plus 32.
+ SDValue First = IsCTLZ ? Hi : Lo;
+ SDValue Second = IsCTLZ ? Lo : Hi;
+ unsigned Opc = Op.getOpcode();
+ SDValue FirstZero =
+ DAG.getSetCC(DL, MVT::i32, First, DAG.getConstant(0, DL, MVT::i32),
+ ISD::SETEQ);
+ SDValue R = DAG.getSelect(
+ DL, MVT::i32, FirstZero,
+ DAG.getNode(ISD::ADD, DL, MVT::i32,
+ DAG.getNode(Opc, DL, MVT::i32, Second),
+ DAG.getConstant(32, DL, MVT::i32)),
+ DAG.getNode(Opc, DL, MVT::i32, First));
+ return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, R);
+}
+
+static SDValue getAGXIntrinsic(SelectionDAG &DAG, const SDLoc &DL,
+ Intrinsic::ID IID, EVT VT, SDValue Arg) {
+ return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
+ DAG.getTargetConstant(IID, DL, MVT::i32), Arg);
+}
+
+SDValue AGXTargetLowering::lowerFDIV(SDValue Op, SelectionDAG &DAG) const {
+ SDLoc DL(Op);
+ EVT VT = Op.getValueType();
+ SDNodeFlags Flags = Op->getFlags();
+ SDValue A = Op.getOperand(0);
+ SDValue B = Op.getOperand(1);
+ SDValue R = getAGXIntrinsic(DAG, DL, Intrinsic::agx_rcp, VT, B);
+
+ auto *CA = dyn_cast<ConstantFPSDNode>(A);
+ if (CA && CA->isExactlyValue(1.0))
+ return R;
+ SDValue Q = DAG.getNode(ISD::FMUL, DL, VT, A, R, Flags);
+ if (Flags.hasAllowReciprocal() || Flags.hasApproximateFuncs() ||
+ VT == MVT::f16)
+ return Q;
+
+ // One Newton-Raphson step on the quotient: q += (a - b * q) * rcp(b).
+ SDValue E = DAG.getNode(ISD::FMA, DL, VT, DAG.getNode(ISD::FNEG, DL, VT, B),
+ Q, A);
+ return DAG.getNode(ISD::FMA, DL, VT, E, R, Q);
+}
+
+SDValue AGXTargetLowering::lowerFREM(SDValue Op, SelectionDAG &DAG) const {
+ // a - trunc(a / b) * b
+ SDLoc DL(Op);
+ EVT VT = Op.getValueType();
+ SDValue A = Op.getOperand(0);
+ SDValue B = Op.getOperand(1);
+ SDValue Q = DAG.getNode(ISD::FDIV, DL, VT, A, B, Op->getFlags());
+ SDValue T = DAG.getNode(ISD::FTRUNC, DL, VT, Q);
+ return DAG.getNode(ISD::FMA, DL, VT, DAG.getNode(ISD::FNEG, DL, VT, T), B,
+ A);
+}
+
+SDValue AGXTargetLowering::lowerTrig(SDValue Op, SelectionDAG &DAG) const {
+ // The hardware evaluates sin(x * pi / 2) for x in [0, 4): reduce the
+ // argument to turns, take the fraction and scale to quadrants. cos(x) is
+ // sin(x + pi / 2), a quarter turn ahead.
+ SDLoc DL(Op);
+ EVT VT = Op.getValueType();
+ SDValue Turns = DAG.getNode(ISD::FMUL, DL, VT, Op.getOperand(0),
+ DAG.getConstantFP(0.5 * M_1_PI, DL, VT));
+ if (Op.getOpcode() == ISD::FCOS)
+ Turns = DAG.getNode(ISD::FADD, DL, VT, Turns,
+ DAG.getConstantFP(0.25, DL, VT));
+ SDValue Frac = DAG.getNode(ISD::FSUB, DL, VT, Turns,
+ DAG.getNode(ISD::FFLOOR, DL, VT, Turns));
+ SDValue Quadrants =
+ DAG.getNode(ISD::FMUL, DL, VT, Frac, DAG.getConstantFP(4.0, DL, VT));
+ return DAG.getNode(AGXISD::SIN_QUADRANTS, DL, VT, Quadrants);
+}
+
+SDValue AGXTargetLowering::lowerExpLog(SDValue Op, SelectionDAG &DAG) const {
+ SDLoc DL(Op);
+ EVT VT = Op.getValueType();
+ SDValue X = Op.getOperand(0);
+ auto K = [&](double V) { return DAG.getConstantFP(V, DL, VT); };
+ switch (Op.getOpcode()) {
+ case ISD::FEXP:
+ return DAG.getNode(ISD::FEXP2, DL, VT,
+ DAG.getNode(ISD::FMUL, DL, VT, X, K(M_LOG2E)));
+ case ISD::FEXP10:
+ return DAG.getNode(ISD::FEXP2, DL, VT,
+ DAG.getNode(ISD::FMUL, DL, VT, X, K(3.321928094887362)));
+ case ISD::FLOG:
+ return DAG.getNode(ISD::FMUL, DL, VT,
+ DAG.getNode(ISD::FLOG2, DL, VT, X), K(M_LN2));
+ case ISD::FLOG10:
+ return DAG.getNode(ISD::FMUL, DL, VT,
+ DAG.getNode(ISD::FLOG2, DL, VT, X),
+ K(0.301029995663981195));
+ case ISD::FPOW:
+ return DAG.getNode(
+ ISD::FEXP2, DL, VT,
+ DAG.getNode(ISD::FMUL, DL, VT, Op.getOperand(1),
+ DAG.getNode(ISD::FLOG2, DL, VT, X)));
+ default:
+ llvm_unreachable("unexpected opcode");
+ }
+}
+
+SDValue AGXTargetLowering::lowerINT_TO_FP64(SDValue Op,
+ SelectionDAG &DAG) const {
+ SDLoc DL(Op);
+ EVT VT = Op.getValueType();
+ bool Signed = Op.getOpcode() == ISD::SINT_TO_FP;
+ SDValue X = Op.getOperand(0);
+ auto C32 = [&](uint64_t V) { return DAG.getConstant(V, DL, MVT::i32); };
+
+ SDValue Sign;
+ if (Signed) {
+ // Convert |x| and apply the sign at the end.
+ SDValue S = DAG.getNode(ISD::SRA, DL, MVT::i32, split64(DAG, DL, X).second,
+ C32(31));
+ SDValue S64 = join64(DAG, DL, S, S);
+ X = DAG.getNode(ISD::SUB, DL, MVT::i64,
+ DAG.getNode(ISD::XOR, DL, MVT::i64, X, S64), S64);
+ Sign = DAG.getNode(ISD::AND, DL, MVT::i32, S, C32(0x80000000));
+ }
+
+ // Normalize so that the top bit is set, keep the high 32 bits with a
+ // sticky bit for the rest, convert with round to nearest even and scale by
+ // 2^(32 - lz). This rounds correctly since 32 bits exceed the 24 bit
+ // significand by more than the two guard bits.
+ SDValue LZ = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32,
+ DAG.getNode(ISD::CTLZ, DL, MVT::i64, X));
+ SDValue Norm = DAG.getNode(ISD::SHL, DL, MVT::i64, X, LZ);
+ auto [NLo, NHi] = split64(DAG, DL, Norm);
+ SDValue Sticky = DAG.getSetCC(DL, MVT::i32, NLo, C32(0), ISD::SETNE);
+ SDValue H = DAG.getNode(ISD::OR, DL, MVT::i32, NHi, Sticky);
+ SDValue F = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, H);
+ SDValue ScaleBits = DAG.getNode(
+ ISD::SHL, DL, MVT::i32,
+ DAG.getNode(ISD::SUB, DL, MVT::i32, C32(127 + 32), LZ), C32(23));
+ SDValue R = DAG.getNode(ISD::FMUL, DL, MVT::f32, F,
+ DAG.getBitcast(MVT::f32, ScaleBits));
+ if (Signed)
+ R = DAG.getBitcast(
+ MVT::f32, DAG.getNode(ISD::OR, DL, MVT::i32,
+ DAG.getBitcast(MVT::i32, R), Sign));
+ if (VT != MVT::f32)
+ R = DAG.getNode(ISD::FP_ROUND, DL, VT, R,
+ DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
+ return R;
+}
+
+SDValue AGXTargetLowering::lowerFP_TO_INT64(SDValue Op,
+ SelectionDAG &DAG) const {
+ SDLoc DL(Op);
+ bool Signed = Op.getOpcode() == ISD::FP_TO_SINT;
+ SDValue X = Op.getOperand(0);
+ if (X.getValueType() != MVT::f32)
+ X = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, X);
+
+ SDValue T = DAG.getNode(ISD::FTRUNC, DL, MVT::f32, X);
+ SDValue Sign;
+ if (Signed) {
+ Sign = DAG.getNode(ISD::SRA, DL, MVT::i32, DAG.getBitcast(MVT::i32, T),
+ DAG.getConstant(31, DL, MVT::i32));
+ T = DAG.getNode(ISD::FABS, DL, MVT::f32, T);
+ }
+
+ // hi = floor(t * 2^-32), lo = t - hi * 2^32, both exact.
+ SDValue K0 = DAG.getConstantFP(0x1p-32, DL, MVT::f32);
+ SDValue K1 = DAG.getConstantFP(-0x1p+32, DL, MVT::f32);
+ SDValue HiF = DAG.getNode(ISD::FFLOOR, DL, MVT::f32,
+ DAG.getNode(ISD::FMUL, DL, MVT::f32, T, K0));
+ SDValue LoF = DAG.getNode(ISD::FMA, DL, MVT::f32, HiF, K1, T);
+ SDValue Hi = DAG.getNode(ISD::FP_TO_UINT, DL, MVT::i32, HiF);
+ SDValue Lo = DAG.getNode(ISD::FP_TO_UINT, DL, MVT::i32, LoF);
+ SDValue R = join64(DAG, DL, Lo, Hi);
+ if (Signed) {
+ SDValue S64 = join64(DAG, DL, Sign, Sign);
+ R = DAG.getNode(ISD::SUB, DL, MVT::i64,
+ DAG.getNode(ISD::XOR, DL, MVT::i64, R, S64), S64);
+ }
+ return R;
+}
diff --git a/llvm/lib/Target/AGX/AGXISelLowering.h b/llvm/lib/Target/AGX/AGXISelLowering.h
new file mode 100644
index 0000000000000..98efb397e4f89
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXISelLowering.h
@@ -0,0 +1,101 @@
+//===-- AGXISelLowering.h - AGX DAG lowering interface ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines the interfaces that AGX uses to lower LLVM code into a selection
+// DAG.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AGX_AGXISELLOWERING_H
+#define LLVM_LIB_TARGET_AGX_AGXISELLOWERING_H
+
+#include "llvm/CodeGen/SelectionDAG.h"
+#include "llvm/CodeGen/TargetLowering.h"
+
+namespace llvm {
+
+class AGXSubtarget;
+
+class AGXTargetLowering final : public TargetLowering {
+ const AGXSubtarget &ST;
+
+ SDValue lowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerBRCOND(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerSETCC64(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerShift64(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerCTLZ_CTTZ64(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerFDIV(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerFREM(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerTrig(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerExpLog(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerFP_TO_INT64(SDValue Op, SelectionDAG &DAG) const;
+
+ SDValue performBRCONDCombine(SDNode *N, DAGCombinerInfo &DCI) const;
+
+public:
+ AGXTargetLowering(const TargetMachine &TM, const AGXSubtarget &ST);
+
+ const AGXSubtarget &getSubtarget() const { return ST; }
+
+ EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
+ EVT VT) const override;
+ MVT getScalarShiftAmountTy(const DataLayout &DL, EVT VT) const override;
+
+ SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
+ void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
+ SelectionDAG &DAG) const override;
+ SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
+
+ SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
+ bool IsVarArg,
+ const SmallVectorImpl<ISD::InputArg> &Ins,
+ const SDLoc &DL, SelectionDAG &DAG,
+ SmallVectorImpl<SDValue> &InVals) const override;
+ bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF,
+ bool IsVarArg,
+ const SmallVectorImpl<ISD::OutputArg> &Outs,
+ LLVMContext &Context, const Type *RetTy) const override {
+ return true;
+ }
+ SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
+ const SmallVectorImpl<ISD::OutputArg> &Outs,
+ const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
+ SelectionDAG &DAG) const override;
+ SDValue LowerCall(CallLoweringInfo &CLI,
+ SmallVectorImpl<SDValue> &InVals) const override;
+
+ bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
+ EVT VT) const override;
+ bool isFPImmLegal(const APFloat &Imm, EVT VT,
+ bool ForCodeSize) const override {
+ return VT == MVT::f32 || VT == MVT::f16;
+ }
+ bool isTruncateFree(EVT SrcVT, EVT DstVT) const override;
+ bool isCheapToSpeculateCttz(Type *Ty) const override { return true; }
+ bool isCheapToSpeculateCtlz(Type *Ty) const override { return true; }
+ bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override {
+ return false;
+ }
+ bool mayBeEmittedAsTailCall(const CallInst *CI) const override {
+ return false;
+ }
+
+ AtomicExpansionKind
+ shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const override;
+ AtomicExpansionKind
+ shouldExpandAtomicLoadInIR(LoadInst *LI) const override;
+ AtomicExpansionKind
+ shouldExpandAtomicStoreInIR(StoreInst *SI) const override;
+};
+
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AGX_AGXISELLOWERING_H
diff --git a/llvm/lib/Target/AGX/AGXInsertWaits.cpp b/llvm/lib/Target/AGX/AGXInsertWaits.cpp
new file mode 100644
index 0000000000000..9c6b2afa93e31
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXInsertWaits.cpp
@@ -0,0 +1,109 @@
+//===-- AGXInsertWaits.cpp - Insert scoreboard waits ----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Memory loads, stack loads and atomics write their results asynchronously,
+// tracked by a scoreboard slot. Like SIInsertWaitcnts, this pass inserts a
+// wait on the slot before any instruction reads or overwrites a register
+// with a pending result, before barriers and at the end of every block with
+// pending results (so that no wait is needed across control flow). At most
+// eight requests are kept outstanding.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AGX.h"
+#include "AGXInstrInfo.h"
+#include "AGXSubtarget.h"
+#include "MCTargetDesc/AGXMCTargetDesc.h"
+#include "llvm/ADT/BitVector.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "agx-insert-waits"
+
+namespace {
+
+class AGXInsertWaits : public MachineFunctionPass {
+public:
+ static char ID;
+ AGXInsertWaits() : MachineFunctionPass(ID) {}
+
+ StringRef getPassName() const override { return "AGX Insert Waits"; }
+ bool runOnMachineFunction(MachineFunction &MF) override;
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesCFG();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+};
+
+} // namespace
+
+char AGXInsertWaits::ID = 0;
+
+INITIALIZE_PASS(AGXInsertWaits, DEBUG_TYPE, "AGX Insert Waits", false, false)
+
+FunctionPass *llvm::createAGXInsertWaitsPass() { return new AGXInsertWaits(); }
+
+static constexpr unsigned MaxPending = 8;
+
+static bool isBarrier(const MachineInstr &MI) {
+ return MI.getOpcode() == AGX::THREADGROUP_BARRIER ||
+ MI.getOpcode() == AGX::MEMORY_BARRIER;
+}
+
+bool AGXInsertWaits::runOnMachineFunction(MachineFunction &MF) {
+ const AGXSubtarget &ST = MF.getSubtarget<AGXSubtarget>();
+ const AGXInstrInfo &TII = *ST.getInstrInfo();
+ const TargetRegisterInfo &TRI = *ST.getRegisterInfo();
+ bool Changed = false;
+
+ for (MachineBasicBlock &MBB : MF) {
+ BitVector Pending(TRI.getNumRegUnits());
+ unsigned NumPending = 0;
+
+ auto Wait = [&](MachineBasicBlock::iterator I) {
+ BuildMI(MBB, I, DebugLoc(), TII.get(AGX::WAIT)).addImm(0);
+ Pending.reset();
+ NumPending = 0;
+ Changed = true;
+ };
+
+ auto Touches = [&](const MachineInstr &MI) {
+ for (const MachineOperand &MO : MI.operands()) {
+ if (!MO.isReg() || !MO.getReg() || MO.getReg() == AGX::R0L)
+ continue;
+ for (MCRegUnit U : TRI.regunits(MO.getReg().asMCReg()))
+ if (Pending.test(static_cast<unsigned>(U)))
+ return true;
+ }
+ return false;
+ };
+
+ for (MachineInstr &MI : MBB) {
+ if (MI.getOpcode() == AGX::WAIT) {
+ Pending.reset();
+ NumPending = 0;
+ continue;
+ }
+ if (NumPending &&
+ (Touches(MI) || isBarrier(MI) || MI.isTerminator() ||
+ (AGXInstrInfo::isAsync(MI) && NumPending >= MaxPending)))
+ Wait(MI.getIterator());
+ if (AGXInstrInfo::isAsync(MI)) {
+ for (const MachineOperand &MO : MI.defs())
+ for (MCRegUnit U : TRI.regunits(MO.getReg().asMCReg()))
+ Pending.set(static_cast<unsigned>(U));
+ ++NumPending;
+ }
+ }
+ if (NumPending)
+ Wait(MBB.end());
+ }
+ return Changed;
+}
diff --git a/llvm/lib/Target/AGX/AGXInstrFormats.td b/llvm/lib/Target/AGX/AGXInstrFormats.td
new file mode 100644
index 0000000000000..4a92e64f7e53d
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXInstrFormats.td
@@ -0,0 +1,116 @@
+//===-- AGXInstrFormats.td - AGX instruction formats -------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Like AMDGPU, the AGX backend separates
+//
+// * pseudo instructions (AGXPseudo), used by code generation, with precise
+// register classes, and
+// * real instructions (AGXReal), used by the MC layer, with a binary
+// encoding for one encoding family.
+//
+// AGXInstrInfo::pseudoToMCOpcode maps a pseudo to the real instruction of the
+// subtarget's encoding family. Both have the same flattened operand list,
+// except that real ALU destinations carry a trailing cache hint immediate.
+//
+// Instructions are up to 12 bytes long. Some instructions have a short and a
+// long form; the long form sets a length bit and appends the high bits of the
+// register and immediate fields. The `Inst` field always describes the long
+// form, and the code emitter drops the extension when it is all zero.
+//
+//===----------------------------------------------------------------------===//
+
+class AGXInst<dag outs, dag ins, string asm = "", list<dag> pattern = []>
+ : Instruction {
+ let Namespace = "AGX";
+ let OutOperandList = outs;
+ let InOperandList = ins;
+ let AsmString = asm;
+ let Pattern = pattern;
+ let UseNamedOperandTable = 1;
+
+ // Encoded length in bytes of the short and long forms. They are equal for
+ // fixed length instructions.
+ bits<4> ShortSize = 0;
+ bits<4> LongSize = 0;
+ // Position of the length bit for variable length instructions.
+ bits<7> LengthBit = 15;
+
+ // The instruction writes its results asynchronously (memory loads,
+ // atomics, ...). A wait on its scoreboard slot is required before any
+ // reader or writer of the results.
+ bit IsAsync = 0;
+
+ // The instruction reads or updates the execution mask nesting counter
+ // (r0l) and the execution mask.
+ bit IsExecMaskOp = 0;
+
+ // The instruction belongs to a family with a common encoding layout.
+ bit IsALU = 0;
+ bit IsMemory = 0;
+
+ let TSFlags{3-0} = ShortSize;
+ let TSFlags{7-4} = LongSize;
+ let TSFlags{14-8} = LengthBit;
+ let TSFlags{15} = IsAsync;
+ let TSFlags{16} = IsExecMaskOp;
+ let TSFlags{17} = IsALU;
+ let TSFlags{18} = IsMemory;
+}
+
+// Real instructions of the G13/G14 ("AGX2") encoding family.
+class AGXReal<string mnemonic, dag outs, dag ins, string operands,
+ int short, int long = short>
+ : AGXInst<outs, ins, mnemonic#!if(!empty(operands), "", " "#operands)> {
+ field bits<96> Inst;
+ field bits<96> SoftFail = 0;
+
+ let ShortSize = short;
+ let LongSize = long;
+ let Size = long;
+ let DecoderNamespace = "G13";
+ let Predicates = [HasAGX2Encoding];
+ let AsmVariantName = "";
+ let isCodeGenOnly = 0;
+ let hasSideEffects = 0;
+ let mayLoad = 0;
+ let mayStore = 0;
+
+ // Instructions without a length bit leave LengthBit at zero.
+ let LengthBit = !if(!eq(short, long), 0, 15);
+}
+
+// Placeholder for "no real instruction in this encoding family".
+def AGX_NO_REAL : AGXInst<(outs), (ins)> {
+ let isPseudo = 1;
+ let isCodeGenOnly = 1;
+}
+
+// Pseudo instructions used by code generation. G13Real names the real
+// instruction for the AGX2 encoding family.
+class AGXPseudo<dag outs, dag ins, list<dag> pattern = [], string asm = "">
+ : AGXInst<outs, ins, asm, pattern> {
+ let isPseudo = 1;
+ let isCodeGenOnly = 1;
+ let hasSideEffects = 0;
+ let Size = 12;
+
+ Instruction Pseudo = !cast<Instruction>(NAME);
+ Instruction G13Real = AGX_NO_REAL;
+ // Real instruction for the Apple9 encoding family, if any.
+ Instruction Apple9Real = AGX_NO_REAL;
+}
+
+// Pseudo instructions without any real instruction: expanded before
+// emission.
+class AGXCodeGenPseudo<dag outs, dag ins, list<dag> pattern = []>
+ : AGXInst<outs, ins, "", pattern> {
+ let isPseudo = 1;
+ let isCodeGenOnly = 1;
+ let hasSideEffects = 0;
+ let Size = 0;
+}
diff --git a/llvm/lib/Target/AGX/AGXInstrInfo.cpp b/llvm/lib/Target/AGX/AGXInstrInfo.cpp
new file mode 100644
index 0000000000000..cbe619c82845b
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXInstrInfo.cpp
@@ -0,0 +1,307 @@
+//===-- AGXInstrInfo.cpp - AGX instruction information --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// AGX implementation of the TargetInstrInfo class.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AGXInstrInfo.h"
+#include "AGXSubtarget.h"
+#include "MCTargetDesc/AGXMCTargetDesc.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineMemOperand.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+
+using namespace llvm;
+
+#define GET_INSTRINFO_CTOR_DTOR
+#include "AGXGenInstrInfo.inc"
+
+namespace llvm {
+namespace AGX {
+struct AGXPseudoToReal {
+ unsigned Pseudo;
+ unsigned G13Real;
+ unsigned Apple9Real;
+};
+
+#define GET_AGXPseudoToRealTable_DECL
+#define GET_AGXPseudoToRealTable_IMPL
+#include "AGXGenSearchableTables.inc"
+} // namespace AGX
+} // namespace llvm
+
+// Truth table of bitop selecting the first source.
+static constexpr unsigned BitopMov = 0b1010;
+
+AGXInstrInfo::AGXInstrInfo(const AGXSubtarget &ST)
+ : AGXGenInstrInfo(ST, RI), RI(ST), ST(ST) {}
+
+void AGXInstrInfo::copy32(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator MI, const DebugLoc &DL,
+ MCRegister Dst, MCRegister Src, bool KillSrc) const {
+ BuildMI(MBB, MI, DL, get(AGX::BITOP_I32), Dst)
+ .addImm(BitopMov)
+ .addImm(0)
+ .addReg(Src, getKillRegState(KillSrc))
+ .addImm(0)
+ .addImm(0);
+}
+
+void AGXInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator MI,
+ const DebugLoc &DL, Register DestReg,
+ Register SrcReg, bool KillSrc,
+ bool RenamableDest, bool RenamableSrc) const {
+ const MCRegisterInfo &MRI = *ST.getRegisterInfo();
+ AGX::RegInfo D = AGX::getRegInfo(DestReg, MRI);
+ AGX::RegInfo S = AGX::getRegInfo(SrcReg, MRI);
+ unsigned Bits = D.getSizeInBits();
+ if (!D.isGPR() || Bits != S.getSizeInBits())
+ report_fatal_error("unsupported AGX register copy");
+
+ if (Bits == 16) {
+ BuildMI(MBB, MI, DL, get(AGX::BITOP_I16), DestReg)
+ .addImm(BitopMov)
+ .addImm(0)
+ .addReg(SrcReg, getKillRegState(KillSrc))
+ .addImm(0)
+ .addImm(0);
+ return;
+ }
+ if (Bits == 32) {
+ copy32(MBB, MI, DL, DestReg, SrcReg, KillSrc);
+ return;
+ }
+
+ // Register tuples are copied one 32-bit register at a time, in an order
+ // that does not clobber source registers before they are read.
+ static const unsigned SubRegs[] = {AGX::sub0, AGX::sub1, AGX::sub2,
+ AGX::sub3};
+ unsigned N = Bits / 32;
+ bool Backwards = D.HalfIndex > S.HalfIndex && !S.isUniform();
+ for (unsigned I = 0; I < N; ++I) {
+ unsigned Idx = SubRegs[Backwards ? N - 1 - I : I];
+ MCRegister DstSub = RI.getSubReg(DestReg, Idx);
+ MCRegister SrcSub = RI.getSubReg(SrcReg, Idx);
+ copy32(MBB, MI, DL, DstSub, SrcSub, KillSrc && I == N - 1);
+ }
+}
+
+static unsigned getStackOpcode(unsigned Bits, bool Store) {
+ switch (Bits) {
+ case 16:
+ return Store ? AGX::STACK_STORE_16 : AGX::STACK_LOAD_16;
+ case 32:
+ return Store ? AGX::STACK_STORE_32 : AGX::STACK_LOAD_32;
+ case 64:
+ return Store ? AGX::STACK_STORE_64 : AGX::STACK_LOAD_64;
+ case 96:
+ return Store ? AGX::STACK_STORE_96 : AGX::STACK_LOAD_96;
+ case 128:
+ return Store ? AGX::STACK_STORE_128 : AGX::STACK_LOAD_128;
+ default:
+ report_fatal_error("unsupported AGX spill size");
+ }
+}
+
+// Memory format and mask for a stack access of the given size.
+static std::pair<unsigned, unsigned> getStackFormat(unsigned Bits) {
+ if (Bits == 16)
+ return {AGX::FMT_I16, 1};
+ return {AGX::FMT_I32, (1u << (Bits / 32)) - 1};
+}
+
+void AGXInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator MI,
+ Register SrcReg, bool IsKill,
+ int FrameIndex,
+ const TargetRegisterClass *RC,
+ Register VReg,
+ MachineInstr::MIFlag Flags) const {
+ MachineFunction &MF = *MBB.getParent();
+ MachineFrameInfo &MFI = MF.getFrameInfo();
+ unsigned Bits = RI.getRegSizeInBits(*RC);
+ auto [Fmt, Mask] = getStackFormat(Bits);
+ MachineMemOperand *MMO = MF.getMachineMemOperand(
+ MachinePointerInfo::getFixedStack(MF, FrameIndex),
+ MachineMemOperand::MOStore, MFI.getObjectSize(FrameIndex),
+ MFI.getObjectAlign(FrameIndex));
+ BuildMI(MBB, MI, DebugLoc(), get(getStackOpcode(Bits, /*Store=*/true)))
+ .addImm(0)
+ .addImm(Fmt)
+ .addImm(Mask)
+ .addReg(SrcReg, getKillRegState(IsKill))
+ .addFrameIndex(FrameIndex)
+ .addMemOperand(MMO)
+ .setMIFlags(Flags);
+}
+
+void AGXInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator MI,
+ Register DestReg, int FrameIndex,
+ const TargetRegisterClass *RC,
+ Register VReg, unsigned SubReg,
+ MachineInstr::MIFlag Flags) const {
+ MachineFunction &MF = *MBB.getParent();
+ MachineFrameInfo &MFI = MF.getFrameInfo();
+ unsigned Bits = RI.getRegSizeInBits(*RC);
+ auto [Fmt, Mask] = getStackFormat(Bits);
+ MachineMemOperand *MMO = MF.getMachineMemOperand(
+ MachinePointerInfo::getFixedStack(MF, FrameIndex),
+ MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIndex),
+ MFI.getObjectAlign(FrameIndex));
+ BuildMI(MBB, MI, DebugLoc(), get(getStackOpcode(Bits, /*Store=*/false)),
+ DestReg)
+ .addImm(0)
+ .addImm(Fmt)
+ .addImm(Mask)
+ .addFrameIndex(FrameIndex)
+ .addMemOperand(MMO)
+ .setMIFlags(Flags);
+}
+
+static const MachineOperand *getNamedOperand(const MachineInstr &MI,
+ AGX::OpName Name) {
+ int Idx = AGX::getNamedOperandIdx(MI.getOpcode(), Name);
+ return Idx < 0 ? nullptr : &MI.getOperand(Idx);
+}
+
+static bool isStackLoad(unsigned Opc) {
+ switch (Opc) {
+ case AGX::STACK_LOAD_16:
+ case AGX::STACK_LOAD_32:
+ case AGX::STACK_LOAD_64:
+ case AGX::STACK_LOAD_96:
+ case AGX::STACK_LOAD_128:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static bool isStackStore(unsigned Opc) {
+ switch (Opc) {
+ case AGX::STACK_STORE_16:
+ case AGX::STACK_STORE_32:
+ case AGX::STACK_STORE_64:
+ case AGX::STACK_STORE_96:
+ case AGX::STACK_STORE_128:
+ return true;
+ default:
+ return false;
+ }
+}
+
+Register AGXInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
+ int &FrameIndex) const {
+ if (!isStackLoad(MI.getOpcode()))
+ return Register();
+ const MachineOperand *O = getNamedOperand(MI, AGX::OpName::O);
+ if (!O || !O->isFI())
+ return Register();
+ FrameIndex = O->getIndex();
+ return MI.getOperand(0).getReg();
+}
+
+Register AGXInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
+ int &FrameIndex) const {
+ if (!isStackStore(MI.getOpcode()))
+ return Register();
+ const MachineOperand *O = getNamedOperand(MI, AGX::OpName::O);
+ if (!O || !O->isFI())
+ return Register();
+ FrameIndex = O->getIndex();
+ return getNamedOperand(MI, AGX::OpName::R)->getReg();
+}
+
+bool AGXInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
+ MachineBasicBlock *&TBB,
+ MachineBasicBlock *&FBB,
+ SmallVectorImpl<MachineOperand> &Cond,
+ bool AllowModify) const {
+ MachineBasicBlock::iterator I = MBB.getFirstTerminator();
+ if (I == MBB.end())
+ return false;
+
+ // Execution mask branches update the nesting counter as well as branching:
+ // they cannot be removed or reversed.
+ if (I->getOpcode() != AGX::AGX_BR)
+ return true;
+ if (std::next(I) != MBB.end())
+ return true;
+ TBB = I->getOperand(0).getMBB();
+ return false;
+}
+
+unsigned AGXInstrInfo::removeBranch(MachineBasicBlock &MBB,
+ int *BytesRemoved) const {
+ MachineBasicBlock::iterator I = MBB.getFirstTerminator();
+ if (I == MBB.end() || I->getOpcode() != AGX::AGX_BR)
+ return 0;
+ if (BytesRemoved)
+ *BytesRemoved = getInstSizeInBytes(*I);
+ I->eraseFromParent();
+ return 1;
+}
+
+unsigned AGXInstrInfo::insertBranch(MachineBasicBlock &MBB,
+ MachineBasicBlock *TBB,
+ MachineBasicBlock *FBB,
+ ArrayRef<MachineOperand> Cond,
+ const DebugLoc &DL,
+ int *BytesAdded) const {
+ assert(TBB && Cond.empty() && !FBB &&
+ "AGX only inserts unconditional branches");
+ MachineInstr &MI = *BuildMI(&MBB, DL, get(AGX::AGX_BR)).addMBB(TBB);
+ if (BytesAdded)
+ *BytesAdded = getInstSizeInBytes(MI);
+ return 1;
+}
+
+bool AGXInstrInfo::isBasicBlockPrologue(const MachineInstr &MI,
+ Register Reg) const {
+ return MI.getOpcode() == AGX::AGX_END_CF ||
+ MI.getOpcode() == AGX::SET_NEST;
+}
+
+unsigned AGXInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
+ switch (MI.getOpcode()) {
+ case AGX::AGX_IF:
+ case AGX::AGX_ELSE:
+ case AGX::AGX_LOOP:
+ // An execution mask instruction and a jump.
+ return 12;
+ case AGX::AGX_BR:
+ return 12;
+ case AGX::AGX_END_CF:
+ case AGX::AGX_LOOP_BEGIN:
+ return 6;
+ default:
+ break;
+ }
+ if (MI.isMetaInstruction())
+ return 0;
+ int Real = pseudoToMCOpcode(MI.getOpcode());
+ if (Real >= 0)
+ return get(Real).getSize();
+ return MI.getDesc().getSize();
+}
+
+int AGXInstrInfo::pseudoToMCOpcode(unsigned Opcode) const {
+ const AGX::AGXPseudoToReal *Entry = AGX::getAGXPseudoToReal(Opcode);
+ if (!Entry)
+ return -1;
+ unsigned Real = ST.getEncodingFamily() == AGXSubtarget::Apple9
+ ? Entry->Apple9Real
+ : Entry->G13Real;
+ if (Real == AGX::AGX_NO_REAL)
+ return -1;
+ return Real;
+}
diff --git a/llvm/lib/Target/AGX/AGXInstrInfo.h b/llvm/lib/Target/AGX/AGXInstrInfo.h
new file mode 100644
index 0000000000000..7599f10ae1d2f
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXInstrInfo.h
@@ -0,0 +1,101 @@
+//===-- AGXInstrInfo.h - AGX instruction information ------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Interface definition for AGXInstrInfo.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AGX_AGXINSTRINFO_H
+#define LLVM_LIB_TARGET_AGX_AGXINSTRINFO_H
+
+#include "AGXRegisterInfo.h"
+#include "Utils/AGXBaseInfo.h"
+#include "llvm/CodeGen/TargetInstrInfo.h"
+
+#define GET_INSTRINFO_HEADER
+#include "AGXGenInstrInfo.inc"
+
+namespace llvm {
+
+class AGXSubtarget;
+
+class AGXInstrInfo final : public AGXGenInstrInfo {
+ const AGXRegisterInfo RI;
+ const AGXSubtarget &ST;
+
+ void copy32(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
+ const DebugLoc &DL, MCRegister Dst, MCRegister Src,
+ bool KillSrc) const;
+
+public:
+ explicit AGXInstrInfo(const AGXSubtarget &ST);
+
+ const AGXRegisterInfo &getRegisterInfo() const { return RI; }
+
+ void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
+ const DebugLoc &DL, Register DestReg, Register SrcReg,
+ bool KillSrc, bool RenamableDest = false,
+ bool RenamableSrc = false) const override;
+
+ void storeRegToStackSlot(
+ MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
+ bool IsKill, int FrameIndex, const TargetRegisterClass *RC,
+ Register VReg,
+ MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
+
+ void loadRegFromStackSlot(
+ MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg,
+ int FrameIndex, const TargetRegisterClass *RC, Register VReg,
+ unsigned SubReg = 0,
+ MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
+
+ Register isLoadFromStackSlot(const MachineInstr &MI,
+ int &FrameIndex) const override;
+ Register isStoreToStackSlot(const MachineInstr &MI,
+ int &FrameIndex) const override;
+
+ bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
+ MachineBasicBlock *&FBB,
+ SmallVectorImpl<MachineOperand> &Cond,
+ bool AllowModify = false) const override;
+ unsigned removeBranch(MachineBasicBlock &MBB,
+ int *BytesRemoved = nullptr) const override;
+ unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
+ MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
+ const DebugLoc &DL,
+ int *BytesAdded = nullptr) const override;
+ bool
+ reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const override {
+ return true;
+ }
+
+ /// Execution mask instructions at the start of a block (the reconvergence
+ /// of the enclosing construct) must stay before everything else.
+ bool isBasicBlockPrologue(const MachineInstr &MI,
+ Register Reg = Register()) const override;
+
+ unsigned getInstSizeInBytes(const MachineInstr &MI) const override;
+
+ /// The real instruction of the subtarget's encoding family for the pseudo
+ /// instruction \p Opcode, or -1 if there is none.
+ int pseudoToMCOpcode(unsigned Opcode) const;
+
+ static bool isAsync(const MachineInstr &MI) {
+ return AGX::TSFlags::isAsync(MI.getDesc().TSFlags);
+ }
+ static bool isALU(const MachineInstr &MI) {
+ return AGX::TSFlags::isALU(MI.getDesc().TSFlags);
+ }
+ static bool isExecMaskOp(const MachineInstr &MI) {
+ return AGX::TSFlags::isExecMaskOp(MI.getDesc().TSFlags);
+ }
+};
+
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AGX_AGXINSTRINFO_H
diff --git a/llvm/lib/Target/AGX/AGXInstrInfo.td b/llvm/lib/Target/AGX/AGXInstrInfo.td
new file mode 100644
index 0000000000000..b744492c1598b
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXInstrInfo.td
@@ -0,0 +1,109 @@
+//===-- AGXInstrInfo.td - AGX instruction information ------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+include "AGXInstrFormats.td"
+include "AGXOperands.td"
+
+//===----------------------------------------------------------------------===//
+// Target nodes
+//===----------------------------------------------------------------------===//
+
+def SDT_AGXCFCond : SDTypeProfile<0, 2, [SDTCisVT<0, i32>, SDTCisVT<1, OtherVT>]>;
+def SDT_AGXCFTarget : SDTypeProfile<0, 1, [SDTCisVT<0, OtherVT>]>;
+
+// Kernel return.
+def AGXstop : SDNode<"AGXISD::STOP", SDTNone,
+ [SDNPHasChain, SDNPOptInGlue]>;
+
+// Structured control flow. The branch target is the block executed when no
+// thread remains active (or, for loops, the loop header).
+def AGXif : SDNode<"AGXISD::IF", SDT_AGXCFCond, [SDNPHasChain]>;
+def AGXelse : SDNode<"AGXISD::ELSE", SDT_AGXCFTarget, [SDNPHasChain]>;
+def AGXloop : SDNode<"AGXISD::LOOP", SDT_AGXCFCond, [SDNPHasChain]>;
+def AGXend_cf : SDNode<"AGXISD::END_CF", SDTNone,
+ [SDNPHasChain, SDNPSideEffect]>;
+def AGXloop_begin : SDNode<"AGXISD::LOOP_BEGIN", SDTNone,
+ [SDNPHasChain, SDNPSideEffect]>;
+
+// SIMD-group ballots of a 0/1 value.
+def AGXballot : SDNode<"AGXISD::BALLOT", SDTIntUnaryOp>;
+def AGXquad_ballot : SDNode<"AGXISD::QUAD_BALLOT", SDTIntUnaryOp>;
+
+// sin(x) for x in quadrants: sin_pt_1, sin_pt_2 and a multiply.
+def AGXsin_quadrants : SDNode<"AGXISD::SIN_QUADRANTS", SDTFPUnaryOp>;
+
+//===----------------------------------------------------------------------===//
+// Complex patterns
+//===----------------------------------------------------------------------===//
+
+// Integer ALU source: a register, a uniform register or an 8-bit immediate.
+def AGXIntSrc16 : ComplexPattern<i16, 1, "SelectIntSrc", [], [], -2>;
+def AGXIntSrc32 : ComplexPattern<i32, 1, "SelectIntSrc", [], [], -2>;
+def AGXIntSrc64 : ComplexPattern<i64, 1, "SelectIntSrc", [], [], -2>;
+
+// Floating point source with abs/neg modifiers: (value, mods). The value may
+// be an 8-bit floating point immediate.
+def AGXFPSrc16 : ComplexPattern<f16, 2, "SelectFPSrc", [], [], -2>;
+def AGXFPSrc32 : ComplexPattern<f32, 2, "SelectFPSrc", [], [], -2>;
+
+// Device memory addresses: (base, index, signedness, shift) for an element
+// size of 1, 2 or 4 bytes.
+def AGXDeviceAddr8 : ComplexPattern<i64, 4, "SelectDeviceAddr8", [], [], -10>;
+def AGXDeviceAddr16 : ComplexPattern<i64, 4, "SelectDeviceAddr16", [], [], -10>;
+def AGXDeviceAddr32 : ComplexPattern<i64, 4, "SelectDeviceAddr32", [], [], -10>;
+
+// Threadgroup memory addresses: (base, index) for an element size of 1, 2 or
+// 4 bytes.
+def AGXLocalAddr8 : ComplexPattern<i32, 2, "SelectLocalAddr8", [], [], -10>;
+def AGXLocalAddr16 : ComplexPattern<i32, 2, "SelectLocalAddr16", [], [], -10>;
+def AGXLocalAddr32 : ComplexPattern<i32, 2, "SelectLocalAddr32", [], [], -10>;
+
+// Stack addresses: a byte offset (frame index, register or immediate).
+def AGXStackAddr : ComplexPattern<i32, 1, "SelectStackAddr", [frameindex],
+ [], -10>;
+
+//===----------------------------------------------------------------------===//
+// Address space fragments
+//===----------------------------------------------------------------------===//
+
+class AddressSpaceList<list<int> AS> {
+ list<int> AddrSpaces = AS;
+}
+
+// Generic (0), device (1) and constant (4) pointers all address device
+// memory.
+defvar DeviceAS = [0, 1, 4];
+defvar LocalAS = [3];
+defvar PrivateAS = [5];
+
+multiclass AGXLoadFrags<list<int> AS> {
+ let AddressSpaces = AS in {
+ def _load : PatFrag<(ops node:$ptr), (load node:$ptr)>;
+ def _extloadi8 : PatFrag<(ops node:$ptr), (extloadi8 node:$ptr)>;
+ def _zextloadi8 : PatFrag<(ops node:$ptr), (zextloadi8 node:$ptr)>;
+ def _extloadi16 : PatFrag<(ops node:$ptr), (extloadi16 node:$ptr)>;
+ def _zextloadi16 : PatFrag<(ops node:$ptr), (zextloadi16 node:$ptr)>;
+ def _store : PatFrag<(ops node:$val, node:$ptr), (store node:$val, node:$ptr)>;
+ def _truncstorei8 : PatFrag<(ops node:$val, node:$ptr),
+ (truncstorei8 node:$val, node:$ptr)>;
+ def _truncstorei16 : PatFrag<(ops node:$val, node:$ptr),
+ (truncstorei16 node:$val, node:$ptr)>;
+ }
+}
+
+defm device : AGXLoadFrags<DeviceAS>;
+defm local : AGXLoadFrags<LocalAS>;
+defm private : AGXLoadFrags<PrivateAS>;
+
+//===----------------------------------------------------------------------===//
+// Instructions
+//===----------------------------------------------------------------------===//
+
+include "G13Instructions.td"
+include "AGXInstructions.td"
+include "AGXSearchableTables.td"
diff --git a/llvm/lib/Target/AGX/AGXInstructions.td b/llvm/lib/Target/AGX/AGXInstructions.td
new file mode 100644
index 0000000000000..cb794eccba301
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXInstructions.td
@@ -0,0 +1,1270 @@
+//===-- AGXInstructions.td - AGX code generation pseudos ---*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Pseudo instructions used by instruction selection, and their selection
+// patterns. Each pseudo has the operand list of the real instruction it maps
+// to (see AGXInstrFormats.td), with typed register classes:
+//
+// * every ALU source is a (SrcMods, Src*Op) pair,
+// * the destination cache hint of real ALU instructions is omitted.
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// Operand helpers
+//===----------------------------------------------------------------------===//
+
+// Device memory index: a 32-bit register or a signed 16-bit immediate.
+def IndexOp : AGXRegOperand<GPR32, "OPERAND_SRC">;
+// Threadgroup memory base: a 16-bit register, a 16-bit uniform or zero.
+def LocalBaseOp : AGXRegOperand<SRC16, "OPERAND_SRC">;
+// Threadgroup memory index: a 16-bit register or a 16-bit immediate.
+def LocalIndexOp : AGXRegOperand<GPR16, "OPERAND_SRC">;
+
+// Encodings of the memory format field.
+defvar FMT_I8 = 0;
+defvar FMT_I16 = 1;
+defvar FMT_I32 = 2;
+
+// AGXSrcMods flags.
+defvar MOD_ABS = 1;
+defvar MOD_NEG = 2;
+defvar MOD_SX = 4;
+
+// 8-bit floating point immediates.
+defvar FPIMM_NEG_ZERO = 0x80;
+
+class AGXALUPseudo<Instruction real, dag outs, dag ins, list<dag> pattern = [],
+ list<SchedReadWrite> sched = [WriteALU]>
+ : AGXPseudo<outs, ins, pattern> {
+ let G13Real = real;
+ let IsALU = 1;
+ let SchedRW = sched;
+}
+
+//===----------------------------------------------------------------------===//
+// Moves
+//===----------------------------------------------------------------------===//
+
+let isAsCheapAsAMove = 1, isReMaterializable = 1 in {
+def MOV_IMM16 : AGXALUPseudo<MOV_IMM16_g13, (outs GPR16:$dst),
+ (ins i32imm:$imm), [], [WriteMovImm]>;
+def MOV_IMM32 : AGXALUPseudo<MOV_IMM32_g13, (outs GPR32:$dst),
+ (ins i32imm:$imm), [], [WriteMovImm]>;
+}
+
+def imm_to_i32 : SDNodeXForm<imm, [{
+ return CurDAG->getTargetConstant(N->getZExtValue(), SDLoc(N), MVT::i32);
+}]>;
+
+def : Pat<(i16 imm:$imm), (MOV_IMM16 (imm_to_i32 $imm))>;
+def : Pat<(i32 imm:$imm), (MOV_IMM32 imm:$imm)>;
+
+def bitcast_fpimm_to_i32 : SDNodeXForm<fpimm, [{
+ return CurDAG->getTargetConstant(
+ N->getValueAPF().bitcastToAPInt().getZExtValue(), SDLoc(N), MVT::i32);
+}]>;
+
+def : Pat<(f32 fpimm:$imm), (MOV_IMM32 (bitcast_fpimm_to_i32 $imm))>;
+def : Pat<(f16 fpimm:$imm), (MOV_IMM16 (bitcast_fpimm_to_i32 $imm))>;
+def : Pat<(bf16 fpimm:$imm), (MOV_IMM16 (bitcast_fpimm_to_i32 $imm))>;
+
+def imm_lo32 : SDNodeXForm<imm, [{
+ return CurDAG->getTargetConstant(N->getZExtValue() & 0xffffffff, SDLoc(N),
+ MVT::i32);
+}]>;
+def imm_hi32 : SDNodeXForm<imm, [{
+ return CurDAG->getTargetConstant(N->getZExtValue() >> 32, SDLoc(N),
+ MVT::i32);
+}]>;
+
+def : Pat<(i64 imm:$imm),
+ (REG_SEQUENCE GPR64, (MOV_IMM32 (imm_lo32 $imm)), sub0,
+ (MOV_IMM32 (imm_hi32 $imm)), sub1)>;
+
+def GET_SR : AGXALUPseudo<GET_SR_g13, (outs GPR32:$dst), (ins i32imm:$sr),
+ [], [WriteSR]>;
+
+class SRPat<Intrinsic intr, int sr> : Pat<(i32 (intr)), (GET_SR sr)>;
+
+def : SRPat<int_agx_threadgroup_position_in_grid_x, 0>;
+def : SRPat<int_agx_threadgroup_position_in_grid_y, 1>;
+def : SRPat<int_agx_threadgroup_position_in_grid_z, 2>;
+def : SRPat<int_agx_threads_per_threadgroup_x, 4>;
+def : SRPat<int_agx_threads_per_threadgroup_y, 5>;
+def : SRPat<int_agx_threads_per_threadgroup_z, 6>;
+def : SRPat<int_agx_core_id, 20>;
+def : SRPat<int_agx_thread_position_in_threadgroup_x, 48>;
+def : SRPat<int_agx_thread_position_in_threadgroup_y, 49>;
+def : SRPat<int_agx_thread_position_in_threadgroup_z, 50>;
+def : SRPat<int_agx_thread_index_in_threadgroup, 51>;
+def : SRPat<int_agx_thread_index_in_simdgroup, 52>;
+def : SRPat<int_agx_simdgroup_index_in_threadgroup, 53>;
+def : SRPat<int_agx_active_thread_index_in_quadgroup, 56>;
+def : SRPat<int_agx_total_active_threads_in_quadgroup, 57>;
+def : SRPat<int_agx_active_thread_index_in_simdgroup, 58>;
+def : SRPat<int_agx_total_active_threads_in_simdgroup, 59>;
+def : SRPat<int_agx_thread_position_in_grid_x, 80>;
+def : SRPat<int_agx_thread_position_in_grid_y, 81>;
+def : SRPat<int_agx_thread_position_in_grid_z, 82>;
+def : Pat<(i32 (int_agx_read_sr timm:$sr)), (GET_SR timm:$sr)>;
+
+//===----------------------------------------------------------------------===//
+// Bitwise operations
+//===----------------------------------------------------------------------===//
+
+// Truth tables, see BITOP_g13.
+defvar TT_AND = 0b1000;
+defvar TT_OR = 0b1110;
+defvar TT_XOR = 0b0110;
+defvar TT_MOV = 0b1010;
+defvar TT_NOT = 0b0101;
+
+multiclass BitopPseudos<RegisterClass rc, RegisterOperand src, ValueType vt,
+ ComplexPattern srcPat> {
+ def NAME : AGXALUPseudo<BITOP_g13, (outs rc:$dst),
+ (ins i32imm:$tt, SrcMods:$src0_mods, src:$src0,
+ SrcMods:$src1_mods, src:$src1)>;
+
+ def : Pat<(vt (and srcPat:$a, srcPat:$b)),
+ (!cast<Instruction>(NAME) TT_AND, 0, $a, 0, $b)>;
+ def : Pat<(vt (or srcPat:$a, srcPat:$b)),
+ (!cast<Instruction>(NAME) TT_OR, 0, $a, 0, $b)>;
+ def : Pat<(vt (xor srcPat:$a, srcPat:$b)),
+ (!cast<Instruction>(NAME) TT_XOR, 0, $a, 0, $b)>;
+ def : Pat<(vt (not srcPat:$a)),
+ (!cast<Instruction>(NAME) TT_NOT, 0, $a, 0, (vt 0))>;
+}
+
+defm BITOP_I16 : BitopPseudos<GPR16, Src16Op, i16, AGXIntSrc16>;
+defm BITOP_I32 : BitopPseudos<GPR32, Src32Op, i32, AGXIntSrc32>;
+
+// Zero extension and register moves with a narrower source.
+def BITOP_I32_FROM16 : AGXALUPseudo<BITOP_g13, (outs GPR32:$dst),
+ (ins i32imm:$tt, SrcMods:$src0_mods,
+ Src16Op:$src0, SrcMods:$src1_mods,
+ Src16Op:$src1)>;
+
+def : Pat<(i32 (zext i16:$a)), (BITOP_I32_FROM16 TT_MOV, 0, $a, 0, (i16 0))>;
+def : Pat<(i32 (anyext i16:$a)), (BITOP_I32_FROM16 TT_MOV, 0, $a, 0, (i16 0))>;
+
+def : Pat<(i64 (and i64:$a, i64:$b)),
+ (REG_SEQUENCE GPR64,
+ (BITOP_I32 TT_AND, 0, (EXTRACT_SUBREG $a, sub0), 0,
+ (EXTRACT_SUBREG $b, sub0)), sub0,
+ (BITOP_I32 TT_AND, 0, (EXTRACT_SUBREG $a, sub1), 0,
+ (EXTRACT_SUBREG $b, sub1)), sub1)>;
+def : Pat<(i64 (or i64:$a, i64:$b)),
+ (REG_SEQUENCE GPR64,
+ (BITOP_I32 TT_OR, 0, (EXTRACT_SUBREG $a, sub0), 0,
+ (EXTRACT_SUBREG $b, sub0)), sub0,
+ (BITOP_I32 TT_OR, 0, (EXTRACT_SUBREG $a, sub1), 0,
+ (EXTRACT_SUBREG $b, sub1)), sub1)>;
+def : Pat<(i64 (xor i64:$a, i64:$b)),
+ (REG_SEQUENCE GPR64,
+ (BITOP_I32 TT_XOR, 0, (EXTRACT_SUBREG $a, sub0), 0,
+ (EXTRACT_SUBREG $b, sub0)), sub0,
+ (BITOP_I32 TT_XOR, 0, (EXTRACT_SUBREG $a, sub1), 0,
+ (EXTRACT_SUBREG $b, sub1)), sub1)>;
+
+def BITREV_I32 : AGXALUPseudo<BITREV_g13, (outs GPR32:$dst),
+ (ins SrcMods:$src0_mods, Src32Op:$src0), [],
+ [WriteIMul]>;
+def POPCOUNT_I32 : AGXALUPseudo<POPCOUNT_g13, (outs GPR32:$dst),
+ (ins SrcMods:$src0_mods, Src32Op:$src0), [],
+ [WriteIMul]>;
+// Index of the most significant set bit, or -1 for zero.
+def FFS_I32 : AGXALUPseudo<FFS_g13, (outs GPR32:$dst),
+ (ins SrcMods:$src0_mods, Src32Op:$src0), [],
+ [WriteIMul]>;
+def INTL_I32 : AGXALUPseudo<INTL_g13, (outs GPR32:$dst),
+ (ins SrcMods:$src0_mods, Src32Op:$src0,
+ SrcMods:$src1_mods, Src32Op:$src1), [],
+ [WriteIMul]>;
+
+def : Pat<(i32 (bitreverse AGXIntSrc32:$a)), (BITREV_I32 0, $a)>;
+def : Pat<(i32 (ctpop AGXIntSrc32:$a)), (POPCOUNT_I32 0, $a)>;
+def : Pat<(i32 (int_agx_interleave AGXIntSrc32:$a, AGXIntSrc32:$b)),
+ (INTL_I32 0, $a, 0, $b)>;
+
+//===----------------------------------------------------------------------===//
+// Integer arithmetic
+//===----------------------------------------------------------------------===//
+
+multiclass IAddPseudos<Instruction real, RegisterClass rc,
+ RegisterOperand src, ValueType vt, ComplexPattern pat,
+ SDPatternOperator node,
+ list<SchedReadWrite> sched = [WriteALU]> {
+ def NAME : AGXALUPseudo<real, (outs rc:$dst),
+ (ins SrcMods:$src0_mods, src:$src0,
+ SrcMods:$src1_mods, src:$src1, i32imm:$shift),
+ [], sched>;
+ def : Pat<(vt (node pat:$a, pat:$b)),
+ (!cast<Instruction>(NAME) 0, $a, 0, $b, 0)>;
+}
+
+defm IADD_I16 : IAddPseudos<IADD_g13, GPR16, Src16Op, i16,
+ AGXIntSrc16, add>;
+defm IADD_I32 : IAddPseudos<IADD_g13, GPR32, Src32Op, i32,
+ AGXIntSrc32, add>;
+defm IADD_I64 : IAddPseudos<IADD_g13, GPR64, Src64Op, i64,
+ AGXIntSrc64, add, [WriteALU64]>;
+defm ISUB_I16 : IAddPseudos<ISUB_g13, GPR16, Src16Op, i16,
+ AGXIntSrc16, sub>;
+defm ISUB_I32 : IAddPseudos<ISUB_g13, GPR32, Src32Op, i32,
+ AGXIntSrc32, sub>;
+defm ISUB_I64 : IAddPseudos<ISUB_g13, GPR64, Src64Op, i64,
+ AGXIntSrc64, sub, [WriteALU64]>;
+
+// Saturating add and subtract. Signedness comes from the sign extension
+// flags of the sources.
+def IADD_SAT_I32 : AGXALUPseudo<IADD_SAT_g13, (outs GPR32:$dst),
+ (ins SrcMods:$src0_mods, Src32Op:$src0,
+ SrcMods:$src1_mods, Src32Op:$src1,
+ i32imm:$shift)>;
+def ISUB_SAT_I32 : AGXALUPseudo<ISUB_SAT_g13, (outs GPR32:$dst),
+ (ins SrcMods:$src0_mods, Src32Op:$src0,
+ SrcMods:$src1_mods, Src32Op:$src1,
+ i32imm:$shift)>;
+
+def : Pat<(i32 (uaddsat AGXIntSrc32:$a, AGXIntSrc32:$b)),
+ (IADD_SAT_I32 0, $a, 0, $b, 0)>;
+def : Pat<(i32 (usubsat AGXIntSrc32:$a, AGXIntSrc32:$b)),
+ (ISUB_SAT_I32 0, $a, 0, $b, 0)>;
+def : Pat<(i32 (saddsat i32:$a, i32:$b)),
+ (IADD_SAT_I32 MOD_SX, $a, MOD_SX, $b, 0)>;
+def : Pat<(i32 (ssubsat i32:$a, i32:$b)),
+ (ISUB_SAT_I32 MOD_SX, $a, MOD_SX, $b, 0)>;
+
+// Sign and zero extension through iadd with a zero addend.
+def IADD_I32_FROM16 : AGXALUPseudo<IADD_g13, (outs GPR32:$dst),
+ (ins SrcMods:$src0_mods, Src16Op:$src0,
+ SrcMods:$src1_mods, Src16Op:$src1,
+ i32imm:$shift)>;
+def IADD_I64_FROM32 : AGXALUPseudo<IADD_g13, (outs GPR64:$dst),
+ (ins SrcMods:$src0_mods, Src32Op:$src0,
+ SrcMods:$src1_mods, Src32Op:$src1,
+ i32imm:$shift), [], [WriteALU64]>;
+
+// 64-bit address arithmetic: base + (extended 32-bit index << shift).
+def IADD_I64_I32 : AGXALUPseudo<IADD_g13, (outs GPR64:$dst),
+ (ins SrcMods:$src0_mods, Src64Op:$src0,
+ SrcMods:$src1_mods, Src32Op:$src1,
+ i32imm:$shift), [], [WriteALU64]>;
+
+// The iadd shift amount, 0 to 4.
+def agx_iadd_shift : ImmLeaf<i32, [{ return Imm >= 0 && Imm <= 4; }]>;
+
+def : Pat<(i64 (shl (zext i32:$x), agx_iadd_shift:$s)),
+ (IADD_I64_FROM32 0, (i32 0), 0, $x, imm:$s)>;
+def : Pat<(i64 (shl (sext i32:$x), agx_iadd_shift:$s)),
+ (IADD_I64_FROM32 0, (i32 0), MOD_SX, $x, imm:$s)>;
+def : Pat<(i64 (add i64:$a, (shl (zext i32:$x), agx_iadd_shift:$s))),
+ (IADD_I64_I32 0, $a, 0, $x, imm:$s)>;
+def : Pat<(i64 (add i64:$a, (shl (sext i32:$x), agx_iadd_shift:$s))),
+ (IADD_I64_I32 0, $a, MOD_SX, $x, imm:$s)>;
+def : Pat<(i64 (add i64:$a, (zext i32:$x))), (IADD_I64_I32 0, $a, 0, $x, 0)>;
+def : Pat<(i64 (add i64:$a, (sext i32:$x))),
+ (IADD_I64_I32 0, $a, MOD_SX, $x, 0)>;
+def : Pat<(i64 (add i64:$a, (shl i64:$b, agx_iadd_shift:$s))),
+ (IADD_I64 0, $a, 0, $b, imm:$s)>;
+
+def : Pat<(i32 (sext i16:$a)), (IADD_I32_FROM16 MOD_SX, $a, 0, (i16 0), 0)>;
+def : Pat<(i64 (sext i32:$a)), (IADD_I64_FROM32 MOD_SX, $a, 0, (i32 0), 0)>;
+def : Pat<(i64 (zext i32:$a)), (IADD_I64_FROM32 0, $a, 0, (i32 0), 0)>;
+def : Pat<(i64 (anyext i32:$a)), (IADD_I64_FROM32 0, $a, 0, (i32 0), 0)>;
+def : Pat<(i32 (sext_inreg i32:$a, i16)),
+ (IADD_I32_FROM16 MOD_SX, (EXTRACT_SUBREG $a, lo16), 0, (i16 0), 0)>;
+def : Pat<(i64 (sext_inreg i64:$a, i32)),
+ (IADD_I64_FROM32 MOD_SX, (EXTRACT_SUBREG $a, sub0), 0, (i32 0), 0)>;
+
+def : Pat<(i16 (trunc i32:$a)), (EXTRACT_SUBREG $a, lo16)>;
+def : Pat<(i32 (trunc i64:$a)), (EXTRACT_SUBREG $a, sub0)>;
+def : Pat<(i16 (trunc i64:$a)), (EXTRACT_SUBREG (EXTRACT_SUBREG $a, sub0), lo16)>;
+
+// imadd: D = A * B + C.
+multiclass IMAddPseudos<RegisterClass rc, RegisterOperand src,
+ RegisterOperand addend, ValueType vt, ComplexPattern pat,
+ list<SchedReadWrite> sched = [WriteIMul]> {
+ def NAME : AGXALUPseudo<IMADD_g13, (outs rc:$dst),
+ (ins SrcMods:$src0_mods, src:$src0,
+ SrcMods:$src1_mods, src:$src1,
+ SrcMods:$src2_mods, addend:$src2, i32imm:$shift),
+ [], sched>;
+ def : Pat<(vt (mul pat:$a, pat:$b)),
+ (!cast<Instruction>(NAME) 0, $a, 0, $b, 0, (vt 0), 0)>;
+ def : Pat<(vt (add (mul pat:$a, pat:$b), pat:$c)),
+ (!cast<Instruction>(NAME) 0, $a, 0, $b, 0, $c, 0)>;
+}
+
+defm IMADD_I16 : IMAddPseudos<GPR16, Src16Op, Src16Op, i16, AGXIntSrc16>;
+defm IMADD_I32 : IMAddPseudos<GPR32, Src32Op, Src32Op, i32, AGXIntSrc32>;
+
+// 32 x 32 -> 64-bit multiply-add.
+def IMADD_I64_FROM32 : AGXALUPseudo<IMADD_g13, (outs GPR64:$dst),
+ (ins SrcMods:$src0_mods, Src32Op:$src0,
+ SrcMods:$src1_mods, Src32Op:$src1,
+ SrcMods:$src2_mods, Src64Op:$src2,
+ i32imm:$shift), [], [WriteALU64]>;
+
+def : Pat<(i64 (mul (zext i32:$a), (zext i32:$b))),
+ (IMADD_I64_FROM32 0, $a, 0, $b, 0, (i64 0), 0)>;
+def : Pat<(i64 (mul (sext i32:$a), (sext i32:$b))),
+ (IMADD_I64_FROM32 MOD_SX, $a, MOD_SX, $b, 0, (i64 0), 0)>;
+def : Pat<(i32 (mulhu i32:$a, i32:$b)),
+ (EXTRACT_SUBREG (IMADD_I64_FROM32 0, $a, 0, $b, 0, (i64 0), 0), sub1)>;
+def : Pat<(i32 (mulhs i32:$a, i32:$b)),
+ (EXTRACT_SUBREG (IMADD_I64_FROM32 MOD_SX, $a, MOD_SX, $b, 0,
+ (i64 0), 0), sub1)>;
+
+// 64-bit multiply from 32-bit pieces.
+def : Pat<(i64 (mul i64:$a, i64:$b)),
+ (REG_SEQUENCE GPR64,
+ (EXTRACT_SUBREG
+ (IMADD_I64_FROM32 0, (EXTRACT_SUBREG $a, sub0),
+ 0, (EXTRACT_SUBREG $b, sub0), 0, (i64 0), 0),
+ sub0), sub0,
+ (IMADD_I32 0, (EXTRACT_SUBREG $a, sub0), 0,
+ (EXTRACT_SUBREG $b, sub1), 0,
+ (IMADD_I32 0, (EXTRACT_SUBREG $a, sub1), 0,
+ (EXTRACT_SUBREG $b, sub0), 0,
+ (EXTRACT_SUBREG
+ (IMADD_I64_FROM32 0, (EXTRACT_SUBREG $a, sub0),
+ 0, (EXTRACT_SUBREG $b, sub0), 0, (i64 0),
+ 0), sub1), 0), 0), sub1)>;
+
+//===----------------------------------------------------------------------===//
+// Shifts and bit fields
+//===----------------------------------------------------------------------===//
+
+multiclass BitfieldPseudo<Instruction real, RegisterClass rc,
+ RegisterOperand src> {
+ def NAME : AGXALUPseudo<real, (outs rc:$dst),
+ (ins SrcMods:$src0_mods, src:$src0,
+ SrcMods:$src1_mods, src:$src1,
+ SrcMods:$src2_mods, src:$src2, i32imm:$mask),
+ [], [WriteIMul]>;
+}
+
+defm BFI_I16 : BitfieldPseudo<BFI_g13, GPR16, Src16Op>;
+defm BFI_I32 : BitfieldPseudo<BFI_g13, GPR32, Src32Op>;
+defm BFEIL_I16 : BitfieldPseudo<BFEIL_g13, GPR16, Src16Op>;
+defm BFEIL_I32 : BitfieldPseudo<BFEIL_g13, GPR32, Src32Op>;
+defm EXTR_I32 : BitfieldPseudo<EXTR_g13, GPR32, Src32Op>;
+
+def ASR_I16 : AGXALUPseudo<ASR_g13, (outs GPR16:$dst),
+ (ins SrcMods:$src0_mods, Src16Op:$src0,
+ SrcMods:$src1_mods, Src16Op:$src1), [],
+ [WriteIMul]>;
+def ASR_I32 : AGXALUPseudo<ASR_g13, (outs GPR32:$dst),
+ (ins SrcMods:$src0_mods, Src32Op:$src0,
+ SrcMods:$src1_mods, Src32Op:$src1), [],
+ [WriteIMul]>;
+
+// shl x, s = bfi 0, x, s (full mask); srl x, s = bfeil 0, x, s.
+def : Pat<(i32 (shl AGXIntSrc32:$a, AGXIntSrc32:$s)),
+ (BFI_I32 0, (i32 0), 0, $a, 0, $s, 0)>;
+def : Pat<(i32 (srl AGXIntSrc32:$a, AGXIntSrc32:$s)),
+ (BFEIL_I32 0, (i32 0), 0, $a, 0, $s, 0)>;
+def : Pat<(i32 (sra AGXIntSrc32:$a, AGXIntSrc32:$s)),
+ (ASR_I32 0, $a, 0, $s)>;
+def : Pat<(i16 (shl AGXIntSrc16:$a, AGXIntSrc16:$s)),
+ (BFI_I16 0, (i16 0), 0, $a, 0, $s, 0)>;
+def : Pat<(i16 (srl AGXIntSrc16:$a, AGXIntSrc16:$s)),
+ (BFEIL_I16 0, (i16 0), 0, $a, 0, $s, 0)>;
+def : Pat<(i16 (sra AGXIntSrc16:$a, AGXIntSrc16:$s)),
+ (ASR_I16 0, $a, 0, $s)>;
+
+// Rotates: extr reads the register pair (B:A).
+def : Pat<(i32 (rotr i32:$a, AGXIntSrc32:$s)),
+ (EXTR_I32 0, $a, 0, $a, 0, $s, 0)>;
+def : Pat<(i32 (fshr i32:$hi, i32:$lo, AGXIntSrc32:$s)),
+ (EXTR_I32 0, $lo, 0, $hi, 0, $s, 0)>;
+
+def : Pat<(i32 (sext_inreg i32:$a, i8)),
+ (ASR_I32 0, (BFI_I32 0, (i32 0), 0, $a, 0, (i32 24), 0), 0, (i32 24))>;
+def : Pat<(i16 (sext_inreg i16:$a, i8)),
+ (ASR_I16 0, (BFI_I16 0, (i16 0), 0, $a, 0, (i16 8), 0), 0, (i16 8))>;
+def : Pat<(i32 (sext_inreg i32:$a, i1)),
+ (ISUB_I32 0, (i32 0), 0, (BITOP_I32 TT_AND, 0, $a, 0, (i32 1)), 0)>;
+def : Pat<(i16 (sext_inreg i16:$a, i1)),
+ (ISUB_I16 0, (i16 0), 0, (BITOP_I16 TT_AND, 0, $a, 0, (i16 1)), 0)>;
+
+// Count leading zeros: 31 - ffs(x); ffs(0) is -1.
+def : Pat<(i32 (ctlz AGXIntSrc32:$a)),
+ (ISUB_I32 0, (i32 31), 0, (FFS_I32 0, $a), 0)>;
+def : Pat<(i32 (cttz AGXIntSrc32:$a)),
+ (ISUB_I32 0, (i32 31), 0, (FFS_I32 0, (BITREV_I32 0, $a)), 0)>;
+def : Pat<(i64 (ctpop i64:$a)),
+ (IADD_I64_FROM32 0, (IADD_I32 0, (POPCOUNT_I32 0, (EXTRACT_SUBREG $a, sub0)),
+ 0, (POPCOUNT_I32 0, (EXTRACT_SUBREG $a, sub1)),
+ 0), 0, (i32 0), 0)>;
+
+//===----------------------------------------------------------------------===//
+// Compare and select
+//===----------------------------------------------------------------------===//
+
+// Integer conditions: ueq 0, ult 1, ugt 2, seq 4, slt 5, sgt 6.
+// Float conditions: eq 0, lt 1, gt 2, ltn 3, gte 5, lte 6, gtn 7.
+defvar IC_UEQ = 0;
+defvar IC_ULT = 1;
+defvar IC_UGT = 2;
+defvar IC_SLT = 5;
+defvar IC_SGT = 6;
+defvar FC_EQ = 0;
+defvar FC_LT = 1;
+defvar FC_GT = 2;
+defvar FC_LTN = 3;
+defvar FC_GTE = 5;
+defvar FC_LTE = 6;
+defvar FC_GTN = 7;
+
+multiclass ICmpselPseudo<RegisterClass rc, RegisterOperand cmp,
+ RegisterOperand sel> {
+ def NAME : AGXALUPseudo<ICMPSEL_g13, (outs rc:$dst),
+ (ins i32imm:$cc, SrcMods:$src0_mods, cmp:$src0,
+ SrcMods:$src1_mods, cmp:$src1,
+ SrcMods:$src2_mods, sel:$src2,
+ SrcMods:$src3_mods, sel:$src3), [], [WriteFP]>;
+}
+
+multiclass FCmpselPseudo<RegisterClass rc, RegisterOperand cmp,
+ RegisterOperand sel> {
+ def NAME : AGXALUPseudo<FCMPSEL_g13, (outs rc:$dst),
+ (ins i32imm:$cc, SrcMods:$src0_mods, cmp:$src0,
+ SrcMods:$src1_mods, cmp:$src1,
+ SrcMods:$src2_mods, sel:$src2,
+ SrcMods:$src3_mods, sel:$src3), [], [WriteFP]>;
+}
+
+// Naming: <compare type>_<result type>.
+defm ICMPSEL_I32 : ICmpselPseudo<GPR32, Src32Op, Src32Op>;
+defm ICMPSEL_I16 : ICmpselPseudo<GPR16, Src16Op, Src16Op>;
+defm ICMPSEL_I16_I32 : ICmpselPseudo<GPR32, Src16Op, Src32Op>;
+defm ICMPSEL_I32_I16 : ICmpselPseudo<GPR16, Src32Op, Src16Op>;
+defm FCMPSEL_F32 : FCmpselPseudo<GPR32, Src32Op, Src32Op>;
+defm FCMPSEL_F16 : FCmpselPseudo<GPR16, Src16Op, Src16Op>;
+defm FCMPSEL_F16_I32 : FCmpselPseudo<GPR32, Src16Op, Src32Op>;
+defm FCMPSEL_F32_I16 : FCmpselPseudo<GPR16, Src32Op, Src16Op>;
+
+// setcc: (cond, hardware condition, swap true/false).
+class AGXCond<CondCode cond, int hw, bit swap> {
+ CondCode CC = cond;
+ int HW = hw;
+ bit Swap = swap;
+}
+
+defvar AGXIntConds = [AGXCond<SETEQ, IC_UEQ, 0>, AGXCond<SETNE, IC_UEQ, 1>,
+ AGXCond<SETULT, IC_ULT, 0>, AGXCond<SETUGE, IC_ULT, 1>,
+ AGXCond<SETUGT, IC_UGT, 0>, AGXCond<SETULE, IC_UGT, 1>,
+ AGXCond<SETLT, IC_SLT, 0>, AGXCond<SETGE, IC_SLT, 1>,
+ AGXCond<SETGT, IC_SGT, 0>, AGXCond<SETLE, IC_SGT, 1>];
+
+// Ordered conditions select their predicate; unordered conditions select the
+// inverse ordered predicate with swapped results.
+defvar AGXFPConds = [AGXCond<SETOEQ, FC_EQ, 0>, AGXCond<SETUNE, FC_EQ, 1>,
+ AGXCond<SETOLT, FC_LT, 0>, AGXCond<SETUGE, FC_LT, 1>,
+ AGXCond<SETOGT, FC_GT, 0>, AGXCond<SETULE, FC_GT, 1>,
+ AGXCond<SETOGE, FC_GTE, 0>, AGXCond<SETULT, FC_GTE, 1>,
+ AGXCond<SETOLE, FC_LTE, 0>, AGXCond<SETUGT, FC_LTE, 1>,
+ AGXCond<SETEQ, FC_EQ, 0>, AGXCond<SETNE, FC_EQ, 1>,
+ AGXCond<SETLT, FC_LT, 0>, AGXCond<SETGE, FC_GTE, 0>,
+ AGXCond<SETGT, FC_GT, 0>, AGXCond<SETLE, FC_LTE, 0>];
+
+multiclass SetCCPats<Instruction inst, ValueType cvt, ValueType rvt,
+ ComplexPattern cpat, ComplexPattern spat,
+ list<AGXCond> conds, bit fp> {
+ foreach C = conds in {
+ if fp then {
+ def : Pat<(rvt (setcc (cpat cvt:$a, i32:$am), (cpat cvt:$b, i32:$bm),
+ C.CC)),
+ !if(C.Swap, (inst C.HW, $am, $a, $bm, $b, 0, (rvt 0), 0, (rvt 1)),
+ (inst C.HW, $am, $a, $bm, $b, 0, (rvt 1), 0, (rvt 0)))>;
+ def : Pat<(rvt (select (i32 (setcc (cpat cvt:$a, i32:$am),
+ (cpat cvt:$b, i32:$bm), C.CC)),
+ spat:$x, spat:$y)),
+ !if(C.Swap, (inst C.HW, $am, $a, $bm, $b, 0, $y, 0, $x),
+ (inst C.HW, $am, $a, $bm, $b, 0, $x, 0, $y))>;
+ } else {
+ def : Pat<(rvt (setcc cpat:$a, cpat:$b, C.CC)),
+ !if(C.Swap, (inst C.HW, 0, $a, 0, $b, 0, (rvt 0), 0, (rvt 1)),
+ (inst C.HW, 0, $a, 0, $b, 0, (rvt 1), 0, (rvt 0)))>;
+ def : Pat<(rvt (select (i32 (setcc cpat:$a, cpat:$b, C.CC)),
+ spat:$x, spat:$y)),
+ !if(C.Swap, (inst C.HW, 0, $a, 0, $b, 0, $y, 0, $x),
+ (inst C.HW, 0, $a, 0, $b, 0, $x, 0, $y))>;
+ }
+ }
+}
+
+defm : SetCCPats<ICMPSEL_I32, i32, i32, AGXIntSrc32, AGXIntSrc32,
+ AGXIntConds, 0>;
+defm : SetCCPats<ICMPSEL_I16_I32, i16, i32, AGXIntSrc16, AGXIntSrc32,
+ AGXIntConds, 0>;
+defm : SetCCPats<FCMPSEL_F32, f32, i32, AGXFPSrc32, AGXIntSrc32,
+ AGXFPConds, 1>;
+defm : SetCCPats<FCMPSEL_F16_I32, f16, i32, AGXFPSrc16, AGXIntSrc32,
+ AGXFPConds, 1>;
+
+// Select on a boolean (0/1 in a 32-bit register).
+def : Pat<(i32 (select i32:$c, AGXIntSrc32:$x, AGXIntSrc32:$y)),
+ (ICMPSEL_I32 IC_UEQ, 0, $c, 0, (i32 0), 0, $y, 0, $x)>;
+def : Pat<(i16 (select i32:$c, AGXIntSrc16:$x, AGXIntSrc16:$y)),
+ (ICMPSEL_I32_I16 IC_UEQ, 0, $c, 0, (i32 0), 0, $y, 0, $x)>;
+def : Pat<(f32 (select i32:$c, f32:$x, f32:$y)),
+ (ICMPSEL_I32 IC_UEQ, 0, $c, 0, (i32 0), 0, $y, 0, $x)>;
+foreach VT = [f16, bf16] in
+def : Pat<(VT (select i32:$c, VT:$x, VT:$y)),
+ (ICMPSEL_I32_I16 IC_UEQ, 0, $c, 0, (i32 0), 0, $y, 0, $x)>;
+
+def : Pat<(i64 (select i32:$c, i64:$x, i64:$y)),
+ (REG_SEQUENCE GPR64,
+ (ICMPSEL_I32 IC_UEQ, 0, $c, 0, (i32 0), 0, (EXTRACT_SUBREG $y, sub0),
+ 0, (EXTRACT_SUBREG $x, sub0)), sub0,
+ (ICMPSEL_I32 IC_UEQ, 0, $c, 0, (i32 0), 0, (EXTRACT_SUBREG $y, sub1),
+ 0, (EXTRACT_SUBREG $x, sub1)), sub1)>;
+
+// Integer min/max.
+def : Pat<(i32 (smin i32:$a, i32:$b)),
+ (ICMPSEL_I32 IC_SLT, 0, $a, 0, $b, 0, $a, 0, $b)>;
+def : Pat<(i32 (smax i32:$a, i32:$b)),
+ (ICMPSEL_I32 IC_SGT, 0, $a, 0, $b, 0, $a, 0, $b)>;
+def : Pat<(i32 (umin i32:$a, i32:$b)),
+ (ICMPSEL_I32 IC_ULT, 0, $a, 0, $b, 0, $a, 0, $b)>;
+def : Pat<(i32 (umax i32:$a, i32:$b)),
+ (ICMPSEL_I32 IC_UGT, 0, $a, 0, $b, 0, $a, 0, $b)>;
+def : Pat<(i16 (smin i16:$a, i16:$b)),
+ (ICMPSEL_I16 IC_SLT, 0, $a, 0, $b, 0, $a, 0, $b)>;
+def : Pat<(i16 (smax i16:$a, i16:$b)),
+ (ICMPSEL_I16 IC_SGT, 0, $a, 0, $b, 0, $a, 0, $b)>;
+def : Pat<(i16 (umin i16:$a, i16:$b)),
+ (ICMPSEL_I16 IC_ULT, 0, $a, 0, $b, 0, $a, 0, $b)>;
+def : Pat<(i16 (umax i16:$a, i16:$b)),
+ (ICMPSEL_I16 IC_UGT, 0, $a, 0, $b, 0, $a, 0, $b)>;
+
+// Float min/max: the "n" conditions return the other operand for a NaN.
+def : Pat<(f32 (fminnum f32:$a, f32:$b)),
+ (FCMPSEL_F32 FC_LTN, 0, $a, 0, $b, 0, $a, 0, $b)>;
+def : Pat<(f32 (fmaxnum f32:$a, f32:$b)),
+ (FCMPSEL_F32 FC_GTN, 0, $a, 0, $b, 0, $a, 0, $b)>;
+def : Pat<(f16 (fminnum f16:$a, f16:$b)),
+ (FCMPSEL_F16 FC_LTN, 0, $a, 0, $b, 0, $a, 0, $b)>;
+def : Pat<(f16 (fmaxnum f16:$a, f16:$b)),
+ (FCMPSEL_F16 FC_GTN, 0, $a, 0, $b, 0, $a, 0, $b)>;
+
+//===----------------------------------------------------------------------===//
+// Floating point arithmetic
+//===----------------------------------------------------------------------===//
+
+multiclass FBinPseudo<Instruction real, RegisterClass rc, RegisterOperand src,
+ list<SchedReadWrite> sched = [WriteFP]> {
+ def NAME : AGXALUPseudo<real, (outs rc:$dst),
+ (ins SrcMods:$src0_mods, src:$src0,
+ SrcMods:$src1_mods, src:$src1), [], sched>;
+}
+
+defm FADD32_F32 : FBinPseudo<FADD32_g13, GPR32, Src32Op>;
+defm FMUL32_F32 : FBinPseudo<FMUL32_g13, GPR32, Src32Op>;
+defm FADD16_F16 : FBinPseudo<FADD16_g13, GPR16, Src16Op>;
+defm FMUL16_F16 : FBinPseudo<FMUL16_g13, GPR16, Src16Op>;
+defm FADD32_SAT_F32 : FBinPseudo<FADD32_SAT_g13, GPR32, Src32Op>;
+defm FADD16_SAT_F16 : FBinPseudo<FADD16_SAT_g13, GPR16, Src16Op>;
+// Conversions between half and single precision through fadd32 -0.0.
+def FADD32_F32_F16 : AGXALUPseudo<FADD32_g13, (outs GPR32:$dst),
+ (ins SrcMods:$src0_mods, Src16Op:$src0,
+ SrcMods:$src1_mods, Src16Op:$src1), [],
+ [WriteFP]>;
+def FADD32_F16_F32 : AGXALUPseudo<FADD32_g13, (outs GPR16:$dst),
+ (ins SrcMods:$src0_mods, Src32Op:$src0,
+ SrcMods:$src1_mods, Src32Op:$src1), [],
+ [WriteFP]>;
+
+def : Pat<(f32 (fadd (AGXFPSrc32 f32:$a, i32:$am), (AGXFPSrc32 f32:$b, i32:$bm))),
+ (FADD32_F32 $am, $a, $bm, $b)>;
+def : Pat<(f32 (fmul (AGXFPSrc32 f32:$a, i32:$am), (AGXFPSrc32 f32:$b, i32:$bm))),
+ (FMUL32_F32 $am, $a, $bm, $b)>;
+def : Pat<(f16 (fadd (AGXFPSrc16 f16:$a, i32:$am), (AGXFPSrc16 f16:$b, i32:$bm))),
+ (FADD16_F16 $am, $a, $bm, $b)>;
+def : Pat<(f16 (fmul (AGXFPSrc16 f16:$a, i32:$am), (AGXFPSrc16 f16:$b, i32:$bm))),
+ (FMUL16_F16 $am, $a, $bm, $b)>;
+
+def : Pat<(f32 (fneg f32:$a)),
+ (FADD32_F32 MOD_NEG, $a, 0, (i32 FPIMM_NEG_ZERO))>;
+def : Pat<(f32 (fabs f32:$a)),
+ (FADD32_F32 MOD_ABS, $a, 0, (i32 FPIMM_NEG_ZERO))>;
+def : Pat<(f16 (fneg f16:$a)),
+ (FADD16_F16 MOD_NEG, $a, 0, (i16 FPIMM_NEG_ZERO))>;
+def : Pat<(f16 (fabs f16:$a)),
+ (FADD16_F16 MOD_ABS, $a, 0, (i16 FPIMM_NEG_ZERO))>;
+def : Pat<(f32 (fcanonicalize (AGXFPSrc32 f32:$a, i32:$am))),
+ (FADD32_F32 $am, $a, 0, (i32 FPIMM_NEG_ZERO))>;
+
+// Clamp to [0, 1].
+def agx_fpimm0 : FPImmLeaf<fAny, [{ return Imm.isExactlyValue(+0.0); }]>;
+def agx_fpimm1 : FPImmLeaf<fAny, [{ return Imm.isExactlyValue(+1.0); }]>;
+
+def : Pat<(f32 (fminnum (fmaxnum (AGXFPSrc32 f32:$a, i32:$am), (f32 agx_fpimm0)),
+ (f32 agx_fpimm1))),
+ (FADD32_SAT_F32 $am, $a, 0, (i32 FPIMM_NEG_ZERO))>;
+
+def : Pat<(f32 (fpextend (AGXFPSrc16 f16:$a, i32:$am))),
+ (FADD32_F32_F16 $am, $a, 0, (i16 FPIMM_NEG_ZERO))>;
+def : Pat<(f16 (fpround (AGXFPSrc32 f32:$a, i32:$am))),
+ (FADD32_F16_F32 $am, $a, 0, (i32 FPIMM_NEG_ZERO))>;
+
+multiclass FMAPseudo<Instruction real, RegisterClass rc, RegisterOperand src> {
+ def NAME : AGXALUPseudo<real, (outs rc:$dst),
+ (ins SrcMods:$src0_mods, src:$src0,
+ SrcMods:$src1_mods, src:$src1,
+ SrcMods:$src2_mods, src:$src2), [], [WriteFP]>;
+}
+
+defm FMADD32_F32 : FMAPseudo<FMADD32_g13, GPR32, Src32Op>;
+defm FMADD16_F16 : FMAPseudo<FMADD16_g13, GPR16, Src16Op>;
+
+def : Pat<(f32 (fma (AGXFPSrc32 f32:$a, i32:$am), (AGXFPSrc32 f32:$b, i32:$bm),
+ (AGXFPSrc32 f32:$c, i32:$cm))),
+ (FMADD32_F32 $am, $a, $bm, $b, $cm, $c)>;
+def : Pat<(f16 (fma (AGXFPSrc16 f16:$a, i32:$am), (AGXFPSrc16 f16:$b, i32:$bm),
+ (AGXFPSrc16 f16:$c, i32:$cm))),
+ (FMADD16_F16 $am, $a, $bm, $b, $cm, $c)>;
+
+multiclass FUnaryPseudo<Instruction real, SDPatternOperator node,
+ list<SchedReadWrite> sched = [WriteFRound]> {
+ def _F32 : AGXALUPseudo<real, (outs GPR32:$dst),
+ (ins SrcMods:$src0_mods, Src32Op:$src0), [], sched>;
+ def _F16 : AGXALUPseudo<real, (outs GPR16:$dst),
+ (ins SrcMods:$src0_mods, Src16Op:$src0), [], sched>;
+ def : Pat<(f32 (node (AGXFPSrc32 f32:$a, i32:$am))),
+ (!cast<Instruction>(NAME#"_F32") $am, $a)>;
+ def : Pat<(f16 (node (AGXFPSrc16 f16:$a, i32:$am))),
+ (!cast<Instruction>(NAME#"_F16") $am, $a)>;
+}
+
+defm FLOOR : FUnaryPseudo<FLOOR_g13, ffloor>;
+defm CEIL : FUnaryPseudo<CEIL_g13, fceil>;
+defm TRUNC : FUnaryPseudo<TRUNC_g13, ftrunc>;
+defm RINT : FUnaryPseudo<RINT_g13, froundeven>;
+defm RCP : FUnaryPseudo<RCP_g13, int_agx_rcp, [WriteSFU]>;
+defm RSQRT : FUnaryPseudo<RSQRT_g13, int_agx_rsqrt, [WriteSFUSlow]>;
+defm RSQRT_SPECIAL : FUnaryPseudo<RSQRT_SPECIAL_g13, int_agx_rsqrt_special,
+ [WriteSFUSlow]>;
+defm LOG2 : FUnaryPseudo<LOG2_g13, flog2, [WriteSFU]>;
+defm EXP2 : FUnaryPseudo<EXP2_g13, fexp2, [WriteSFU]>;
+defm SIN_PT_1 : FUnaryPseudo<SIN_PT_1_g13, int_agx_sin_pt_1>;
+defm SIN_PT_2 : FUnaryPseudo<SIN_PT_2_g13, int_agx_sin_pt_2, [WriteSFU]>;
+defm DFDX : FUnaryPseudo<DFDX_g13, int_agx_dfdx>;
+defm DFDY : FUnaryPseudo<DFDY_g13, int_agx_dfdy>;
+
+def : Pat<(f32 (frint (AGXFPSrc32 f32:$a, i32:$am))), (RINT_F32 $am, $a)>;
+def : Pat<(f32 (fnearbyint (AGXFPSrc32 f32:$a, i32:$am))), (RINT_F32 $am, $a)>;
+def : Pat<(f16 (frint (AGXFPSrc16 f16:$a, i32:$am))), (RINT_F16 $am, $a)>;
+def : Pat<(f16 (fnearbyint (AGXFPSrc16 f16:$a, i32:$am))), (RINT_F16 $am, $a)>;
+
+// sqrt(x) = x * rsqrt_special(x).
+def : Pat<(f32 (fsqrt f32:$a)),
+ (FMUL32_F32 0, $a, 0, (RSQRT_SPECIAL_F32 0, $a))>;
+def : Pat<(f16 (fsqrt f16:$a)),
+ (FMUL16_F16 0, $a, 0, (RSQRT_SPECIAL_F16 0, $a))>;
+
+def : Pat<(f32 (AGXsin_quadrants f32:$q)),
+ (FMUL32_F32 0, (SIN_PT_2_F32 0, (SIN_PT_1_F32 0, $q)), 0,
+ (SIN_PT_1_F32 0, $q))>;
+
+//===----------------------------------------------------------------------===//
+// Conversions
+//===----------------------------------------------------------------------===//
+
+// Modes of the convert instruction.
+defvar CVT_U8_TO_F = 0;
+defvar CVT_S8_TO_F = 1;
+defvar CVT_F_TO_U16 = 4;
+defvar CVT_F_TO_S16 = 5;
+defvar CVT_U16_TO_F = 6;
+defvar CVT_S16_TO_F = 7;
+defvar CVT_F_TO_U32 = 8;
+defvar CVT_F_TO_S32 = 9;
+defvar CVT_U32_TO_F = 10;
+defvar CVT_S32_TO_F = 11;
+defvar ROUND_RTZ = 0;
+defvar ROUND_RTE = 1;
+
+multiclass ConvertPseudo<RegisterClass rc, RegisterOperand src> {
+ def NAME : AGXALUPseudo<CONVERT_g13, (outs rc:$dst),
+ (ins i32imm:$mode, SrcMods:$src0_mods, src:$src0,
+ i32imm:$round), [], [WriteIMul]>;
+}
+
+// Naming: CONVERT_<dst bits>_<src bits>.
+defm CONVERT_32_32 : ConvertPseudo<GPR32, Src32Op>;
+defm CONVERT_32_16 : ConvertPseudo<GPR32, Src16Op>;
+defm CONVERT_16_32 : ConvertPseudo<GPR16, Src32Op>;
+defm CONVERT_16_16 : ConvertPseudo<GPR16, Src16Op>;
+
+def : Pat<(i32 (fp_to_sint f32:$a)), (CONVERT_32_32 CVT_F_TO_S32, 0, $a, ROUND_RTZ)>;
+def : Pat<(i32 (fp_to_uint f32:$a)), (CONVERT_32_32 CVT_F_TO_U32, 0, $a, ROUND_RTZ)>;
+def : Pat<(i32 (fp_to_sint f16:$a)), (CONVERT_32_16 CVT_F_TO_S32, 0, $a, ROUND_RTZ)>;
+def : Pat<(i32 (fp_to_uint f16:$a)), (CONVERT_32_16 CVT_F_TO_U32, 0, $a, ROUND_RTZ)>;
+def : Pat<(i16 (fp_to_sint f32:$a)), (CONVERT_16_32 CVT_F_TO_S16, 0, $a, ROUND_RTZ)>;
+def : Pat<(i16 (fp_to_uint f32:$a)), (CONVERT_16_32 CVT_F_TO_U16, 0, $a, ROUND_RTZ)>;
+def : Pat<(i16 (fp_to_sint f16:$a)), (CONVERT_16_16 CVT_F_TO_S16, 0, $a, ROUND_RTZ)>;
+def : Pat<(i16 (fp_to_uint f16:$a)), (CONVERT_16_16 CVT_F_TO_U16, 0, $a, ROUND_RTZ)>;
+def : Pat<(f32 (sint_to_fp i32:$a)), (CONVERT_32_32 CVT_S32_TO_F, 0, $a, ROUND_RTE)>;
+def : Pat<(f32 (uint_to_fp i32:$a)), (CONVERT_32_32 CVT_U32_TO_F, 0, $a, ROUND_RTE)>;
+def : Pat<(f32 (sint_to_fp i16:$a)), (CONVERT_32_16 CVT_S16_TO_F, 0, $a, ROUND_RTE)>;
+def : Pat<(f32 (uint_to_fp i16:$a)), (CONVERT_32_16 CVT_U16_TO_F, 0, $a, ROUND_RTE)>;
+def : Pat<(f16 (sint_to_fp i32:$a)), (CONVERT_16_32 CVT_S32_TO_F, 0, $a, ROUND_RTE)>;
+def : Pat<(f16 (uint_to_fp i32:$a)), (CONVERT_16_32 CVT_U32_TO_F, 0, $a, ROUND_RTE)>;
+def : Pat<(f16 (sint_to_fp i16:$a)), (CONVERT_16_16 CVT_S16_TO_F, 0, $a, ROUND_RTE)>;
+def : Pat<(f16 (uint_to_fp i16:$a)), (CONVERT_16_16 CVT_U16_TO_F, 0, $a, ROUND_RTE)>;
+
+// Bit casts between same sized types are free.
+def : Pat<(i32 (bitconvert f32:$a)), (COPY_TO_REGCLASS $a, GPR32)>;
+def : Pat<(f32 (bitconvert i32:$a)), (COPY_TO_REGCLASS $a, GPR32)>;
+def : Pat<(i16 (bitconvert f16:$a)), (COPY_TO_REGCLASS $a, GPR16)>;
+def : Pat<(f16 (bitconvert i16:$a)), (COPY_TO_REGCLASS $a, GPR16)>;
+def : Pat<(i16 (bitconvert bf16:$a)), (COPY_TO_REGCLASS $a, GPR16)>;
+def : Pat<(bf16 (bitconvert i16:$a)), (COPY_TO_REGCLASS $a, GPR16)>;
+def : Pat<(i64 (bitconvert f64:$a)), (COPY_TO_REGCLASS $a, GPR64)>;
+def : Pat<(f64 (bitconvert i64:$a)), (COPY_TO_REGCLASS $a, GPR64)>;
+
+//===----------------------------------------------------------------------===//
+// Device memory
+//===----------------------------------------------------------------------===//
+
+class DeviceLoadPseudo<RegisterClass rc>
+ : AGXPseudo<(outs rc:$R),
+ (ins i32imm:$g, i32imm:$fmt, i32imm:$mask, Base64Op:$A,
+ IndexOp:$O, i32imm:$sgn, i32imm:$shift)> {
+ let G13Real = DEVICE_LOAD_g13;
+ let mayLoad = 1;
+ let IsAsync = 1;
+ let IsMemory = 1;
+ let SchedRW = [WriteDeviceLoad];
+}
+
+class DeviceStorePseudo<RegisterClass rc>
+ : AGXPseudo<(outs),
+ (ins i32imm:$g, i32imm:$fmt, i32imm:$mask, rc:$R,
+ Base64Op:$A, IndexOp:$O, i32imm:$sgn, i32imm:$shift)> {
+ let G13Real = DEVICE_STORE_g13;
+ let mayStore = 1;
+ let IsMemory = 1;
+ let SchedRW = [WriteDeviceStore];
+}
+
+def DEVICE_LOAD_16 : DeviceLoadPseudo<GPR16>;
+def DEVICE_LOAD_32 : DeviceLoadPseudo<GPR32>;
+def DEVICE_LOAD_64 : DeviceLoadPseudo<GPR32x2>;
+def DEVICE_LOAD_96 : DeviceLoadPseudo<GPR96>;
+def DEVICE_LOAD_128 : DeviceLoadPseudo<GPR128>;
+def DEVICE_STORE_16 : DeviceStorePseudo<GPR16>;
+def DEVICE_STORE_32 : DeviceStorePseudo<GPR32>;
+def DEVICE_STORE_64 : DeviceStorePseudo<GPR32x2>;
+def DEVICE_STORE_96 : DeviceStorePseudo<GPR96>;
+def DEVICE_STORE_128 : DeviceStorePseudo<GPR128>;
+
+multiclass DeviceLoadPat<PatFrag frag, ValueType vt, Instruction inst,
+ ComplexPattern addr, int fmt, int mask> {
+ def : Pat<(vt (frag (addr i64:$base, i32:$idx, i32:$sgn, i32:$shift))),
+ (inst 0, fmt, mask, $base, $idx, $sgn, $shift)>;
+}
+
+multiclass DeviceStorePat<PatFrag frag, ValueType vt, Instruction inst,
+ ComplexPattern addr, int fmt, int mask> {
+ def : Pat<(frag vt:$val, (addr i64:$base, i32:$idx, i32:$sgn, i32:$shift)),
+ (inst 0, fmt, mask, $val, $base, $idx, $sgn, $shift)>;
+}
+
+defm : DeviceLoadPat<device_load, i32, DEVICE_LOAD_32, AGXDeviceAddr32, FMT_I32, 1>;
+defm : DeviceLoadPat<device_load, f32, DEVICE_LOAD_32, AGXDeviceAddr32, FMT_I32, 1>;
+defm : DeviceLoadPat<device_load, i64, DEVICE_LOAD_64, AGXDeviceAddr32, FMT_I32, 3>;
+defm : DeviceLoadPat<device_load, f64, DEVICE_LOAD_64, AGXDeviceAddr32, FMT_I32, 3>;
+defm : DeviceLoadPat<device_load, i16, DEVICE_LOAD_16, AGXDeviceAddr16, FMT_I16, 1>;
+defm : DeviceLoadPat<device_load, f16, DEVICE_LOAD_16, AGXDeviceAddr16, FMT_I16, 1>;
+defm : DeviceLoadPat<device_load, bf16, DEVICE_LOAD_16, AGXDeviceAddr16, FMT_I16, 1>;
+defm : DeviceLoadPat<device_extloadi8, i16, DEVICE_LOAD_16, AGXDeviceAddr8, FMT_I8, 1>;
+defm : DeviceLoadPat<device_zextloadi8, i16, DEVICE_LOAD_16, AGXDeviceAddr8, FMT_I8, 1>;
+defm : DeviceLoadPat<device_extloadi8, i32, DEVICE_LOAD_32, AGXDeviceAddr8, FMT_I8, 1>;
+defm : DeviceLoadPat<device_zextloadi8, i32, DEVICE_LOAD_32, AGXDeviceAddr8, FMT_I8, 1>;
+defm : DeviceLoadPat<device_extloadi16, i32, DEVICE_LOAD_32, AGXDeviceAddr16, FMT_I16, 1>;
+defm : DeviceLoadPat<device_zextloadi16, i32, DEVICE_LOAD_32, AGXDeviceAddr16, FMT_I16, 1>;
+
+defm : DeviceStorePat<device_store, i32, DEVICE_STORE_32, AGXDeviceAddr32, FMT_I32, 1>;
+defm : DeviceStorePat<device_store, f32, DEVICE_STORE_32, AGXDeviceAddr32, FMT_I32, 1>;
+defm : DeviceStorePat<device_store, i64, DEVICE_STORE_64, AGXDeviceAddr32, FMT_I32, 3>;
+defm : DeviceStorePat<device_store, f64, DEVICE_STORE_64, AGXDeviceAddr32, FMT_I32, 3>;
+defm : DeviceStorePat<device_store, i16, DEVICE_STORE_16, AGXDeviceAddr16, FMT_I16, 1>;
+defm : DeviceStorePat<device_store, f16, DEVICE_STORE_16, AGXDeviceAddr16, FMT_I16, 1>;
+defm : DeviceStorePat<device_store, bf16, DEVICE_STORE_16, AGXDeviceAddr16, FMT_I16, 1>;
+defm : DeviceStorePat<device_truncstorei8, i16, DEVICE_STORE_16, AGXDeviceAddr8, FMT_I8, 1>;
+defm : DeviceStorePat<device_truncstorei8, i32, DEVICE_STORE_32, AGXDeviceAddr8, FMT_I8, 1>;
+defm : DeviceStorePat<device_truncstorei16, i32, DEVICE_STORE_32, AGXDeviceAddr16, FMT_I16, 1>;
+
+//===----------------------------------------------------------------------===//
+// Threadgroup memory
+//===----------------------------------------------------------------------===//
+
+class LocalLoadPseudo<RegisterClass rc>
+ : AGXPseudo<(outs rc:$R),
+ (ins i32imm:$fmt, i32imm:$mask, LocalBaseOp:$A,
+ LocalIndexOp:$O)> {
+ let G13Real = THREADGROUP_LOAD_g13;
+ let mayLoad = 1;
+ let IsMemory = 1;
+ let SchedRW = [WriteLocalLoad];
+}
+
+class LocalStorePseudo<RegisterClass rc>
+ : AGXPseudo<(outs),
+ (ins i32imm:$fmt, i32imm:$mask, rc:$R, LocalBaseOp:$A,
+ LocalIndexOp:$O)> {
+ let G13Real = THREADGROUP_STORE_g13;
+ let mayStore = 1;
+ let IsMemory = 1;
+ let SchedRW = [WriteLocalStore];
+}
+
+def LOCAL_LOAD_16 : LocalLoadPseudo<GPR16>;
+def LOCAL_LOAD_32 : LocalLoadPseudo<GPR32>;
+def LOCAL_LOAD_64 : LocalLoadPseudo<GPR32x2>;
+def LOCAL_STORE_16 : LocalStorePseudo<GPR16>;
+def LOCAL_STORE_32 : LocalStorePseudo<GPR32>;
+def LOCAL_STORE_64 : LocalStorePseudo<GPR32x2>;
+
+multiclass LocalLoadPat<PatFrag frag, ValueType vt, Instruction inst,
+ ComplexPattern addr, int fmt, int mask> {
+ def : Pat<(vt (frag (addr i16:$base, i16:$idx))),
+ (inst fmt, mask, $base, $idx)>;
+}
+
+multiclass LocalStorePat<PatFrag frag, ValueType vt, Instruction inst,
+ ComplexPattern addr, int fmt, int mask> {
+ def : Pat<(frag vt:$val, (addr i16:$base, i16:$idx)),
+ (inst fmt, mask, $val, $base, $idx)>;
+}
+
+defm : LocalLoadPat<local_load, i32, LOCAL_LOAD_32, AGXLocalAddr32, FMT_I32, 1>;
+defm : LocalLoadPat<local_load, f32, LOCAL_LOAD_32, AGXLocalAddr32, FMT_I32, 1>;
+defm : LocalLoadPat<local_load, i64, LOCAL_LOAD_64, AGXLocalAddr32, FMT_I32, 3>;
+defm : LocalLoadPat<local_load, f64, LOCAL_LOAD_64, AGXLocalAddr32, FMT_I32, 3>;
+defm : LocalLoadPat<local_load, i16, LOCAL_LOAD_16, AGXLocalAddr16, FMT_I16, 1>;
+defm : LocalLoadPat<local_load, f16, LOCAL_LOAD_16, AGXLocalAddr16, FMT_I16, 1>;
+defm : LocalLoadPat<local_extloadi8, i16, LOCAL_LOAD_16, AGXLocalAddr8, FMT_I8, 1>;
+defm : LocalLoadPat<local_zextloadi8, i16, LOCAL_LOAD_16, AGXLocalAddr8, FMT_I8, 1>;
+defm : LocalLoadPat<local_extloadi8, i32, LOCAL_LOAD_32, AGXLocalAddr8, FMT_I8, 1>;
+defm : LocalLoadPat<local_zextloadi8, i32, LOCAL_LOAD_32, AGXLocalAddr8, FMT_I8, 1>;
+defm : LocalLoadPat<local_extloadi16, i32, LOCAL_LOAD_32, AGXLocalAddr16, FMT_I16, 1>;
+defm : LocalLoadPat<local_zextloadi16, i32, LOCAL_LOAD_32, AGXLocalAddr16, FMT_I16, 1>;
+
+defm : LocalStorePat<local_store, i32, LOCAL_STORE_32, AGXLocalAddr32, FMT_I32, 1>;
+defm : LocalStorePat<local_store, f32, LOCAL_STORE_32, AGXLocalAddr32, FMT_I32, 1>;
+defm : LocalStorePat<local_store, i64, LOCAL_STORE_64, AGXLocalAddr32, FMT_I32, 3>;
+defm : LocalStorePat<local_store, f64, LOCAL_STORE_64, AGXLocalAddr32, FMT_I32, 3>;
+defm : LocalStorePat<local_store, i16, LOCAL_STORE_16, AGXLocalAddr16, FMT_I16, 1>;
+defm : LocalStorePat<local_store, f16, LOCAL_STORE_16, AGXLocalAddr16, FMT_I16, 1>;
+defm : LocalStorePat<local_truncstorei8, i16, LOCAL_STORE_16, AGXLocalAddr8, FMT_I8, 1>;
+defm : LocalStorePat<local_truncstorei8, i32, LOCAL_STORE_32, AGXLocalAddr8, FMT_I8, 1>;
+defm : LocalStorePat<local_truncstorei16, i32, LOCAL_STORE_32, AGXLocalAddr16, FMT_I16, 1>;
+
+//===----------------------------------------------------------------------===//
+// Stack memory
+//===----------------------------------------------------------------------===//
+
+// Byte offset into the thread's stack.
+def StackIndexOp : AGXRegOperand<GPR32, "OPERAND_SRC">;
+
+class StackLoadPseudo<RegisterClass rc>
+ : AGXPseudo<(outs rc:$R),
+ (ins i32imm:$g, i32imm:$fmt, i32imm:$mask, StackIndexOp:$O)> {
+ let G13Real = STACK_LOAD_g13;
+ let mayLoad = 1;
+ let IsAsync = 1;
+ let IsMemory = 1;
+ let SchedRW = [WriteDeviceLoad];
+}
+
+class StackStorePseudo<RegisterClass rc>
+ : AGXPseudo<(outs),
+ (ins i32imm:$g, i32imm:$fmt, i32imm:$mask, rc:$R,
+ StackIndexOp:$O)> {
+ let G13Real = STACK_STORE_g13;
+ let mayStore = 1;
+ let IsMemory = 1;
+ let SchedRW = [WriteDeviceStore];
+}
+
+def STACK_LOAD_16 : StackLoadPseudo<GPR16>;
+def STACK_LOAD_32 : StackLoadPseudo<GPR32>;
+def STACK_LOAD_64 : StackLoadPseudo<GPR32x2>;
+def STACK_LOAD_96 : StackLoadPseudo<GPR96>;
+def STACK_LOAD_128 : StackLoadPseudo<GPR128>;
+def STACK_STORE_16 : StackStorePseudo<GPR16>;
+def STACK_STORE_32 : StackStorePseudo<GPR32>;
+def STACK_STORE_64 : StackStorePseudo<GPR32x2>;
+def STACK_STORE_96 : StackStorePseudo<GPR96>;
+def STACK_STORE_128 : StackStorePseudo<GPR128>;
+
+def : Pat<(i32 (private_load (AGXStackAddr i32:$off))),
+ (STACK_LOAD_32 0, FMT_I32, 1, $off)>;
+def : Pat<(f32 (private_load (AGXStackAddr i32:$off))),
+ (STACK_LOAD_32 0, FMT_I32, 1, $off)>;
+def : Pat<(i64 (private_load (AGXStackAddr i32:$off))),
+ (STACK_LOAD_64 0, FMT_I32, 3, $off)>;
+def : Pat<(i16 (private_load (AGXStackAddr i32:$off))),
+ (STACK_LOAD_16 0, FMT_I16, 1, $off)>;
+def : Pat<(f16 (private_load (AGXStackAddr i32:$off))),
+ (STACK_LOAD_16 0, FMT_I16, 1, $off)>;
+def : Pat<(i16 (private_extloadi8 (AGXStackAddr i32:$off))),
+ (STACK_LOAD_16 0, FMT_I8, 1, $off)>;
+def : Pat<(i16 (private_zextloadi8 (AGXStackAddr i32:$off))),
+ (STACK_LOAD_16 0, FMT_I8, 1, $off)>;
+def : Pat<(i32 (private_extloadi8 (AGXStackAddr i32:$off))),
+ (STACK_LOAD_32 0, FMT_I8, 1, $off)>;
+def : Pat<(i32 (private_zextloadi8 (AGXStackAddr i32:$off))),
+ (STACK_LOAD_32 0, FMT_I8, 1, $off)>;
+def : Pat<(i32 (private_extloadi16 (AGXStackAddr i32:$off))),
+ (STACK_LOAD_32 0, FMT_I16, 1, $off)>;
+def : Pat<(i32 (private_zextloadi16 (AGXStackAddr i32:$off))),
+ (STACK_LOAD_32 0, FMT_I16, 1, $off)>;
+
+def : Pat<(private_store i32:$v, (AGXStackAddr i32:$off)),
+ (STACK_STORE_32 0, FMT_I32, 1, $v, $off)>;
+def : Pat<(private_store f32:$v, (AGXStackAddr i32:$off)),
+ (STACK_STORE_32 0, FMT_I32, 1, $v, $off)>;
+def : Pat<(private_store i64:$v, (AGXStackAddr i32:$off)),
+ (STACK_STORE_64 0, FMT_I32, 3, $v, $off)>;
+def : Pat<(private_store i16:$v, (AGXStackAddr i32:$off)),
+ (STACK_STORE_16 0, FMT_I16, 1, $v, $off)>;
+def : Pat<(private_store f16:$v, (AGXStackAddr i32:$off)),
+ (STACK_STORE_16 0, FMT_I16, 1, $v, $off)>;
+def : Pat<(private_truncstorei8 i16:$v, (AGXStackAddr i32:$off)),
+ (STACK_STORE_16 0, FMT_I8, 1, $v, $off)>;
+def : Pat<(private_truncstorei8 i32:$v, (AGXStackAddr i32:$off)),
+ (STACK_STORE_32 0, FMT_I8, 1, $v, $off)>;
+def : Pat<(private_truncstorei16 i32:$v, (AGXStackAddr i32:$off)),
+ (STACK_STORE_32 0, FMT_I16, 1, $v, $off)>;
+
+// The address of a stack object, as a byte offset.
+let isReMaterializable = 1 in
+def FRAME_ADDR : AGXALUPseudo<MOV_IMM32_g13, (outs GPR32:$dst),
+ (ins i32imm:$imm), [], [WriteMovImm]>;
+def agx_to_tframeindex : SDNodeXForm<frameindex, [{
+ return CurDAG->getTargetFrameIndex(N->getIndex(), MVT::i32);
+}]>;
+
+def : Pat<(i32 frameindex:$fi), (FRAME_ADDR (agx_to_tframeindex $fi))>;
+
+let hasSideEffects = 1 in
+def STACK_ADJUST : AGXPseudo<(outs), (ins i32imm:$size)> {
+ let G13Real = STACK_ADJUST_g13;
+}
+
+//===----------------------------------------------------------------------===//
+// Atomics
+//===----------------------------------------------------------------------===//
+
+// Atomic operation numbers.
+defvar AOP_ADD = 0;
+defvar AOP_SUB = 1;
+defvar AOP_XCHG = 2;
+defvar AOP_CMPXCHG = 3;
+defvar AOP_UMIN = 4;
+defvar AOP_IMIN = 5;
+defvar AOP_UMAX = 6;
+defvar AOP_IMAX = 7;
+defvar AOP_AND = 8;
+defvar AOP_OR = 9;
+defvar AOP_XOR = 10;
+
+let mayLoad = 1, mayStore = 1, hasSideEffects = 1, IsMemory = 1 in {
+def DEVICE_ATOMIC : AGXPseudo<(outs GPR32:$R),
+ (ins i32imm:$op, i32imm:$g, Base64Op:$A,
+ IndexOp:$O, i32imm:$sgn, GPR32:$S,
+ i32imm:$coherent)> {
+ let G13Real = ATOMIC_g13;
+ let IsAsync = 1;
+ let SchedRW = [WriteAtomic];
+}
+
+// The data operand is the pair {new value, compare value}.
+def DEVICE_ATOMIC_CMPXCHG : AGXPseudo<(outs GPR32:$R),
+ (ins i32imm:$op, i32imm:$g, Base64Op:$A,
+ IndexOp:$O, i32imm:$sgn, GPR64:$S,
+ i32imm:$coherent)> {
+ let G13Real = ATOMIC_g13;
+ let IsAsync = 1;
+ let SchedRW = [WriteAtomic];
+}
+
+def LOCAL_ATOMIC : AGXPseudo<(outs GPR32:$R),
+ (ins i32imm:$op, LocalBaseOp:$A, LocalIndexOp:$O,
+ GPR32:$S)> {
+ let G13Real = THREADGROUP_ATOMIC_g13;
+ let SchedRW = [WriteAtomic];
+}
+
+def LOCAL_ATOMIC_CMPXCHG : AGXPseudo<(outs GPR32:$R),
+ (ins i32imm:$op, LocalBaseOp:$A,
+ LocalIndexOp:$O, GPR64:$S)> {
+ let G13Real = THREADGROUP_ATOMIC_g13;
+ let SchedRW = [WriteAtomic];
+}
+}
+
+class AtomicFrag<SDPatternOperator op, list<int> as>
+ : PatFrag<(ops node:$ptr, node:$val), (op node:$ptr, node:$val)> {
+ let IsAtomic = 1;
+ let MemoryVT = i32;
+ let AddressSpaces = as;
+}
+
+class AtomicCmpSwapFrag<list<int> as>
+ : PatFrag<(ops node:$ptr, node:$cmp, node:$new),
+ (atomic_cmp_swap node:$ptr, node:$cmp, node:$new)> {
+ let IsAtomic = 1;
+ let MemoryVT = i32;
+ let AddressSpaces = as;
+}
+
+multiclass AGXAtomicPats<SDPatternOperator node, int op> {
+ def _device : AtomicFrag<node, DeviceAS>;
+ def _local : AtomicFrag<node, LocalAS>;
+
+ // Atomics take the full address in the base register; the element index
+ // is always zero.
+ def : Pat<(i32 (!cast<PatFrag>(NAME#"_device") i64:$ptr, i32:$val)),
+ (DEVICE_ATOMIC op, 0, $ptr, (i32 0), 0, $val, 0)>;
+ def : Pat<(i32 (!cast<PatFrag>(NAME#"_local") i32:$ptr, i32:$val)),
+ (LOCAL_ATOMIC op, (EXTRACT_SUBREG $ptr, lo16), (i16 0), $val)>;
+}
+
+defm agx_atomic_add : AGXAtomicPats<atomic_load_add, AOP_ADD>;
+defm agx_atomic_sub : AGXAtomicPats<atomic_load_sub, AOP_SUB>;
+defm agx_atomic_xchg : AGXAtomicPats<atomic_swap, AOP_XCHG>;
+defm agx_atomic_umin : AGXAtomicPats<atomic_load_umin, AOP_UMIN>;
+defm agx_atomic_imin : AGXAtomicPats<atomic_load_min, AOP_IMIN>;
+defm agx_atomic_umax : AGXAtomicPats<atomic_load_umax, AOP_UMAX>;
+defm agx_atomic_imax : AGXAtomicPats<atomic_load_max, AOP_IMAX>;
+defm agx_atomic_and : AGXAtomicPats<atomic_load_and, AOP_AND>;
+defm agx_atomic_or : AGXAtomicPats<atomic_load_or, AOP_OR>;
+defm agx_atomic_xor : AGXAtomicPats<atomic_load_xor, AOP_XOR>;
+
+def agx_cmpxchg_device : AtomicCmpSwapFrag<DeviceAS>;
+def agx_cmpxchg_local : AtomicCmpSwapFrag<LocalAS>;
+
+def : Pat<(i32 (agx_cmpxchg_device i64:$ptr, i32:$cmp, i32:$new)),
+ (DEVICE_ATOMIC_CMPXCHG AOP_CMPXCHG, 0, $ptr, (i32 0), 0,
+ (REG_SEQUENCE GPR64, $new, sub0, $cmp, sub1), 0)>;
+def : Pat<(i32 (agx_cmpxchg_local i32:$ptr, i32:$cmp, i32:$new)),
+ (LOCAL_ATOMIC_CMPXCHG AOP_CMPXCHG, (EXTRACT_SUBREG $ptr, lo16),
+ (i16 0), (REG_SEQUENCE GPR64, $new, sub0, $cmp, sub1))>;
+
+//===----------------------------------------------------------------------===//
+// Synchronization
+//===----------------------------------------------------------------------===//
+
+let hasSideEffects = 1, isConvergent = 1 in {
+def THREADGROUP_BARRIER : AGXPseudo<(outs), (ins),
+ [(int_agx_threadgroup_barrier)]> {
+ let G13Real = THREADGROUP_BARRIER_g13;
+ let SchedRW = [WriteBarrier];
+}
+}
+
+let hasSideEffects = 1 in
+def MEMORY_BARRIER : AGXPseudo<(outs), (ins i32imm:$a, i32imm:$b, i32imm:$c),
+ [(int_agx_memory_barrier timm:$a, timm:$b,
+ timm:$c)]> {
+ let G13Real = MEMORY_BARRIER_g13;
+ let SchedRW = [WriteBarrier];
+}
+
+// Device scope barrier, as emitted for fences.
+def : Pat<(atomic_fence (i64 timm), (i64 timm)), (MEMORY_BARRIER 1, 2, 9)>;
+
+let hasSideEffects = 1 in
+def WAIT : AGXPseudo<(outs), (ins i32imm:$slot), [(int_agx_wait timm:$slot)]> {
+ let G13Real = WAIT_g13;
+ let SchedRW = [WriteControl];
+}
+
+//===----------------------------------------------------------------------===//
+// SIMD-group operations
+//===----------------------------------------------------------------------===//
+
+multiclass ShufflePseudo<Instruction real, SDPatternOperator intr> {
+ let isConvergent = 1 in {
+ def _32 : AGXALUPseudo<real, (outs GPR32:$dst),
+ (ins SrcMods:$src0_mods, Src32Op:$src0,
+ SrcMods:$src1_mods, Src16Op:$src1), [],
+ [WriteShuffle]>;
+ def _16 : AGXALUPseudo<real, (outs GPR16:$dst),
+ (ins SrcMods:$src0_mods, Src16Op:$src0,
+ SrcMods:$src1_mods, Src16Op:$src1), [],
+ [WriteShuffle]>;
+ }
+ foreach VT = [i32, f32] in
+ def : Pat<(VT (intr VT:$v, i32:$idx)),
+ (!cast<Instruction>(NAME#"_32") 0, $v, 0,
+ (EXTRACT_SUBREG $idx, lo16))>;
+ foreach VT = [i16, f16] in
+ def : Pat<(VT (intr VT:$v, i32:$idx)),
+ (!cast<Instruction>(NAME#"_16") 0, $v, 0,
+ (EXTRACT_SUBREG $idx, lo16))>;
+}
+
+defm SIMD_SHUFFLE : ShufflePseudo<SIMD_SHUFFLE_g13, int_agx_simd_shuffle>;
+defm SIMD_SHUFFLE_XOR : ShufflePseudo<SIMD_SHUFFLE_XOR_g13, int_agx_simd_shuffle_xor>;
+defm SIMD_SHUFFLE_UP : ShufflePseudo<SIMD_SHUFFLE_UP_g13, int_agx_simd_shuffle_up>;
+defm SIMD_SHUFFLE_DOWN : ShufflePseudo<SIMD_SHUFFLE_DOWN_g13, int_agx_simd_shuffle_down>;
+defm QUAD_SHUFFLE : ShufflePseudo<QUAD_SHUFFLE_g13, int_agx_quad_shuffle>;
+defm QUAD_SHUFFLE_XOR : ShufflePseudo<QUAD_SHUFFLE_XOR_g13, int_agx_quad_shuffle_xor>;
+defm QUAD_SHUFFLE_UP : ShufflePseudo<QUAD_SHUFFLE_UP_g13, int_agx_quad_shuffle_up>;
+defm QUAD_SHUFFLE_DOWN : ShufflePseudo<QUAD_SHUFFLE_DOWN_g13, int_agx_quad_shuffle_down>;
+
+let isConvergent = 1 in {
+def ICMP_BALLOT : AGXALUPseudo<ICMP_BALLOT_g13, (outs GPR32:$dst),
+ (ins i32imm:$cc, SrcMods:$src0_mods,
+ Src32Op:$src0, SrcMods:$src1_mods,
+ Src32Op:$src1), [], [WriteShuffle]>;
+def ICMP_QUAD_BALLOT : AGXALUPseudo<ICMP_QUAD_BALLOT_g13, (outs GPR32:$dst),
+ (ins i32imm:$cc, SrcMods:$src0_mods,
+ Src32Op:$src0, SrcMods:$src1_mods,
+ Src32Op:$src1), [], [WriteShuffle]>;
+}
+
+// ballot(c) = icmp_ballot nueq c, 0 (the inverted ueq condition is 8).
+def : Pat<(i32 (AGXballot i32:$c)),
+ (ICMP_BALLOT 8, 0, $c, 0, (i32 0))>;
+def : Pat<(i32 (AGXquad_ballot i32:$c)),
+ (BITOP_I32 TT_AND, 0, (ICMP_QUAD_BALLOT 8, 0, $c, 0, (i32 0)), 0,
+ (i32 15))>;
+
+foreach Op = AGXSIMDOps in
+def AGXSIMDOpImm#Op.Op : TImmLeaf<i32, "return Imm == "#Op.Op#";">;
+
+foreach Kind = AGXReduceKinds in
+foreach Op = AGXSIMDOps in {
+ defvar Real = !cast<Instruction>(Kind.Name#"_"#Op.Name#"_g13");
+ let isConvergent = 1 in
+ def Kind.Name#_#Op.Name#_32
+ : AGXALUPseudo<Real, (outs GPR32:$dst),
+ (ins SrcMods:$src0_mods, Src32Op:$src0), [],
+ [!if(!eq(Kind.Opc, 1), WriteSIMDReduce,
+ !if(!eq(Kind.Opc, 3), WriteSIMDReduce,
+ WriteQuadReduce))]>;
+ defvar Intr = !cond(!eq(Kind.Opc, 0) : int_agx_quad_reduce,
+ !eq(Kind.Opc, 1) : int_agx_simd_reduce,
+ !eq(Kind.Opc, 2) : int_agx_quad_prefix,
+ true : int_agx_simd_prefix);
+ foreach VT = [i32, f32] in
+ def : Pat<(VT (Intr VT:$v, (i32 !cast<TImmLeaf>("AGXSIMDOpImm"#Op.Op)))),
+ (!cast<Instruction>(Kind.Name#"_"#Op.Name#"_32") 0, $v)>;
+}
+
+let isConvergent = 1 in
+def SIMD_MATRIX_FMADD32 : AGXALUPseudo<SIMD_MATRIX_FMADD32_g13,
+ (outs GPR64:$dst),
+ (ins SrcMods:$src0_mods, Src64Op:$src0,
+ SrcMods:$src1_mods, Src64Op:$src1,
+ SrcMods:$src2_mods, Src64Op:$src2),
+ [], [WriteSIMDReduce]>;
+
+//===----------------------------------------------------------------------===//
+// Control flow
+//===----------------------------------------------------------------------===//
+
+// Exec mask instructions, operating on the nesting counter r0l.
+let hasSideEffects = 1, Uses = [R0L], Defs = [R0L], IsExecMaskOp = 1,
+ SchedRW = [WriteControl] in {
+def IF_ICMP : AGXPseudo<(outs), (ins i32imm:$nest, i32imm:$cc,
+ SrcMods:$src0_mods, Src32Op:$src0,
+ SrcMods:$src1_mods, Src32Op:$src1,
+ i32imm:$n)> {
+ let G13Real = IF_ICMP_g13;
+}
+def ELSE_ICMP : AGXPseudo<(outs), (ins i32imm:$nest, i32imm:$cc,
+ SrcMods:$src0_mods, Src32Op:$src0,
+ SrcMods:$src1_mods, Src32Op:$src1,
+ i32imm:$n)> {
+ let G13Real = ELSE_ICMP_g13;
+}
+def WHILE_ICMP : AGXPseudo<(outs), (ins i32imm:$nest, i32imm:$cc,
+ SrcMods:$src0_mods, Src32Op:$src0,
+ SrcMods:$src1_mods, Src32Op:$src1,
+ i32imm:$n)> {
+ let G13Real = WHILE_ICMP_g13;
+}
+def IF_FCMP : AGXPseudo<(outs), (ins i32imm:$nest, i32imm:$cc,
+ SrcMods:$src0_mods, Src32Op:$src0,
+ SrcMods:$src1_mods, Src32Op:$src1,
+ i32imm:$n)> {
+ let G13Real = IF_FCMP_g13;
+}
+def ELSE_FCMP : AGXPseudo<(outs), (ins i32imm:$nest, i32imm:$cc,
+ SrcMods:$src0_mods, Src32Op:$src0,
+ SrcMods:$src1_mods, Src32Op:$src1,
+ i32imm:$n)> {
+ let G13Real = ELSE_FCMP_g13;
+}
+def POP_EXEC : AGXPseudo<(outs), (ins i32imm:$nest, i32imm:$n)> {
+ let G13Real = POP_EXEC_g13;
+}
+}
+
+// mov_imm r0l, n: initialise or set the nesting counter.
+let hasSideEffects = 1, Defs = [R0L] in
+def SET_NEST : AGXPseudo<(outs), (ins i32imm:$imm)> {
+ let G13Real = MOV_IMM16_g13;
+}
+
+let isBranch = 1, isTerminator = 1, Uses = [R0L] in {
+def JMP_EXEC_ANY : AGXPseudo<(outs), (ins AGXBrTarget:$target)> {
+ let G13Real = JMP_EXEC_ANY_g13;
+}
+def JMP_EXEC_NONE : AGXPseudo<(outs), (ins AGXBrTarget:$target)> {
+ let G13Real = JMP_EXEC_NONE_g13;
+}
+}
+
+// Structured control flow, expanded by AGXLowerControlFlow.
+let isBranch = 1, isTerminator = 1, hasSideEffects = 1, Uses = [R0L],
+ Defs = [R0L] in {
+// Deactivate the threads where $cond is zero; branch to $target if no thread
+// remains active.
+def AGX_IF : AGXCodeGenPseudo<(outs), (ins GPR32:$cond, AGXBrTarget:$target),
+ [(AGXif i32:$cond, bb:$target)]>;
+// Swap the active threads of the innermost if; branch to $target if none.
+def AGX_ELSE : AGXCodeGenPseudo<(outs), (ins AGXBrTarget:$target),
+ [(AGXelse bb:$target)]>;
+// Deactivate the threads where $brk is nonzero; branch to $target (the
+// loop header) if any thread remains active.
+def AGX_LOOP : AGXCodeGenPseudo<(outs), (ins GPR32:$brk, AGXBrTarget:$target),
+ [(AGXloop i32:$brk, bb:$target)]>;
+}
+
+let hasSideEffects = 1, Uses = [R0L], Defs = [R0L] in {
+// Reactivate the threads deactivated by the innermost construct.
+def AGX_END_CF : AGXCodeGenPseudo<(outs), (ins), [(AGXend_cf)]>;
+// Enter a loop: protect the threads that are already inactive.
+def AGX_LOOP_BEGIN : AGXCodeGenPseudo<(outs), (ins), [(AGXloop_begin)]>;
+}
+
+let isBranch = 1, isTerminator = 1, isBarrier = 1 in
+def AGX_BR : AGXCodeGenPseudo<(outs), (ins AGXBrTarget:$target), [(br bb:$target)]>;
+
+let isTerminator = 1, isReturn = 1, isBarrier = 1, hasSideEffects = 1 in
+def AGX_STOP : AGXPseudo<(outs), (ins), [(AGXstop)]> {
+ let G13Real = STOP_g13;
+}
+
+let isTrap = 1, isTerminator = 1, isBarrier = 1, hasSideEffects = 1 in
+def AGX_TRAP : AGXPseudo<(outs), (ins), [(trap)]> {
+ let G13Real = TRAP_g13;
+}
diff --git a/llvm/lib/Target/AGX/AGXLowerControlFlow.cpp b/llvm/lib/Target/AGX/AGXLowerControlFlow.cpp
new file mode 100644
index 0000000000000..6e16961a5cfbf
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXLowerControlFlow.cpp
@@ -0,0 +1,151 @@
+//===-- AGXLowerControlFlow.cpp - Lower structured control flow -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Expands the structured control flow pseudos into execution mask
+// instructions on the nesting counter r0l and jumps (like SILowerControlFlow):
+//
+// AGX_IF c, %flow if_icmp r0l, ne, c, 0, 1; jmp_exec_none %flow
+// AGX_ELSE %end else_fcmp r0l, eq, 0.0, 0.0, 1; jmp_exec_none %end
+// AGX_LOOP brk, %hdr while_icmp r0l, eq, brk, 0, 1; jmp_exec_any %hdr
+// AGX_LOOP_BEGIN if_fcmp r0l, eq, 0.0, 0.0, 1 (push_exec 1)
+// AGX_END_CF pop_exec r0l, 1
+// AGX_BR %bb jmp_exec_any %bb; jmp_exec_none %bb
+//
+// The jumps only skip code no thread executes; the execution mask
+// instructions alone implement the semantics. The nesting counter is zeroed
+// at kernel entry.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AGX.h"
+#include "AGXInstrInfo.h"
+#include "AGXSubtarget.h"
+#include "MCTargetDesc/AGXMCTargetDesc.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "agx-lower-control-flow"
+
+namespace {
+
+class AGXLowerControlFlow : public MachineFunctionPass {
+public:
+ static char ID;
+ AGXLowerControlFlow() : MachineFunctionPass(ID) {}
+
+ StringRef getPassName() const override {
+ return "AGX Lower Control Flow";
+ }
+ bool runOnMachineFunction(MachineFunction &MF) override;
+};
+
+} // namespace
+
+char AGXLowerControlFlow::ID = 0;
+
+INITIALIZE_PASS(AGXLowerControlFlow, DEBUG_TYPE, "AGX Lower Control Flow",
+ false, false)
+
+FunctionPass *llvm::createAGXLowerControlFlowPass() {
+ return new AGXLowerControlFlow();
+}
+
+// Integer conditions.
+static constexpr unsigned ICondUEQ = 0;
+static constexpr unsigned ICondNE = 8;
+// Float condition eq, and the 8-bit float immediate 0.0.
+static constexpr unsigned FCondEQ = 0;
+static constexpr unsigned FPImmZero = 0;
+
+/// Build an execution mask instruction: (nest, cc, a, b, n).
+static MachineInstrBuilder buildExecOp(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator I,
+ const DebugLoc &DL,
+ const AGXInstrInfo &TII, unsigned Opc,
+ unsigned CC) {
+ return BuildMI(MBB, I, DL, TII.get(Opc)).addImm(0).addImm(CC);
+}
+
+bool AGXLowerControlFlow::runOnMachineFunction(MachineFunction &MF) {
+ const AGXInstrInfo &TII = *MF.getSubtarget<AGXSubtarget>().getInstrInfo();
+ bool HasControlFlow = false;
+
+ for (MachineBasicBlock &MBB : MF) {
+ for (MachineInstr &MI : make_early_inc_range(MBB)) {
+ MachineBasicBlock::iterator I = MI.getIterator();
+ const DebugLoc &DL = MI.getDebugLoc();
+ switch (MI.getOpcode()) {
+ case AGX::AGX_IF: {
+ buildExecOp(MBB, I, DL, TII, AGX::IF_ICMP, ICondNE)
+ .addImm(0)
+ .add(MI.getOperand(0))
+ .addImm(0)
+ .addImm(0)
+ .addImm(1);
+ BuildMI(MBB, I, DL, TII.get(AGX::JMP_EXEC_NONE))
+ .addMBB(MI.getOperand(1).getMBB());
+ break;
+ }
+ case AGX::AGX_ELSE:
+ buildExecOp(MBB, I, DL, TII, AGX::ELSE_FCMP, FCondEQ)
+ .addImm(0)
+ .addImm(FPImmZero)
+ .addImm(0)
+ .addImm(FPImmZero)
+ .addImm(1);
+ BuildMI(MBB, I, DL, TII.get(AGX::JMP_EXEC_NONE))
+ .addMBB(MI.getOperand(0).getMBB());
+ break;
+ case AGX::AGX_LOOP:
+ // Threads stay in the loop while brk is zero.
+ buildExecOp(MBB, I, DL, TII, AGX::WHILE_ICMP, ICondUEQ)
+ .addImm(0)
+ .add(MI.getOperand(0))
+ .addImm(0)
+ .addImm(0)
+ .addImm(1);
+ BuildMI(MBB, I, DL, TII.get(AGX::JMP_EXEC_ANY))
+ .addMBB(MI.getOperand(1).getMBB());
+ break;
+ case AGX::AGX_LOOP_BEGIN:
+ buildExecOp(MBB, I, DL, TII, AGX::IF_FCMP, FCondEQ)
+ .addImm(0)
+ .addImm(FPImmZero)
+ .addImm(0)
+ .addImm(FPImmZero)
+ .addImm(1);
+ break;
+ case AGX::AGX_END_CF:
+ BuildMI(MBB, I, DL, TII.get(AGX::POP_EXEC)).addImm(0).addImm(1);
+ break;
+ case AGX::AGX_BR: {
+ MachineBasicBlock *Target = MI.getOperand(0).getMBB();
+ if (!MBB.isLayoutSuccessor(Target)) {
+ BuildMI(MBB, I, DL, TII.get(AGX::JMP_EXEC_ANY)).addMBB(Target);
+ BuildMI(MBB, I, DL, TII.get(AGX::JMP_EXEC_NONE)).addMBB(Target);
+ }
+ MI.eraseFromParent();
+ continue;
+ }
+ default:
+ continue;
+ }
+ HasControlFlow = true;
+ MI.eraseFromParent();
+ }
+ }
+
+ if (HasControlFlow) {
+ MachineBasicBlock &Entry = MF.front();
+ BuildMI(Entry, Entry.begin(), DebugLoc(), TII.get(AGX::SET_NEST))
+ .addImm(0);
+ }
+ return true;
+}
diff --git a/llvm/lib/Target/AGX/AGXMCInstLower.cpp b/llvm/lib/Target/AGX/AGXMCInstLower.cpp
new file mode 100644
index 0000000000000..38d95ba76990a
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXMCInstLower.cpp
@@ -0,0 +1,82 @@
+//===-- AGXMCInstLower.cpp - Lower MachineInstr to MCInst -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Pseudo instructions have the operands of their real instruction, except
+// that real ALU destinations carry a cache hint immediate. The hint is
+// inserted here (as zero: no hint).
+//
+//===----------------------------------------------------------------------===//
+
+#include "AGXMCInstLower.h"
+#include "AGXInstrInfo.h"
+#include "AGXSubtarget.h"
+#include "MCTargetDesc/AGXMCTargetDesc.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/Support/ErrorHandling.h"
+
+using namespace llvm;
+
+bool AGXMCInstLower::lowerOperand(const MachineOperand &MO,
+ MCOperand &MCOp) const {
+ switch (MO.getType()) {
+ case MachineOperand::MO_Register:
+ if (MO.isImplicit())
+ return false;
+ MCOp = MCOperand::createReg(MO.getReg());
+ return true;
+ case MachineOperand::MO_Immediate:
+ MCOp = MCOperand::createImm(MO.getImm());
+ return true;
+ case MachineOperand::MO_MachineBasicBlock:
+ MCOp = MCOperand::createExpr(
+ MCSymbolRefExpr::create(MO.getMBB()->getSymbol(), Ctx));
+ return true;
+ case MachineOperand::MO_RegisterMask:
+ return false;
+ default:
+ report_fatal_error("unsupported AGX machine operand");
+ }
+}
+
+bool AGXMCInstLower::lower(const MachineInstr *MI, MCInst &OutMI) const {
+ const AGXInstrInfo *TII = ST.getInstrInfo();
+ int Opcode = TII->pseudoToMCOpcode(MI->getOpcode());
+ if (Opcode < 0)
+ return false;
+ OutMI.setOpcode(Opcode);
+
+ // mov_imm r0l, n.
+ if (MI->getOpcode() == AGX::SET_NEST) {
+ OutMI.addOperand(MCOperand::createReg(AGX::R0L));
+ OutMI.addOperand(MCOperand::createImm(0));
+ OutMI.addOperand(MCOperand::createImm(MI->getOperand(0).getImm()));
+ return true;
+ }
+
+ const MCInstrDesc &PseudoDesc = MI->getDesc();
+ const MCInstrDesc &RealDesc = TII->get(Opcode);
+ bool InsertCache = PseudoDesc.getNumDefs() > 0 &&
+ RealDesc.getNumOperands() == PseudoDesc.getNumOperands() + 1;
+
+ unsigned Idx = 0;
+ for (const MachineOperand &MO : MI->operands()) {
+ MCOperand MCOp;
+ if (!lowerOperand(MO, MCOp))
+ continue;
+ OutMI.addOperand(MCOp);
+ if (Idx++ == 0 && InsertCache)
+ OutMI.addOperand(MCOperand::createImm(0));
+ }
+ assert(OutMI.getNumOperands() == RealDesc.getNumOperands() &&
+ "pseudo and real instruction operands do not match");
+ return true;
+}
diff --git a/llvm/lib/Target/AGX/AGXMCInstLower.h b/llvm/lib/Target/AGX/AGXMCInstLower.h
new file mode 100644
index 0000000000000..8767c4ddb78e1
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXMCInstLower.h
@@ -0,0 +1,39 @@
+//===-- AGXMCInstLower.h - Lower MachineInstr to MCInst ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AGX_AGXMCINSTLOWER_H
+#define LLVM_LIB_TARGET_AGX_AGXMCINSTLOWER_H
+
+namespace llvm {
+
+class AGXSubtarget;
+class MCContext;
+class MCInst;
+class MCOperand;
+class MachineInstr;
+class MachineOperand;
+
+/// Lowers code generation pseudo instructions to the real instructions of
+/// the subtarget's encoding family.
+class AGXMCInstLower {
+ MCContext &Ctx;
+ const AGXSubtarget &ST;
+
+public:
+ AGXMCInstLower(MCContext &Ctx, const AGXSubtarget &ST) : Ctx(Ctx), ST(ST) {}
+
+ bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const;
+
+ /// Lower \p MI. Returns false if the instruction has no encoding for the
+ /// subtarget.
+ bool lower(const MachineInstr *MI, MCInst &OutMI) const;
+};
+
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AGX_AGXMCINSTLOWER_H
diff --git a/llvm/lib/Target/AGX/AGXMachineFunctionInfo.cpp b/llvm/lib/Target/AGX/AGXMachineFunctionInfo.cpp
new file mode 100644
index 0000000000000..84eaa2bab76c1
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXMachineFunctionInfo.cpp
@@ -0,0 +1,27 @@
+//===-- AGXMachineFunctionInfo.cpp - AGX machine function info ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AGXMachineFunctionInfo.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/GlobalVariable.h"
+#include "llvm/Support/MathExtras.h"
+
+using namespace llvm;
+
+unsigned
+AGXMachineFunctionInfo::allocateThreadgroupVariable(const DataLayout &DL,
+ const GlobalVariable &GV) {
+ auto [It, Inserted] = ThreadgroupOffsets.try_emplace(&GV, 0);
+ if (!Inserted)
+ return It->second;
+ Align A = DL.getValueOrABITypeAlignment(GV.getAlign(), GV.getValueType());
+ unsigned Offset = alignTo(ThreadgroupSize, A);
+ It->second = Offset;
+ ThreadgroupSize = Offset + DL.getTypeAllocSize(GV.getValueType());
+ return Offset;
+}
diff --git a/llvm/lib/Target/AGX/AGXMachineFunctionInfo.h b/llvm/lib/Target/AGX/AGXMachineFunctionInfo.h
new file mode 100644
index 0000000000000..a6c3d998d07f3
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXMachineFunctionInfo.h
@@ -0,0 +1,63 @@
+//===-- AGXMachineFunctionInfo.h - AGX machine function info ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AGX_AGXMACHINEFUNCTIONINFO_H
+#define LLVM_LIB_TARGET_AGX_AGXMACHINEFUNCTIONINFO_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/CodeGen/MachineFunction.h"
+
+namespace llvm {
+
+class GlobalVariable;
+
+/// Per kernel state: the uniform register layout of the arguments and the
+/// threadgroup memory allocation.
+class AGXMachineFunctionInfo : public MachineFunctionInfo {
+ /// Half register index of the first implicit argument
+ /// (threadgroups_per_grid.xyz), and the end of the uniform range used by
+ /// arguments.
+ unsigned ImplicitArgHalfIndex = 0;
+ unsigned UniformHalfCount = 0;
+ /// Virtual registers holding threadgroups_per_grid.xyz.
+ Register ImplicitArgVRegs[3];
+
+ /// Threadgroup memory: offset of each __shared__ style variable.
+ DenseMap<const GlobalVariable *, unsigned> ThreadgroupOffsets;
+ unsigned ThreadgroupSize = 0;
+
+public:
+ AGXMachineFunctionInfo(const Function &F, const TargetSubtargetInfo *STI) {}
+
+ MachineFunctionInfo *
+ clone(BumpPtrAllocator &Allocator, MachineFunction &DestMF,
+ const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &Src2DstMBB)
+ const override {
+ return DestMF.cloneInfo<AGXMachineFunctionInfo>(*this);
+ }
+
+ void setImplicitArgHalfIndex(unsigned I) { ImplicitArgHalfIndex = I; }
+ unsigned getImplicitArgHalfIndex() const { return ImplicitArgHalfIndex; }
+ void setUniformHalfCount(unsigned N) { UniformHalfCount = N; }
+ unsigned getUniformHalfCount() const { return UniformHalfCount; }
+ void setImplicitArgVReg(unsigned Dim, Register R) {
+ ImplicitArgVRegs[Dim] = R;
+ }
+ Register getImplicitArgVReg(unsigned Dim) const {
+ return ImplicitArgVRegs[Dim];
+ }
+
+ /// Offset of \p GV in threadgroup memory, allocating it on first use.
+ unsigned allocateThreadgroupVariable(const DataLayout &DL,
+ const GlobalVariable &GV);
+ unsigned getThreadgroupSize() const { return ThreadgroupSize; }
+};
+
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AGX_AGXMACHINEFUNCTIONINFO_H
diff --git a/llvm/lib/Target/AGX/AGXOperands.td b/llvm/lib/Target/AGX/AGXOperands.td
new file mode 100644
index 0000000000000..7644f7a2a298c
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXOperands.td
@@ -0,0 +1,312 @@
+//===-- AGXOperands.td - AGX operand definitions -----------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Operands of the real (encoded) instructions and of the pseudo instructions.
+//
+// Encoded ALU sources are complex operands made of a modifier immediate and a
+// value. The value is a general purpose register, a uniform register or an
+// 8-bit immediate; the register class carries the access size. The modifier
+// immediate holds AGXSrcMods flags (abs, neg, sx, cache, discard).
+//
+// The encoder methods produce the operand fields laid out as follows (bit
+// numbers within the encoded operand value):
+//
+// Src, Src64, Src16 [5:0] value, [9:6] type, [11:10] extension
+// SrcSX, SrcSX64 [5:0] value, [9:6] type, [10] sign extend,
+// [12:11] extension
+// SrcF [5:0] value, [9:6] type, [11:10] abs/neg,
+// [13:12] extension
+// SrcF16 [5:0] value, [8:6] type, [10:9] abs/neg,
+// [12:11] extension
+// CmpselSrc [5:0] value, [8:6] type, [10:9] extension
+// Dst* [0] cache, [1] 32-bit, [7:2] value, [9:8] extension
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// Assembler operand classes
+//===----------------------------------------------------------------------===//
+
+class AGXAsmOperand<string name, string parser, string render> :
+ AsmOperandClass {
+ let Name = name;
+ let PredicateMethod = "is"#name;
+ let ParserMethod = parser;
+ let RenderMethod = render;
+ let DiagnosticType = "Invalid"#name;
+}
+
+class AGXSrcAsmOperand<string name>
+ : AGXAsmOperand<name, "parseSrc", "addSrcOperands">;
+
+def AGXSrcAsm : AGXSrcAsmOperand<"Src">;
+def AGXSrc64Asm : AGXSrcAsmOperand<"Src64">;
+def AGXSrc16Asm : AGXSrcAsmOperand<"Src16">;
+def AGXSrcSXAsm : AGXSrcAsmOperand<"SrcSX">;
+def AGXSrcSX64Asm : AGXSrcAsmOperand<"SrcSX64">;
+def AGXSrcFAsm : AGXSrcAsmOperand<"SrcF">;
+def AGXSrcF16Asm : AGXSrcAsmOperand<"SrcF16">;
+def AGXCmpselSrcAsm : AGXSrcAsmOperand<"CmpselSrc">;
+
+class AGXDstAsmOperand<string name>
+ : AGXAsmOperand<name, "parseDst", "addDstOperands">;
+
+def AGXDstAsm : AGXDstAsmOperand<"Dst">;
+def AGXDst64Asm : AGXDstAsmOperand<"Dst64">;
+def AGXDst16Asm : AGXDstAsmOperand<"Dst16">;
+def AGXDst32Asm : AGXDstAsmOperand<"Dst32">;
+
+def AGXImmAsm : AGXAsmOperand<"AGXImm", "parseImm", "addImmOperands">;
+
+def AGXShiftAsm : AGXAsmOperand<"Shift", "parseShift", "addImmOperands"> {
+ let IsOptional = 1;
+ let DefaultMethod = "defaultZeroOperand";
+}
+
+def AGXBitfieldMaskAsm : AGXAsmOperand<"BitfieldMask", "parseBitfieldMask",
+ "addImmOperands"> {
+ let IsOptional = 1;
+ let DefaultMethod = "defaultZeroOperand";
+}
+
+def AGXTruthTableAsm : AGXAsmOperand<"TruthTable", "parseTruthTable",
+ "addImmOperands">;
+
+class AGXEnumAsmOperand<string name>
+ : AGXAsmOperand<name, "parse"#name, "addImmOperands">;
+
+def AGXMemRegAsm : AGXAsmOperand<"MemReg", "parseMemReg", "addRegOperands">;
+def AGXAtomicDstAsm : AGXAsmOperand<"AtomicDst", "parseMemReg",
+ "addRegOperands">;
+def AGXMemBaseAsm : AGXAsmOperand<"MemBase", "parseMemReg", "addRegOperands">;
+def AGXAtomicSrcAsm : AGXAsmOperand<"AtomicSrc", "parseMemReg",
+ "addRegOperands">;
+def AGXMemIndexAsm : AGXAsmOperand<"MemIndex", "parseMemIndex",
+ "addRegOrImmOperands">;
+def AGXLocalBaseAsm : AGXAsmOperand<"LocalBase", "parseMemIndex",
+ "addRegOrImmOperands">;
+def AGXLocalIndexAsm : AGXAsmOperand<"LocalIndex", "parseMemIndex",
+ "addRegOrImmOperands">;
+def AGXBrTargetAsm : AGXAsmOperand<"BrTarget", "parseBrTarget",
+ "addBrTargetOperands">;
+def AGXNestCounterAsm : AGXAsmOperand<"NestCounter", "parseNestCounter",
+ "addImmOperands">;
+
+//===----------------------------------------------------------------------===//
+// ALU operands
+//===----------------------------------------------------------------------===//
+
+// Sub-operand of a real instruction that holds either a register or an
+// immediate.
+def AGXAnyValue : Operand<i32>;
+
+class AGXSrcOperandBase<string kind, AsmOperandClass asm> : Operand<i32> {
+ let MIOperandInfo = (ops i32imm:$mods, AGXAnyValue:$val);
+ let PrintMethod = "print"#kind;
+ let EncoderMethod = "get"#kind#"Encoding";
+ let DecoderMethod = "decode"#kind;
+ let ParserMatchClass = asm;
+ let OperandType = "OPERAND_SRC";
+ let OperandNamespace = "AGX";
+}
+
+// Zero extended 16/32-bit source.
+def AGXSrc : AGXSrcOperandBase<"Src", AGXSrcAsm>;
+// Zero extended 16/32/64-bit source.
+def AGXSrc64 : AGXSrcOperandBase<"Src64", AGXSrc64Asm>;
+// 16-bit only source.
+def AGXSrc16 : AGXSrcOperandBase<"Src16", AGXSrc16Asm>;
+// Sign-extendable 16/32-bit source (imadd A/B).
+def AGXSrcSX : AGXSrcOperandBase<"SrcSX", AGXSrcSXAsm>;
+// Sign-extendable 16/32/64-bit source (iadd, imadd C).
+def AGXSrcSX64 : AGXSrcOperandBase<"SrcSX64", AGXSrcSX64Asm>;
+// Floating point 16/32-bit source with abs/neg; immediates are 8-bit floats.
+def AGXSrcF : AGXSrcOperandBase<"SrcF", AGXSrcFAsm>;
+// Floating point 16-bit source with abs/neg.
+def AGXSrcF16 : AGXSrcOperandBase<"SrcF16", AGXSrcF16Asm>;
+// Compare-and-select value source, sized like the destination.
+def AGXCmpselSrc : AGXSrcOperandBase<"CmpselSrc", AGXCmpselSrcAsm>;
+
+class AGXDstOperandBase<string kind, AsmOperandClass asm> : Operand<i32> {
+ let MIOperandInfo = (ops AGXAnyValue:$reg, i32imm:$cache);
+ let PrintMethod = "printDst";
+ let EncoderMethod = "getDstEncoding";
+ let DecoderMethod = "decode"#kind;
+ let ParserMatchClass = asm;
+ let OperandType = "OPERAND_DST";
+ let OperandNamespace = "AGX";
+}
+
+// 16/32-bit destination.
+def AGXDst : AGXDstOperandBase<"Dst", AGXDstAsm>;
+// 16/32/64-bit destination.
+def AGXDst64 : AGXDstOperandBase<"Dst64", AGXDst64Asm>;
+// Destinations whose size bit is part of the opcode.
+def AGXDst16 : AGXDstOperandBase<"Dst16", AGXDst16Asm>;
+def AGXDst32 : AGXDstOperandBase<"Dst32", AGXDst32Asm>;
+
+//===----------------------------------------------------------------------===//
+// Immediate operands
+//===----------------------------------------------------------------------===//
+
+def AGXImm : Operand<i32> {
+ let PrintMethod = "printImm";
+ let ParserMatchClass = AGXImmAsm;
+ let DecoderMethod = "decodeImm";
+}
+
+def AGXImm64 : Operand<i64> {
+ let PrintMethod = "printImm";
+ let ParserMatchClass = AGXImmAsm;
+ let DecoderMethod = "decodeImm";
+}
+
+// Pseudo instruction operand holding AGXSrcMods flags.
+def SrcMods : Operand<i32> {
+ let PrintMethod = "printImm";
+}
+
+def AGXShift : Operand<i32> {
+ let PrintMethod = "printShift";
+ let ParserMatchClass = AGXShiftAsm;
+ let DecoderMethod = "decodeImm";
+}
+
+def AGXBitfieldMask : Operand<i32> {
+ let PrintMethod = "printBitfieldMask";
+ let ParserMatchClass = AGXBitfieldMaskAsm;
+ let DecoderMethod = "decodeImm";
+}
+
+def AGXTruthTable : Operand<i32> {
+ let PrintMethod = "printTruthTable";
+ let ParserMatchClass = AGXTruthTableAsm;
+ let DecoderMethod = "decodeImm";
+}
+
+multiclass AGXEnum<string name> {
+ def Asm : AGXEnumAsmOperand<name>;
+ def "" : Operand<i32> {
+ let PrintMethod = "print"#name;
+ let ParserMatchClass = !cast<AsmOperandClass>(NAME#"Asm");
+ let DecoderMethod = "decodeImm";
+ }
+}
+
+// Integer conditions including the inverted forms (4 bits).
+defm AGXICond : AGXEnum<"ICond">;
+// Float conditions including the inverted forms (4 bits).
+defm AGXFCond : AGXEnum<"FCond">;
+// Conditions without inversion (3 bits), for compare-and-select.
+defm AGXICond3 : AGXEnum<"ICond3">;
+defm AGXFCond3 : AGXEnum<"FCond3">;
+defm AGXConvertMode : AGXEnum<"ConvertMode">;
+defm AGXRoundMode : AGXEnum<"RoundMode">;
+defm AGXMemFormat : AGXEnum<"MemFormat">;
+defm AGXMemMask : AGXEnum<"MemMask">;
+defm AGXSignedness : AGXEnum<"Signedness">;
+defm AGXAtomicOp : AGXEnum<"AtomicOp">;
+defm AGXSpecialReg : AGXEnum<"SpecialReg">;
+
+//===----------------------------------------------------------------------===//
+// Memory operands
+//===----------------------------------------------------------------------===//
+
+// Register (or first register of the register tuple) read or written by a
+// memory instruction. The element count is the popcount of the mask.
+def AGXMemReg : Operand<i32> {
+ let PrintMethod = "printMemReg";
+ let EncoderMethod = "getMemRegEncoding";
+ let DecoderMethod = "decodeMemReg";
+ let ParserMatchClass = AGXMemRegAsm;
+}
+
+// Optional atomic result register ('_' for none).
+def AGXAtomicDst : Operand<i32> {
+ let PrintMethod = "printAtomicDst";
+ let EncoderMethod = "getAtomicDstEncoding";
+ let DecoderMethod = "decodeAtomicDst";
+ let ParserMatchClass = AGXAtomicDstAsm;
+}
+
+// 64-bit device memory base: general purpose or uniform register pair.
+def AGXMemBase : Operand<i64> {
+ let PrintMethod = "printMemBase";
+ let EncoderMethod = "getMemBaseEncoding";
+ let DecoderMethod = "decodeMemBase";
+ let ParserMatchClass = AGXMemBaseAsm;
+}
+
+// Device memory index: 32-bit register or signed 16-bit immediate.
+def AGXMemIndex : Operand<i32> {
+ let PrintMethod = "printMemIndex";
+ let EncoderMethod = "getMemIndexEncoding";
+ let DecoderMethod = "decodeMemIndex";
+ let ParserMatchClass = AGXMemIndexAsm;
+}
+
+// Threadgroup memory base: 16-bit register, 16-bit uniform or zero.
+def AGXLocalBase : Operand<i32> {
+ let PrintMethod = "printLocalBase";
+ let EncoderMethod = "getLocalBaseEncoding";
+ let DecoderMethod = "decodeLocalBase";
+ let ParserMatchClass = AGXLocalBaseAsm;
+}
+
+// Threadgroup memory index: 16-bit register or 16-bit immediate.
+def AGXLocalIndex : Operand<i32> {
+ let PrintMethod = "printLocalIndex";
+ let EncoderMethod = "getLocalIndexEncoding";
+ let DecoderMethod = "decodeLocalIndex";
+ let ParserMatchClass = AGXLocalIndexAsm;
+}
+
+// Atomic data source: 32-bit register (64-bit pair for cmpxchg).
+def AGXAtomicSrc : Operand<i32> {
+ let PrintMethod = "printAtomicSrc";
+ let EncoderMethod = "getAtomicSrcEncoding";
+ let DecoderMethod = "decodeAtomicSrc";
+ let ParserMatchClass = AGXAtomicSrcAsm;
+}
+
+//===----------------------------------------------------------------------===//
+// Branch targets
+//===----------------------------------------------------------------------===//
+
+def AGXBrTarget : Operand<OtherVT> {
+ let PrintMethod = "printBrTarget";
+ let EncoderMethod = "getBrTargetEncoding";
+ let DecoderMethod = "decodeBrTarget";
+ let OperandType = "OPERAND_PCREL";
+ let ParserMatchClass = AGXBrTargetAsm;
+}
+
+// The implicit nesting counter operand of execution mask instructions. The
+// immediate is its cache hint; it is printed as r0l.
+def AGXNestCounter : Operand<i32> {
+ let PrintMethod = "printNestCounter";
+ let ParserMatchClass = AGXNestCounterAsm;
+ let DecoderMethod = "decodeImm";
+}
+
+//===----------------------------------------------------------------------===//
+// Pseudo instruction register operands
+//===----------------------------------------------------------------------===//
+
+class AGXRegOperand<RegisterClass rc, string type, string ns = "AGX">
+ : RegisterOperand<rc> {
+ let OperandNamespace = ns;
+ let OperandType = type;
+}
+
+// ALU sources accept general purpose registers, uniform registers and,
+// after operand folding, 8-bit immediates.
+def Src16Op : AGXRegOperand<SRC16, "OPERAND_SRC">;
+def Src32Op : AGXRegOperand<SRC32, "OPERAND_SRC">;
+def Src64Op : AGXRegOperand<SRC64, "OPERAND_SRC">;
+def Base64Op : AGXRegOperand<BASE64, "OPERAND_REGISTER", "MCOI">;
diff --git a/llvm/lib/Target/AGX/AGXProcessors.td b/llvm/lib/Target/AGX/AGXProcessors.td
new file mode 100644
index 0000000000000..2fb868e567833
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXProcessors.td
@@ -0,0 +1,50 @@
+//===-- AGXProcessors.td - AGX processor definitions -------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Processor names follow the GPU names used by the Asahi Linux driver
+// (agx-<generation><variant>). The variant letter is G (base), S (Pro),
+// C (Max), D (Ultra) and P (A-series phone/tablet parts).
+//
+//===----------------------------------------------------------------------===//
+
+class AGXProc<string Name, SchedMachineModel Model,
+ list<SubtargetFeature> Features>
+ : ProcessorModel<Name, Model, Features>;
+
+// Generic target: the G13 base feature set.
+def : AGXProc<"generic", AGXG13Model, [FeatureG13]>;
+
+// G13: Apple M1 family.
+def : AGXProc<"agx-g13g", AGXG13Model, [FeatureG13]>; // M1
+def : AGXProc<"agx-g13s", AGXG13Model, // M1 Pro
+ [FeatureG13, FeatureMultiDieCoherency]>;
+def : AGXProc<"agx-g13c", AGXG13Model, // M1 Max
+ [FeatureG13, FeatureMultiDieCoherency]>;
+def : AGXProc<"agx-g13d", AGXG13Model, // M1 Ultra
+ [FeatureG13, FeatureMultiDieCoherency]>;
+
+// G14: Apple M2 family. The scheduling model is measured on G14C.
+def : AGXProc<"agx-g14g", AGXG14Model, [FeatureG14]>; // M2
+def : AGXProc<"agx-g14s", AGXG14Model, [FeatureG14]>; // M2 Pro
+def : AGXProc<"agx-g14c", AGXG14Model, [FeatureG14]>; // M2 Max
+def : AGXProc<"agx-g14d", AGXG14Model, [FeatureG14]>; // M2 Ultra
+
+// G15: Apple M3 family.
+def : AGXProc<"agx-g15g", AGXG14Model, [FeatureG15]>; // M3
+def : AGXProc<"agx-g15s", AGXG14Model, [FeatureG15]>; // M3 Pro
+def : AGXProc<"agx-g15c", AGXG14Model, [FeatureG15]>; // M3 Max
+def : AGXProc<"agx-g15d", AGXG14Model, [FeatureG15]>; // M3 Ultra
+
+// G16: Apple M4 family.
+def : AGXProc<"agx-g16g", AGXApple9Model, [FeatureG16]>; // M4
+def : AGXProc<"agx-g16s", AGXApple9Model, [FeatureG16]>; // M4 Pro
+def : AGXProc<"agx-g16c", AGXApple9Model, [FeatureG16]>; // M4 Max
+
+// G17: Apple A18 Pro and M5.
+def : AGXProc<"agx-g17p", AGXApple9Model, [FeatureG17]>; // A18 Pro
+def : AGXProc<"agx-g17g", AGXApple9Model, [FeatureG17]>; // M5
diff --git a/llvm/lib/Target/AGX/AGXRegisterInfo.cpp b/llvm/lib/Target/AGX/AGXRegisterInfo.cpp
new file mode 100644
index 0000000000000..ecdbe6a0fa84e
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXRegisterInfo.cpp
@@ -0,0 +1,98 @@
+//===-- AGXRegisterInfo.cpp - AGX register information --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AGXRegisterInfo.h"
+#include "AGXSubtarget.h"
+#include "MCTargetDesc/AGXMCTargetDesc.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+
+using namespace llvm;
+
+#define GET_REGINFO_TARGET_DESC
+#include "AGXGenRegisterInfo.inc"
+
+AGXRegisterInfo::AGXRegisterInfo(const AGXSubtarget &ST)
+ : AGXGenRegisterInfo(AGX::R1), ST(ST) {}
+
+const MCPhysReg *
+AGXRegisterInfo::getCalleeSavedRegs(const MachineFunction *) const {
+ return CSR_AGX_NoRegs_SaveList;
+}
+
+const uint32_t *AGXRegisterInfo::getCallPreservedMask(const MachineFunction &,
+ CallingConv::ID) const {
+ return CSR_AGX_NoRegs_RegMask;
+}
+
+BitVector AGXRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
+ BitVector Reserved(getNumRegs());
+ auto ReserveWithAliases = [&](MCRegister Reg) {
+ for (MCRegAliasIterator AI(Reg, this, /*IncludeSelf=*/true); AI.isValid();
+ ++AI)
+ Reserved.set((*AI).id());
+ };
+
+ // r0l is the execution mask nesting counter. r0h is left alone so that
+ // r0 is never clobbered as a whole.
+ ReserveWithAliases(AGX::R0);
+
+ // Registers beyond the subtarget's register file.
+ for (unsigned I = ST.getMaxGPRs(); I < 128; ++I)
+ ReserveWithAliases(AGX::GPR32RegClass.getRegister(I));
+
+ // Uniform registers are never allocated.
+ for (MCRegister Reg : AGX::UGPR16RegClass)
+ ReserveWithAliases(Reg);
+ return Reserved;
+}
+
+bool AGXRegisterInfo::isUniformClass(const TargetRegisterClass *RC) {
+ return AGX::UGPR16RegClass.hasSubClassEq(RC) ||
+ AGX::UGPR32RegClass.hasSubClassEq(RC) ||
+ AGX::UGPR64RegClass.hasSubClassEq(RC);
+}
+
+const TargetRegisterClass *
+AGXRegisterInfo::getGPRClassForSize(unsigned Bits) const {
+ switch (Bits) {
+ case 16:
+ return &AGX::GPR16RegClass;
+ case 32:
+ return &AGX::GPR32RegClass;
+ case 64:
+ return &AGX::GPR32x2RegClass;
+ case 96:
+ return &AGX::GPR96RegClass;
+ case 128:
+ return &AGX::GPR128RegClass;
+ default:
+ return nullptr;
+ }
+}
+
+Register AGXRegisterInfo::getFrameRegister(const MachineFunction &) const {
+ // Stack accesses address the thread's stack directly; there is no frame
+ // register.
+ return Register();
+}
+
+bool AGXRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
+ int SPAdj, unsigned FIOperandNum,
+ RegScavenger *RS) const {
+ MachineInstr &MI = *II;
+ MachineFunction &MF = *MI.getMF();
+ MachineOperand &FIOp = MI.getOperand(FIOperandNum);
+ int FI = FIOp.getIndex();
+ int64_t Offset = MF.getFrameInfo().getObjectOffset(FI);
+ if (Offset < 0 || Offset > 0x7fff)
+ report_fatal_error("AGX stack frame too large");
+ FIOp.ChangeToImmediate(Offset);
+ return false;
+}
diff --git a/llvm/lib/Target/AGX/AGXRegisterInfo.h b/llvm/lib/Target/AGX/AGXRegisterInfo.h
new file mode 100644
index 0000000000000..10c745ae2fa35
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXRegisterInfo.h
@@ -0,0 +1,51 @@
+//===-- AGXRegisterInfo.h - AGX register information ------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AGX_AGXREGISTERINFO_H
+#define LLVM_LIB_TARGET_AGX_AGXREGISTERINFO_H
+
+#define GET_REGINFO_HEADER
+#include "AGXGenRegisterInfo.inc"
+
+namespace llvm {
+
+class AGXSubtarget;
+
+class AGXRegisterInfo : public AGXGenRegisterInfo {
+ const AGXSubtarget &ST;
+
+public:
+ explicit AGXRegisterInfo(const AGXSubtarget &ST);
+
+ const MCPhysReg *getCalleeSavedRegs(const MachineFunction *MF) const override;
+ const uint32_t *getCallPreservedMask(const MachineFunction &MF,
+ CallingConv::ID CC) const override;
+ BitVector getReservedRegs(const MachineFunction &MF) const override;
+
+ bool requiresRegisterScavenging(const MachineFunction &MF) const override {
+ return false;
+ }
+ bool requiresFrameIndexScavenging(const MachineFunction &MF) const override {
+ return false;
+ }
+
+ bool eliminateFrameIndex(MachineBasicBlock::iterator MI, int SPAdj,
+ unsigned FIOperandNum,
+ RegScavenger *RS = nullptr) const override;
+
+ Register getFrameRegister(const MachineFunction &MF) const override;
+
+ /// The general purpose register class of the same size as \p RC.
+ const TargetRegisterClass *getGPRClassForSize(unsigned Bits) const;
+
+ static bool isUniformClass(const TargetRegisterClass *RC);
+};
+
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AGX_AGXREGISTERINFO_H
diff --git a/llvm/lib/Target/AGX/AGXRegisterInfo.td b/llvm/lib/Target/AGX/AGXRegisterInfo.td
new file mode 100644
index 0000000000000..cc1791a38ba7e
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXRegisterInfo.td
@@ -0,0 +1,138 @@
+//===-- AGXRegisterInfo.td - AGX register definitions ------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The AGX register file is addressed in 16-bit units ("half registers").
+// A 32-bit register rN is the pair of halves rNl (low) and rNh (high), and a
+// 64-bit register is a pair of 32-bit registers. The encoding value of every
+// register is its first half register index; instructions carry the access
+// size separately.
+//
+// General purpose registers (r) are per thread. Uniform registers (u) are
+// shared by the SIMD-group, read-only for ALU instructions, and preloaded by
+// the driver or written with uniform_store.
+//
+//===----------------------------------------------------------------------===//
+
+class AGXReg<string n, bits<16> enc, list<Register> subregs = []>
+ : Register<n> {
+ let Namespace = "AGX";
+ let HWEncoding = enc;
+ let SubRegs = subregs;
+}
+
+let Namespace = "AGX" in {
+def lo16 : SubRegIndex<16, 0>;
+def hi16 : SubRegIndex<16, 16>;
+def sub0 : SubRegIndex<32, 0>;
+def sub1 : SubRegIndex<32, 32>;
+def sub2 : SubRegIndex<32, 64>;
+def sub3 : SubRegIndex<32, 96>;
+def sub0_sub1 : SubRegIndex<64, 0>;
+def sub1_sub2 : SubRegIndex<64, 32>;
+def sub2_sub3 : SubRegIndex<64, 64>;
+def sub0_sub1_sub2 : SubRegIndex<96, 0>;
+def sub1_sub2_sub3 : SubRegIndex<96, 32>;
+}
+
+//===----------------------------------------------------------------------===//
+// General purpose registers
+//===----------------------------------------------------------------------===//
+
+foreach Index = 0...127 in {
+ def R#Index#L : AGXReg<"r"#Index#"l", !mul(Index, 2)>;
+ def R#Index#H : AGXReg<"r"#Index#"h", !add(!mul(Index, 2), 1)>;
+
+ let SubRegIndices = [lo16, hi16], CoveredBySubRegs = 1 in
+ def R#Index : AGXReg<"r"#Index, !mul(Index, 2),
+ [!cast<Register>("R"#Index#"L"),
+ !cast<Register>("R"#Index#"H")]>;
+}
+
+//===----------------------------------------------------------------------===//
+// Uniform registers
+//===----------------------------------------------------------------------===//
+
+foreach Index = 0...255 in {
+ def U#Index#L : AGXReg<"u"#Index#"l", !mul(Index, 2)>;
+ def U#Index#H : AGXReg<"u"#Index#"h", !add(!mul(Index, 2), 1)>;
+
+ let SubRegIndices = [lo16, hi16], CoveredBySubRegs = 1 in
+ def U#Index : AGXReg<"u"#Index, !mul(Index, 2),
+ [!cast<Register>("U"#Index#"L"),
+ !cast<Register>("U"#Index#"H")]>;
+}
+
+//===----------------------------------------------------------------------===//
+// Register classes
+//===----------------------------------------------------------------------===//
+
+def GPR16 : RegisterClass<"AGX", [i16, f16, bf16], 16,
+ (interleave (sequence "R%uL", 0, 127),
+ (sequence "R%uH", 0, 127))>;
+
+def GPR32 : RegisterClass<"AGX", [i32, f32, v2i16, v2f16, v2bf16], 32,
+ (sequence "R%u", 0, 127)>;
+
+def UGPR16 : RegisterClass<"AGX", [i16, f16, bf16], 16,
+ (interleave (sequence "U%uL", 0, 255),
+ (sequence "U%uH", 0, 255))> {
+ let isAllocatable = 0;
+}
+
+def UGPR32 : RegisterClass<"AGX", [i32, f32, v2i16, v2f16, v2bf16], 32,
+ (sequence "U%u", 0, 255)> {
+ let isAllocatable = 0;
+}
+
+// Consecutive register tuples. Vector memory operations accept any start
+// register; 64-bit scalar ALU operands require an even first register.
+def GPR32x2Tuples : RegisterTuples<[sub0, sub1],
+ [(sequence "R%u", 0, 126),
+ (sequence "R%u", 1, 127)]>;
+def GPR32x3Tuples : RegisterTuples<[sub0, sub1, sub2],
+ [(sequence "R%u", 0, 125),
+ (sequence "R%u", 1, 126),
+ (sequence "R%u", 2, 127)]>;
+def GPR32x4Tuples : RegisterTuples<[sub0, sub1, sub2, sub3],
+ [(sequence "R%u", 0, 124),
+ (sequence "R%u", 1, 125),
+ (sequence "R%u", 2, 126),
+ (sequence "R%u", 3, 127)]>;
+def UGPR32x2Tuples : RegisterTuples<[sub0, sub1],
+ [(sequence "U%u", 0, 254),
+ (sequence "U%u", 1, 255)]>;
+
+def GPR32x2 : RegisterClass<"AGX", [i64, f64, v2i32, v2f32, v4i16, v4f16],
+ 32, (add GPR32x2Tuples)>;
+def GPR64 : RegisterClass<"AGX", [i64, f64, v2i32, v2f32, v4i16, v4f16],
+ 64, (decimate GPR32x2Tuples, 2)>;
+def GPR96 : RegisterClass<"AGX", [v3i32, v3f32], 32, (add GPR32x3Tuples)>;
+def GPR128 : RegisterClass<"AGX", [v4i32, v4f32, v2i64, v2f64], 32,
+ (add GPR32x4Tuples)>;
+
+def UGPR64 : RegisterClass<"AGX", [i64, f64, v2i32, v2f32], 32,
+ (add UGPR32x2Tuples)> {
+ let isAllocatable = 0;
+}
+
+// ALU source classes: general purpose registers or uniform registers.
+def SRC16 : RegisterClass<"AGX", [i16, f16, bf16], 16, (add GPR16, UGPR16)> {
+ let isAllocatable = 0;
+}
+def SRC32 : RegisterClass<"AGX", [i32, f32, v2i16, v2f16, v2bf16], 32,
+ (add GPR32, UGPR32)> {
+ let isAllocatable = 0;
+}
+def SRC64 : RegisterClass<"AGX", [i64, f64, v2i32, v2f32], 32,
+ (add GPR64, UGPR64)> {
+ let isAllocatable = 0;
+}
+// 64-bit memory base addresses: general purpose or uniform register pairs.
+def BASE64 : RegisterClass<"AGX", [i64], 32, (add GPR32x2, UGPR64)> {
+ let isAllocatable = 0;
+}
diff --git a/llvm/lib/Target/AGX/AGXSchedule.td b/llvm/lib/Target/AGX/AGXSchedule.td
new file mode 100644
index 0000000000000..88a9f379ea00a
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXSchedule.td
@@ -0,0 +1,108 @@
+//===-- AGXSchedule.td - AGX scheduling models -------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Latencies are in core cycles, for a single SIMD-group. The G14 numbers are
+// measurements on G14C (M2 Max) under Asahi Linux, see AGXUsage.rst for the
+// methodology and the source data set. G13 reuses them; the Apple9 model is a
+// placeholder with the same values.
+//
+//===----------------------------------------------------------------------===//
+
+// Simple integer ALU: iadd, bitop, mov.
+def WriteALU : SchedWrite;
+// 64-bit iadd and 32x32->64 imadd.
+def WriteALU64 : SchedWrite;
+// imadd, bit field, shift, bit count, convert.
+def WriteIMul : SchedWrite;
+// fadd, fmul, fmadd (32 and 16 bit), cmpsel.
+def WriteFP : SchedWrite;
+// floor, ceil, trunc, rint, sin_pt_1.
+def WriteFRound : SchedWrite;
+// rcp, log2, exp2, sin_pt_2.
+def WriteSFU : SchedWrite;
+// rsqrt, rsqrt_special.
+def WriteSFUSlow : SchedWrite;
+// mov_imm.
+def WriteMovImm : SchedWrite;
+// get_sr.
+def WriteSR : SchedWrite;
+// simd_shuffle*, ballots.
+def WriteShuffle : SchedWrite;
+// quad_shuffle*.
+def WriteQuadShuffle : SchedWrite;
+// simd reductions and scans.
+def WriteSIMDReduce : SchedWrite;
+// quad reductions and scans.
+def WriteQuadReduce : SchedWrite;
+// device memory and stack loads.
+def WriteDeviceLoad : SchedWrite;
+def WriteDeviceStore : SchedWrite;
+// threadgroup memory.
+def WriteLocalLoad : SchedWrite;
+def WriteLocalStore : SchedWrite;
+def WriteAtomic : SchedWrite;
+// Control flow, barriers and waits.
+def WriteControl : SchedWrite;
+def WriteBarrier : SchedWrite;
+
+class AGXModel : SchedMachineModel {
+ let MicroOpBufferSize = 0;
+ let IssueWidth = 1;
+ let LoadLatency = 46;
+ let MispredictPenalty = 0;
+ let CompleteModel = 0;
+ let PostRAScheduler = 1;
+}
+
+def AGXG14Model : AGXModel;
+def AGXG13Model : AGXModel;
+def AGXApple9Model : AGXModel;
+
+// Execution resources of one scheduler of a core.
+let SchedModel = AGXG14Model in {
+def G14ALU : ProcResource<1>;
+def G14SFU : ProcResource<1>;
+def G14Mem : ProcResource<1>;
+}
+let SchedModel = AGXG13Model in {
+def G13ALU : ProcResource<1>;
+def G13SFU : ProcResource<1>;
+def G13Mem : ProcResource<1>;
+}
+let SchedModel = AGXApple9Model in {
+def A9ALU : ProcResource<1>;
+def A9SFU : ProcResource<1>;
+def A9Mem : ProcResource<1>;
+}
+
+multiclass AGXWrites<ProcResource ALU, ProcResource SFU, ProcResource Mem> {
+ def : WriteRes<WriteALU, [ALU]> { let Latency = 2; }
+ def : WriteRes<WriteALU64, [SFU]> { let Latency = 5; let ReleaseAtCycles = [4]; }
+ def : WriteRes<WriteIMul, [SFU]> { let Latency = 3; let ReleaseAtCycles = [2]; }
+ def : WriteRes<WriteFP, [ALU]> { let Latency = 2; }
+ def : WriteRes<WriteFRound, [SFU]> { let Latency = 3; let ReleaseAtCycles = [2]; }
+ def : WriteRes<WriteSFU, [SFU]> { let Latency = 5; let ReleaseAtCycles = [2]; }
+ def : WriteRes<WriteSFUSlow, [SFU]> { let Latency = 6; let ReleaseAtCycles = [4]; }
+ def : WriteRes<WriteMovImm, [ALU]> { let Latency = 1; }
+ def : WriteRes<WriteSR, [ALU]> { let Latency = 3; }
+ def : WriteRes<WriteShuffle, [SFU]> { let Latency = 5; let ReleaseAtCycles = [2]; }
+ def : WriteRes<WriteQuadShuffle, [ALU]> { let Latency = 4; }
+ def : WriteRes<WriteSIMDReduce, [SFU]> { let Latency = 24; let ReleaseAtCycles = [24]; }
+ def : WriteRes<WriteQuadReduce, [SFU]> { let Latency = 12; let ReleaseAtCycles = [9]; }
+ def : WriteRes<WriteDeviceLoad, [Mem]> { let Latency = 46; }
+ def : WriteRes<WriteDeviceStore, [Mem]> { let Latency = 3; let ReleaseAtCycles = [3]; }
+ def : WriteRes<WriteLocalLoad, [Mem]> { let Latency = 20; let ReleaseAtCycles = [20]; }
+ def : WriteRes<WriteLocalStore, [Mem]> { let Latency = 18; let ReleaseAtCycles = [18]; }
+ def : WriteRes<WriteAtomic, [Mem]> { let Latency = 60; }
+ def : WriteRes<WriteControl, [ALU]> { let Latency = 2; }
+ def : WriteRes<WriteBarrier, [Mem]> { let Latency = 14; }
+}
+
+let SchedModel = AGXG14Model in defm : AGXWrites<G14ALU, G14SFU, G14Mem>;
+let SchedModel = AGXG13Model in defm : AGXWrites<G13ALU, G13SFU, G13Mem>;
+let SchedModel = AGXApple9Model in defm : AGXWrites<A9ALU, A9SFU, A9Mem>;
diff --git a/llvm/lib/Target/AGX/AGXSearchableTables.td b/llvm/lib/Target/AGX/AGXSearchableTables.td
new file mode 100644
index 0000000000000..922717d7dc47c
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXSearchableTables.td
@@ -0,0 +1,18 @@
+//===-- AGXSearchableTables.td - AGX lookup tables ---------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+include "llvm/TableGen/SearchableTable.td"
+
+// Pseudo instruction to real instruction, per encoding family.
+def AGXPseudoToRealTable : GenericTable {
+ let FilterClass = "AGXPseudo";
+ let CppTypeName = "AGXPseudoToReal";
+ let Fields = ["Pseudo", "G13Real", "Apple9Real"];
+ let PrimaryKey = ["Pseudo"];
+ let PrimaryKeyName = "getAGXPseudoToReal";
+}
diff --git a/llvm/lib/Target/AGX/AGXSelectionDAGInfo.cpp b/llvm/lib/Target/AGX/AGXSelectionDAGInfo.cpp
new file mode 100644
index 0000000000000..56e6df69a4f1e
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXSelectionDAGInfo.cpp
@@ -0,0 +1,17 @@
+//===-- AGXSelectionDAGInfo.cpp - AGX SelectionDAG info -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AGXSelectionDAGInfo.h"
+
+#define GET_SDNODE_DESC
+#include "AGXGenSDNodeInfo.inc"
+
+using namespace llvm;
+
+AGXSelectionDAGInfo::AGXSelectionDAGInfo()
+ : SelectionDAGGenTargetInfo(AGXGenSDNodeInfo) {}
diff --git a/llvm/lib/Target/AGX/AGXSelectionDAGInfo.h b/llvm/lib/Target/AGX/AGXSelectionDAGInfo.h
new file mode 100644
index 0000000000000..53c7e711ff2f0
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXSelectionDAGInfo.h
@@ -0,0 +1,26 @@
+//===-- AGXSelectionDAGInfo.h - AGX SelectionDAG info -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AGX_AGXSELECTIONDAGINFO_H
+#define LLVM_LIB_TARGET_AGX_AGXSELECTIONDAGINFO_H
+
+#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
+
+#define GET_SDNODE_ENUM
+#include "AGXGenSDNodeInfo.inc"
+
+namespace llvm {
+
+class AGXSelectionDAGInfo : public SelectionDAGGenTargetInfo {
+public:
+ AGXSelectionDAGInfo();
+};
+
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AGX_AGXSELECTIONDAGINFO_H
diff --git a/llvm/lib/Target/AGX/AGXSubtarget.cpp b/llvm/lib/Target/AGX/AGXSubtarget.cpp
new file mode 100644
index 0000000000000..1041f62304235
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXSubtarget.cpp
@@ -0,0 +1,33 @@
+//===-- AGXSubtarget.cpp - AGX subtarget information ----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AGXSubtarget.h"
+#include "AGXTargetMachine.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "agx-subtarget"
+
+#define GET_SUBTARGETINFO_TARGET_DESC
+#define GET_SUBTARGETINFO_CTOR
+#include "AGXGenSubtargetInfo.inc"
+
+AGXSubtarget &AGXSubtarget::initializeSubtargetDependencies(StringRef CPU,
+ StringRef FS) {
+ if (CPU.empty())
+ CPU = "generic";
+ ParseSubtargetFeatures(CPU, /*TuneCPU=*/CPU, FS);
+ return *this;
+}
+
+AGXSubtarget::AGXSubtarget(const Triple &TT, StringRef CPU, StringRef FS,
+ const AGXTargetMachine &TM)
+ : AGXGenSubtargetInfo(TT, CPU.empty() ? "generic" : CPU,
+ /*TuneCPU=*/CPU.empty() ? "generic" : CPU, FS),
+ InstrInfo(initializeSubtargetDependencies(CPU, FS)),
+ FrameLowering(*this), TLInfo(TM, *this) {}
diff --git a/llvm/lib/Target/AGX/AGXSubtarget.h b/llvm/lib/Target/AGX/AGXSubtarget.h
new file mode 100644
index 0000000000000..ebbc4aaa05eaf
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXSubtarget.h
@@ -0,0 +1,89 @@
+//===-- AGXSubtarget.h - Define subtarget for AGX ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The AGX subtarget: generation, encoding family and capability features.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AGX_AGXSUBTARGET_H
+#define LLVM_LIB_TARGET_AGX_AGXSUBTARGET_H
+
+#include "AGXFrameLowering.h"
+#include "AGXISelLowering.h"
+#include "AGXInstrInfo.h"
+#include "AGXSelectionDAGInfo.h"
+#include "MCTargetDesc/AGXMCTargetDesc.h"
+#include "llvm/CodeGen/TargetSubtargetInfo.h"
+
+#define GET_SUBTARGETINFO_HEADER
+#include "AGXGenSubtargetInfo.inc"
+
+namespace llvm {
+
+class AGXTargetMachine;
+
+class AGXSubtarget : public AGXGenSubtargetInfo {
+public:
+ enum Generation { Unknown, G13, G14, G15, G16, G17 };
+
+ /// Instruction encoding families, selecting the real instructions that
+ /// pseudo instructions lower to.
+ enum EncodingFamily { AGX2, Apple9 };
+
+private:
+ Generation Gen = Unknown;
+ unsigned MaxGPRs = 0;
+
+#define GET_SUBTARGETINFO_MACRO(ATTRIBUTE, DEFAULT, GETTER) \
+ bool ATTRIBUTE = DEFAULT;
+#include "AGXGenSubtargetInfo.inc"
+
+ AGXInstrInfo InstrInfo;
+ AGXFrameLowering FrameLowering;
+ AGXTargetLowering TLInfo;
+ AGXSelectionDAGInfo TSInfo;
+
+public:
+ AGXSubtarget(const Triple &TT, StringRef CPU, StringRef FS,
+ const AGXTargetMachine &TM);
+
+ AGXSubtarget &initializeSubtargetDependencies(StringRef CPU, StringRef FS);
+ void ParseSubtargetFeatures(StringRef CPU, StringRef TuneCPU, StringRef FS);
+
+#define GET_SUBTARGETINFO_MACRO(ATTRIBUTE, DEFAULT, GETTER) \
+ bool GETTER() const { return ATTRIBUTE; }
+#include "AGXGenSubtargetInfo.inc"
+
+ Generation getGeneration() const { return Gen; }
+ EncodingFamily getEncodingFamily() const {
+ return HasApple9Encoding ? Apple9 : AGX2;
+ }
+ /// The number of addressable 32-bit general purpose registers.
+ unsigned getMaxGPRs() const { return MaxGPRs ? MaxGPRs : 128; }
+
+ const AGXInstrInfo *getInstrInfo() const override { return &InstrInfo; }
+ const AGXFrameLowering *getFrameLowering() const override {
+ return &FrameLowering;
+ }
+ const AGXTargetLowering *getTargetLowering() const override {
+ return &TLInfo;
+ }
+ const AGXRegisterInfo *getRegisterInfo() const override {
+ return &InstrInfo.getRegisterInfo();
+ }
+ const AGXSelectionDAGInfo *getSelectionDAGInfo() const override {
+ return &TSInfo;
+ }
+
+ bool enableMachineScheduler() const override { return true; }
+ bool enablePostRAScheduler() const override { return false; }
+};
+
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AGX_AGXSUBTARGET_H
diff --git a/llvm/lib/Target/AGX/AGXTargetMachine.cpp b/llvm/lib/Target/AGX/AGXTargetMachine.cpp
new file mode 100644
index 0000000000000..a472d9ef76fdc
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXTargetMachine.cpp
@@ -0,0 +1,178 @@
+//===-- AGXTargetMachine.cpp - Define TargetMachine for AGX ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The AGX code generation pipeline. Like AMDGPU, all calls are inlined and
+// the control flow graph is structurized before instruction selection.
+// Unlike AMDGPU, AGX has no scalar unit and no scalar branches: every
+// conditional branch, uniform or not, is lowered to execution mask
+// operations on the per-thread nesting counter r0l.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AGXTargetMachine.h"
+#include "AGX.h"
+#include "AGXMachineFunctionInfo.h"
+#include "AGXTargetTransformInfo.h"
+#include "TargetInfo/AGXTargetInfo.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/Function.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Transforms/IPO/AlwaysInliner.h"
+#include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Utils.h"
+
+using namespace llvm;
+
+extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAGXTarget() {
+ RegisterTargetMachine<AGXTargetMachine> X(getTheAGXTarget());
+ PassRegistry &PR = *PassRegistry::getPassRegistry();
+ initializeAGXAlwaysInlinePass(PR);
+ initializeAGXCodeGenPreparePass(PR);
+ initializeAGXAnnotateControlFlowPass(PR);
+ initializeAGXDAGToDAGISelLegacyPass(PR);
+ initializeAGXFoldOperandsPass(PR);
+ initializeAGXLowerControlFlowPass(PR);
+ initializeAGXInsertWaitsPass(PR);
+ initializeAGXAsmPrinterPass(PR);
+}
+
+static Reloc::Model getEffectiveRelocModel(std::optional<Reloc::Model> RM) {
+ return RM.value_or(Reloc::PIC_);
+}
+
+AGXTargetMachine::AGXTargetMachine(const Target &T, const Triple &TT,
+ StringRef CPU, StringRef FS,
+ const TargetOptions &Options,
+ std::optional<Reloc::Model> RM,
+ std::optional<CodeModel::Model> CM,
+ CodeGenOptLevel OL, bool JIT)
+ : CodeGenTargetMachineImpl(T, TT, CPU.empty() ? "generic" : CPU, FS,
+ Options,
+ getEffectiveRelocModel(RM),
+ getEffectiveCodeModel(CM, CodeModel::Small),
+ OL),
+ TLOF(std::make_unique<TargetLoweringObjectFileELF>()) {
+ setRequiresStructuredCFG(true);
+ initAsmInfo();
+}
+
+AGXTargetMachine::~AGXTargetMachine() = default;
+
+const AGXSubtarget *
+AGXTargetMachine::getSubtargetImpl(const Function &F) const {
+ Attribute CPUAttr = F.getFnAttribute("target-cpu");
+ Attribute FSAttr = F.getFnAttribute("target-features");
+ std::string CPU =
+ CPUAttr.isValid() ? CPUAttr.getValueAsString().str() : TargetCPU;
+ std::string FS =
+ FSAttr.isValid() ? FSAttr.getValueAsString().str() : TargetFS;
+
+ auto &I = SubtargetMap[CPU + FS];
+ if (!I)
+ I = std::make_unique<AGXSubtarget>(TargetTriple, CPU, FS, *this);
+ return I.get();
+}
+
+TargetTransformInfo
+AGXTargetMachine::getTargetTransformInfo(const Function &F) const {
+ return TargetTransformInfo(std::make_unique<AGXTTIImpl>(this, F));
+}
+
+MachineFunctionInfo *AGXTargetMachine::createMachineFunctionInfo(
+ BumpPtrAllocator &Allocator, const Function &F,
+ const TargetSubtargetInfo *STI) const {
+ return AGXMachineFunctionInfo::create<AGXMachineFunctionInfo>(Allocator, F,
+ STI);
+}
+
+bool AGXTargetMachine::isNoopAddrSpaceCast(const DataLayout &DL, unsigned SrcAS,
+ unsigned DestAS) const {
+ // Generic, device and constant pointers all address device memory.
+ auto IsDevice = [](unsigned AS) {
+ return AS == AGX::GENERIC_ADDRESS || AS == AGX::DEVICE_ADDRESS ||
+ AS == AGX::CONSTANT_ADDRESS;
+ };
+ return IsDevice(SrcAS) && IsDevice(DestAS);
+}
+
+namespace {
+
+class AGXPassConfig : public TargetPassConfig {
+public:
+ AGXPassConfig(AGXTargetMachine &TM, PassManagerBase &PM)
+ : TargetPassConfig(TM, PM) {
+ // There are no calls, no stack unwinding and no shrink wrapping.
+ disablePass(&StackMapLivenessID);
+ disablePass(&FuncletLayoutID);
+ disablePass(&PatchableFunctionID);
+ disablePass(&ShrinkWrapID);
+ disablePass(&PostRAMachineSinkingID);
+ }
+
+ AGXTargetMachine &getAGXTargetMachine() const {
+ return getTM<AGXTargetMachine>();
+ }
+
+ void addIRPasses() override;
+ bool addPreISel() override;
+ bool addInstSelector() override;
+ void addMachineSSAOptimization() override;
+ void addPreEmitPass() override;
+};
+
+} // namespace
+
+TargetPassConfig *AGXTargetMachine::createPassConfig(PassManagerBase &PM) {
+ return new AGXPassConfig(*this, PM);
+}
+
+void AGXPassConfig::addIRPasses() {
+ // There are no function calls: inline everything into the kernels.
+ addPass(createAGXAlwaysInlinePass());
+ addPass(createAlwaysInlinerLegacyPass());
+
+ addPass(createAtomicExpandLegacyPass());
+ if (getOptLevel() != CodeGenOptLevel::None) {
+ addPass(createInferAddressSpacesPass());
+ addPass(createSROAPass());
+ }
+ addPass(createAGXCodeGenPreparePass());
+
+ TargetPassConfig::addIRPasses();
+}
+
+bool AGXPassConfig::addPreISel() {
+ // Bring the CFG into the structured form AGXAnnotateControlFlow expects.
+ addPass(createLowerSwitchPass());
+ addPass(createFixIrreduciblePass());
+ addPass(createUnifyLoopExitsPass());
+ // Every branch is lowered to execution mask operations, so uniform
+ // regions are structurized too.
+ addPass(createStructurizeCFGPass(/*SkipUniformRegions=*/false));
+ addPass(createAGXAnnotateControlFlowPass());
+ addPass(createLCSSAPass());
+ return false;
+}
+
+bool AGXPassConfig::addInstSelector() {
+ addPass(createAGXISelDag(getAGXTargetMachine()));
+ return false;
+}
+
+void AGXPassConfig::addMachineSSAOptimization() {
+ TargetPassConfig::addMachineSSAOptimization();
+ addPass(createAGXFoldOperandsPass());
+}
+
+void AGXPassConfig::addPreEmitPass() {
+ addPass(createAGXLowerControlFlowPass());
+ addPass(createAGXInsertWaitsPass());
+}
diff --git a/llvm/lib/Target/AGX/AGXTargetMachine.h b/llvm/lib/Target/AGX/AGXTargetMachine.h
new file mode 100644
index 0000000000000..d6fedb9e0273a
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXTargetMachine.h
@@ -0,0 +1,51 @@
+//===-- AGXTargetMachine.h - Define TargetMachine for AGX -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AGX_AGXTARGETMACHINE_H
+#define LLVM_LIB_TARGET_AGX_AGXTARGETMACHINE_H
+
+#include "AGXSubtarget.h"
+#include "llvm/CodeGen/CodeGenTargetMachineImpl.h"
+#include "llvm/ADT/StringMap.h"
+#include <optional>
+
+namespace llvm {
+
+class AGXTargetMachine : public CodeGenTargetMachineImpl {
+ std::unique_ptr<TargetLoweringObjectFile> TLOF;
+ mutable StringMap<std::unique_ptr<AGXSubtarget>> SubtargetMap;
+
+public:
+ AGXTargetMachine(const Target &T, const Triple &TT, StringRef CPU,
+ StringRef FS, const TargetOptions &Options,
+ std::optional<Reloc::Model> RM,
+ std::optional<CodeModel::Model> CM, CodeGenOptLevel OL,
+ bool JIT);
+ ~AGXTargetMachine() override;
+
+ const AGXSubtarget *getSubtargetImpl(const Function &F) const override;
+
+ TargetPassConfig *createPassConfig(PassManagerBase &PM) override;
+
+ TargetLoweringObjectFile *getObjFileLowering() const override {
+ return TLOF.get();
+ }
+
+ TargetTransformInfo getTargetTransformInfo(const Function &F) const override;
+
+ MachineFunctionInfo *
+ createMachineFunctionInfo(BumpPtrAllocator &Allocator, const Function &F,
+ const TargetSubtargetInfo *STI) const override;
+
+ bool isNoopAddrSpaceCast(const DataLayout &DL, unsigned SrcAS,
+ unsigned DestAS) const override;
+};
+
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AGX_AGXTARGETMACHINE_H
diff --git a/llvm/lib/Target/AGX/AGXTargetTransformInfo.cpp b/llvm/lib/Target/AGX/AGXTargetTransformInfo.cpp
new file mode 100644
index 0000000000000..bff65a7c4ff6b
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXTargetTransformInfo.cpp
@@ -0,0 +1,84 @@
+//===-- AGXTargetTransformInfo.cpp - AGX specific TTI ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AGXTargetTransformInfo.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IntrinsicsAGX.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "agxtti"
+
+static bool isThreadVaryingIntrinsic(Intrinsic::ID ID) {
+ switch (ID) {
+ case Intrinsic::agx_thread_position_in_grid_x:
+ case Intrinsic::agx_thread_position_in_grid_y:
+ case Intrinsic::agx_thread_position_in_grid_z:
+ case Intrinsic::agx_thread_position_in_threadgroup_x:
+ case Intrinsic::agx_thread_position_in_threadgroup_y:
+ case Intrinsic::agx_thread_position_in_threadgroup_z:
+ case Intrinsic::agx_thread_index_in_threadgroup:
+ case Intrinsic::agx_thread_index_in_simdgroup:
+ case Intrinsic::agx_simdgroup_index_in_threadgroup:
+ case Intrinsic::agx_active_thread_index_in_simdgroup:
+ case Intrinsic::agx_active_thread_index_in_quadgroup:
+ case Intrinsic::agx_read_sr:
+ case Intrinsic::agx_simd_shuffle:
+ case Intrinsic::agx_simd_shuffle_xor:
+ case Intrinsic::agx_simd_shuffle_up:
+ case Intrinsic::agx_simd_shuffle_down:
+ case Intrinsic::agx_quad_shuffle:
+ case Intrinsic::agx_quad_shuffle_xor:
+ case Intrinsic::agx_quad_shuffle_up:
+ case Intrinsic::agx_quad_shuffle_down:
+ case Intrinsic::agx_simd_prefix:
+ case Intrinsic::agx_quad_prefix:
+ case Intrinsic::agx_quad_reduce:
+ case Intrinsic::agx_quad_ballot:
+ case Intrinsic::agx_dfdx:
+ case Intrinsic::agx_dfdy:
+ case Intrinsic::agx_simdgroup_matrix_fma:
+ return true;
+ default:
+ return false;
+ }
+}
+
+ValueUniformity AGXTTIImpl::getValueUniformity(const Value *V) const {
+ if (const auto *II = dyn_cast<IntrinsicInst>(V)) {
+ if (isThreadVaryingIntrinsic(II->getIntrinsicID()))
+ return ValueUniformity::NeverUniform;
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::agx_simd_ballot:
+ case Intrinsic::agx_simd_reduce:
+ case Intrinsic::agx_total_active_threads_in_simdgroup:
+ case Intrinsic::agx_threadgroup_position_in_grid_x:
+ case Intrinsic::agx_threadgroup_position_in_grid_y:
+ case Intrinsic::agx_threadgroup_position_in_grid_z:
+ case Intrinsic::agx_threads_per_threadgroup_x:
+ case Intrinsic::agx_threads_per_threadgroup_y:
+ case Intrinsic::agx_threads_per_threadgroup_z:
+ case Intrinsic::agx_threadgroups_per_grid_x:
+ case Intrinsic::agx_threadgroups_per_grid_y:
+ case Intrinsic::agx_threadgroups_per_grid_z:
+ return ValueUniformity::AlwaysUniform;
+ default:
+ return ValueUniformity::Default;
+ }
+ }
+
+ // Stack memory is per thread, and atomics return per thread values.
+ if (const auto *LI = dyn_cast<LoadInst>(V))
+ return LI->getPointerAddressSpace() == AGX::PRIVATE_ADDRESS
+ ? ValueUniformity::NeverUniform
+ : ValueUniformity::Default;
+ if (isa<AtomicRMWInst>(V) || isa<AtomicCmpXchgInst>(V) || isa<CallInst>(V))
+ return ValueUniformity::NeverUniform;
+ return ValueUniformity::Default;
+}
diff --git a/llvm/lib/Target/AGX/AGXTargetTransformInfo.h b/llvm/lib/Target/AGX/AGXTargetTransformInfo.h
new file mode 100644
index 0000000000000..7847fe32db96c
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXTargetTransformInfo.h
@@ -0,0 +1,56 @@
+//===-- AGXTargetTransformInfo.h - AGX specific TTI -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AGX_AGXTARGETTRANSFORMINFO_H
+#define LLVM_LIB_TARGET_AGX_AGXTARGETTRANSFORMINFO_H
+
+#include "AGXTargetMachine.h"
+#include "llvm/CodeGen/BasicTTIImpl.h"
+
+namespace llvm {
+
+class AGXTTIImpl final : public BasicTTIImplBase<AGXTTIImpl> {
+ using BaseT = BasicTTIImplBase<AGXTTIImpl>;
+ friend BaseT;
+
+ const AGXSubtarget *ST;
+ const AGXTargetLowering *TLI;
+
+ const AGXSubtarget *getST() const { return ST; }
+ const AGXTargetLowering *getTLI() const { return TLI; }
+
+public:
+ explicit AGXTTIImpl(const AGXTargetMachine *TM, const Function &F)
+ : BaseT(TM, F.getDataLayout()), ST(TM->getSubtargetImpl(F)),
+ TLI(ST->getTargetLowering()) {}
+
+ bool hasBranchDivergence(const Function *F = nullptr) const override {
+ return true;
+ }
+
+ ValueUniformity getValueUniformity(const Value *V) const override;
+
+ unsigned getFlatAddressSpace() const override {
+ return AGX::GENERIC_ADDRESS;
+ }
+
+ unsigned getNumberOfRegisters(unsigned ClassID) const override {
+ return ST->getMaxGPRs();
+ }
+
+ TypeSize
+ getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const override {
+ return TypeSize::getFixed(32);
+ }
+
+ unsigned getInliningThresholdMultiplier() const override { return 11; }
+};
+
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AGX_AGXTARGETTRANSFORMINFO_H
diff --git a/llvm/lib/Target/AGX/AsmParser/AGXAsmParser.cpp b/llvm/lib/Target/AGX/AsmParser/AGXAsmParser.cpp
new file mode 100644
index 0000000000000..995003afe12c6
--- /dev/null
+++ b/llvm/lib/Target/AGX/AsmParser/AGXAsmParser.cpp
@@ -0,0 +1,864 @@
+//===-- AGXAsmParser.cpp - Parse AGX assembly to MCInst instructions ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Accepts the syntax printed by AGXInstPrinter, which follows the applegpu
+// disassembler.
+//
+//===----------------------------------------------------------------------===//
+
+#include "MCTargetDesc/AGXMCTargetDesc.h"
+#include "TargetInfo/AGXTargetInfo.h"
+#include "llvm/ADT/SmallBitVector.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCParser/AsmLexer.h"
+#include "llvm/MC/MCParser/MCParsedAsmOperand.h"
+#include "llvm/MC/MCParser/MCTargetAsmParser.h"
+#include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/MathExtras.h"
+
+using namespace llvm;
+
+namespace {
+
+/// A parsed register: the first register of a tuple and the tuple length.
+struct ParsedReg {
+ AGX::RegKind Kind = AGX::RegKind::Invalid;
+ unsigned HalfIndex = 0;
+ unsigned Count = 1;
+};
+
+class AGXOperand : public MCParsedAsmOperand {
+public:
+ enum KindTy { Token, Register, Immediate, Expression } Kind;
+ /// What an immediate was parsed as.
+ enum ImmKindTy {
+ IK_Int,
+ IK_Float,
+ IK_Shift,
+ IK_BitfieldMask,
+ IK_TruthTable,
+ IK_ICond,
+ IK_FCond,
+ IK_ConvertMode,
+ IK_RoundMode,
+ IK_MemFormat,
+ IK_MemMask,
+ IK_Signedness,
+ IK_AtomicOp,
+ IK_SpecialReg,
+ IK_NestCounter,
+ };
+
+ SMLoc StartLoc, EndLoc;
+ StringRef Tok;
+ MCRegister Reg;
+ AGX::RegKind RegKind = AGX::RegKind::Invalid;
+ unsigned Mods = 0;
+ int64_t Imm = 0;
+ double FPImm = 0;
+ ImmKindTy ImmKind = IK_Int;
+ const MCExpr *Expr = nullptr;
+
+ explicit AGXOperand(KindTy K) : Kind(K) {}
+
+ static std::unique_ptr<AGXOperand> createToken(StringRef Str, SMLoc S) {
+ auto Op = std::make_unique<AGXOperand>(Token);
+ Op->Tok = Str;
+ Op->StartLoc = Op->EndLoc = S;
+ return Op;
+ }
+ static std::unique_ptr<AGXOperand> createReg(MCRegister R, AGX::RegKind K,
+ unsigned Mods, SMLoc S,
+ SMLoc E) {
+ auto Op = std::make_unique<AGXOperand>(Register);
+ Op->Reg = R;
+ Op->RegKind = K;
+ Op->Mods = Mods;
+ Op->StartLoc = S;
+ Op->EndLoc = E;
+ return Op;
+ }
+ static std::unique_ptr<AGXOperand> createImm(int64_t V, ImmKindTy IK,
+ SMLoc S, SMLoc E) {
+ auto Op = std::make_unique<AGXOperand>(Immediate);
+ Op->Imm = V;
+ Op->ImmKind = IK;
+ Op->StartLoc = S;
+ Op->EndLoc = E;
+ return Op;
+ }
+ static std::unique_ptr<AGXOperand> createFPImm(double V, SMLoc S, SMLoc E) {
+ auto Op = createImm(0, IK_Float, S, E);
+ Op->FPImm = V;
+ return Op;
+ }
+ static std::unique_ptr<AGXOperand> createExpr(const MCExpr *Ex, SMLoc S,
+ SMLoc E) {
+ auto Op = std::make_unique<AGXOperand>(Expression);
+ Op->Expr = Ex;
+ Op->StartLoc = S;
+ Op->EndLoc = E;
+ return Op;
+ }
+
+ bool isToken() const override { return Kind == Token; }
+ bool isImm() const override {
+ return Kind == Immediate && ImmKind == IK_Int;
+ }
+ bool isAGXImm() const { return isImm(); }
+ bool isReg() const override { return Kind == Register; }
+ bool isMem() const override { return false; }
+ MCRegister getReg() const override { return Reg; }
+ StringRef getToken() const { return Tok; }
+ SMLoc getStartLoc() const override { return StartLoc; }
+ SMLoc getEndLoc() const override { return EndLoc; }
+ void print(raw_ostream &OS, const MCAsmInfo &) const override {
+ switch (Kind) {
+ case Token:
+ OS << "'" << Tok << "'";
+ break;
+ case Register:
+ OS << "<register " << Reg.id() << ">";
+ break;
+ case Immediate:
+ OS << Imm;
+ break;
+ case Expression:
+ OS << "<expr>";
+ break;
+ }
+ }
+
+ bool isRegKind(std::initializer_list<AGX::RegKind> Kinds) const {
+ return Kind == Register && is_contained(Kinds, RegKind);
+ }
+ bool isIntImm8() const {
+ return Kind == Immediate && ImmKind == IK_Int && isUInt<8>(Imm);
+ }
+ bool hasOnlyMods(unsigned Allowed) const { return (Mods & ~Allowed) == 0; }
+
+ static constexpr unsigned HintMods =
+ AGX::SRC_MOD_CACHE | AGX::SRC_MOD_DISCARD;
+ static constexpr unsigned FPMods = AGX::SRC_MOD_ABS | AGX::SRC_MOD_NEG;
+
+ bool isSrcImpl(std::initializer_list<AGX::RegKind> Kinds,
+ unsigned AllowedMods) const {
+ if (Kind == Immediate)
+ return isIntImm8();
+ return isRegKind(Kinds) && hasOnlyMods(HintMods | AllowedMods);
+ }
+
+ bool isSrc() const {
+ return isSrcImpl({AGX::RegKind::GPR16, AGX::RegKind::GPR32,
+ AGX::RegKind::UGPR16, AGX::RegKind::UGPR32},
+ 0);
+ }
+ bool isSrc64() const {
+ return isSrcImpl({AGX::RegKind::GPR16, AGX::RegKind::GPR32,
+ AGX::RegKind::GPR64, AGX::RegKind::UGPR16,
+ AGX::RegKind::UGPR32},
+ 0);
+ }
+ bool isSrc16() const {
+ return isSrcImpl(
+ {AGX::RegKind::GPR16, AGX::RegKind::UGPR16, AGX::RegKind::UGPR32}, 0);
+ }
+ bool isSrcSX() const {
+ return isSrcImpl({AGX::RegKind::GPR16, AGX::RegKind::GPR32,
+ AGX::RegKind::UGPR16, AGX::RegKind::UGPR32},
+ AGX::SRC_MOD_SX);
+ }
+ bool isSrcSX64() const {
+ return isSrcImpl({AGX::RegKind::GPR16, AGX::RegKind::GPR32,
+ AGX::RegKind::GPR64, AGX::RegKind::UGPR16,
+ AGX::RegKind::UGPR32},
+ AGX::SRC_MOD_SX);
+ }
+ bool isFPImm() const {
+ if (Kind != Immediate)
+ return false;
+ if (ImmKind == IK_Float)
+ return AGX::encodeFPImm8(FPImm).has_value();
+ return isIntImm8();
+ }
+ bool isSrcF() const {
+ if (Kind == Immediate)
+ return isFPImm();
+ return isRegKind({AGX::RegKind::GPR16, AGX::RegKind::GPR32,
+ AGX::RegKind::UGPR16, AGX::RegKind::UGPR32}) &&
+ hasOnlyMods(HintMods | FPMods);
+ }
+ bool isSrcF16() const {
+ if (Kind == Immediate)
+ return isFPImm();
+ return isRegKind({AGX::RegKind::GPR16, AGX::RegKind::UGPR16,
+ AGX::RegKind::UGPR32}) &&
+ hasOnlyMods(HintMods | FPMods);
+ }
+ bool isCmpselSrc() const { return isSrc(); }
+
+ bool isDstImpl(std::initializer_list<AGX::RegKind> Kinds) const {
+ return isRegKind(Kinds) && hasOnlyMods(AGX::SRC_MOD_CACHE);
+ }
+ bool isDst() const {
+ return isDstImpl({AGX::RegKind::GPR16, AGX::RegKind::GPR32});
+ }
+ bool isDst64() const {
+ return isDstImpl(
+ {AGX::RegKind::GPR16, AGX::RegKind::GPR32, AGX::RegKind::GPR64});
+ }
+ bool isDst16() const { return isDstImpl({AGX::RegKind::GPR16}); }
+ bool isDst32() const { return isDstImpl({AGX::RegKind::GPR32}); }
+
+ bool isImmKind(ImmKindTy K) const {
+ return Kind == Immediate && ImmKind == K;
+ }
+ bool isShift() const { return isImmKind(IK_Shift); }
+ bool isBitfieldMask() const { return isImmKind(IK_BitfieldMask); }
+ bool isTruthTable() const { return isImmKind(IK_TruthTable); }
+ bool isICond() const { return isImmKind(IK_ICond); }
+ bool isFCond() const { return isImmKind(IK_FCond); }
+ bool isICond3() const { return isImmKind(IK_ICond) && Imm < 8; }
+ bool isFCond3() const { return isImmKind(IK_FCond) && Imm < 8; }
+ bool isConvertMode() const { return isImmKind(IK_ConvertMode); }
+ bool isRoundMode() const { return isImmKind(IK_RoundMode); }
+ bool isMemFormat() const { return isImmKind(IK_MemFormat); }
+ bool isMemMask() const { return isImmKind(IK_MemMask); }
+ bool isSignedness() const { return isImmKind(IK_Signedness); }
+ bool isAtomicOp() const { return isImmKind(IK_AtomicOp); }
+ bool isSpecialReg() const { return isImmKind(IK_SpecialReg); }
+ bool isNestCounter() const { return isImmKind(IK_NestCounter); }
+
+ bool isMemReg() const {
+ return isRegKind({AGX::RegKind::GPR16, AGX::RegKind::GPR32,
+ AGX::RegKind::GPR64, AGX::RegKind::GPR96,
+ AGX::RegKind::GPR128}) &&
+ !Mods;
+ }
+ bool isAtomicDst() const {
+ return (Kind == Register && !Reg) || isRegKind({AGX::RegKind::GPR32});
+ }
+ bool isAtomicSrc() const {
+ return isRegKind({AGX::RegKind::GPR32, AGX::RegKind::GPR64});
+ }
+ bool isMemBase() const {
+ return isRegKind({AGX::RegKind::GPR64, AGX::RegKind::UGPR64});
+ }
+ bool isMemIndex() const {
+ return (Kind == Immediate && ImmKind == IK_Int &&
+ (isInt<16>(Imm) || isUInt<16>(Imm))) ||
+ isRegKind({AGX::RegKind::GPR32});
+ }
+ bool isLocalBase() const {
+ return (Kind == Immediate && ImmKind == IK_Int && Imm == 0) ||
+ isRegKind({AGX::RegKind::GPR16, AGX::RegKind::UGPR16});
+ }
+ bool isLocalIndex() const {
+ return (Kind == Immediate && ImmKind == IK_Int &&
+ (isInt<16>(Imm) || isUInt<16>(Imm))) ||
+ isRegKind({AGX::RegKind::GPR16});
+ }
+ bool isBrTarget() const {
+ return Kind == Expression || (Kind == Immediate && ImmKind == IK_Int);
+ }
+
+ void addSrcOperands(MCInst &Inst, unsigned N) const {
+ assert(N == 2 && "invalid number of operands");
+ if (Kind == Immediate) {
+ Inst.addOperand(MCOperand::createImm(0));
+ int64_t V = ImmKind == IK_Float ? *AGX::encodeFPImm8(FPImm) : Imm;
+ Inst.addOperand(MCOperand::createImm(V));
+ return;
+ }
+ Inst.addOperand(MCOperand::createImm(Mods));
+ Inst.addOperand(MCOperand::createReg(Reg));
+ }
+ void addDstOperands(MCInst &Inst, unsigned N) const {
+ assert(N == 2 && "invalid number of operands");
+ Inst.addOperand(MCOperand::createReg(Reg));
+ Inst.addOperand(MCOperand::createImm((Mods & AGX::SRC_MOD_CACHE) ? 1 : 0));
+ }
+ void addImmOperands(MCInst &Inst, unsigned N) const {
+ assert(N == 1 && "invalid number of operands");
+ Inst.addOperand(MCOperand::createImm(Imm));
+ }
+ void addRegOperands(MCInst &Inst, unsigned N) const {
+ assert(N == 1 && "invalid number of operands");
+ Inst.addOperand(MCOperand::createReg(Reg));
+ }
+ void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
+ assert(N == 1 && "invalid number of operands");
+ if (Kind == Register)
+ Inst.addOperand(MCOperand::createReg(Reg));
+ else
+ Inst.addOperand(MCOperand::createImm(Imm));
+ }
+ void addBrTargetOperands(MCInst &Inst, unsigned N) const {
+ assert(N == 1 && "invalid number of operands");
+ if (Kind == Expression)
+ Inst.addOperand(MCOperand::createExpr(Expr));
+ else
+ Inst.addOperand(MCOperand::createImm(Imm));
+ }
+};
+
+class AGXAsmParser : public MCTargetAsmParser {
+ MCAsmParser &Parser;
+
+#define GET_ASSEMBLER_HEADER
+#include "AGXGenAsmMatcher.inc"
+
+public:
+ enum AGXMatchResultTy {
+ Match_Dummy = FIRST_TARGET_MATCH_RESULT_TY,
+#define GET_OPERAND_DIAGNOSTIC_TYPES
+#include "AGXGenAsmMatcher.inc"
+ };
+
+private:
+
+ bool parseRegister(MCRegister &Reg, SMLoc &StartLoc, SMLoc &EndLoc) override;
+ ParseStatus tryParseRegister(MCRegister &Reg, SMLoc &StartLoc,
+ SMLoc &EndLoc) override;
+ bool parseInstruction(ParseInstructionInfo &Info, StringRef Name,
+ SMLoc NameLoc, OperandVector &Operands) override;
+ bool matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
+ OperandVector &Operands, MCStreamer &Out,
+ uint64_t &ErrorInfo,
+ bool MatchingInlineAsm) override;
+
+ /// Parse a register name, with an optional tuple ("r0_r1") and flag
+ /// suffixes (".cache", ".neg", ...).
+ bool parseRegName(StringRef Name, ParsedReg &PR, unsigned &Mods);
+ ParseStatus parseRegOperand(OperandVector &Operands, bool AllowFlags);
+ ParseStatus parseIntOperand(OperandVector &Operands,
+ AGXOperand::ImmKindTy IK);
+
+ ParseStatus parseSrc(OperandVector &Operands);
+ ParseStatus parseDst(OperandVector &Operands);
+ ParseStatus parseImm(OperandVector &Operands);
+ ParseStatus parseShift(OperandVector &Operands);
+ ParseStatus parseBitfieldMask(OperandVector &Operands);
+ ParseStatus parseTruthTable(OperandVector &Operands);
+ ParseStatus parseNamedEnum(OperandVector &Operands, AGXOperand::ImmKindTy IK,
+ std::optional<unsigned> (*Lookup)(StringRef));
+ ParseStatus parseICond(OperandVector &Operands);
+ ParseStatus parseFCond(OperandVector &Operands);
+ ParseStatus parseICond3(OperandVector &Operands) {
+ return parseICond(Operands);
+ }
+ ParseStatus parseFCond3(OperandVector &Operands) {
+ return parseFCond(Operands);
+ }
+ ParseStatus parseConvertMode(OperandVector &Operands);
+ ParseStatus parseRoundMode(OperandVector &Operands);
+ ParseStatus parseMemFormat(OperandVector &Operands);
+ ParseStatus parseMemMask(OperandVector &Operands);
+ ParseStatus parseSignedness(OperandVector &Operands);
+ ParseStatus parseAtomicOp(OperandVector &Operands);
+ ParseStatus parseSpecialReg(OperandVector &Operands);
+ ParseStatus parseNestCounter(OperandVector &Operands);
+ ParseStatus parseMemReg(OperandVector &Operands);
+ ParseStatus parseMemIndex(OperandVector &Operands);
+ ParseStatus parseBrTarget(OperandVector &Operands);
+
+ std::unique_ptr<AGXOperand> defaultZeroOperand() const {
+ return AGXOperand::createImm(0, AGXOperand::IK_Int, SMLoc(), SMLoc());
+ }
+
+ const MCRegisterInfo &MRI() { return *getContext().getRegisterInfo(); }
+
+public:
+ AGXAsmParser(const MCSubtargetInfo &STI, MCAsmParser &Parser,
+ const MCInstrInfo &MII)
+ : MCTargetAsmParser(STI, MII), Parser(Parser) {
+ setAvailableFeatures(ComputeAvailableFeatures(STI.getFeatureBits()));
+ }
+};
+
+} // namespace
+
+#define GET_REGISTER_MATCHER
+#define GET_SUBTARGET_FEATURE_NAME
+#define GET_MATCHER_IMPLEMENTATION
+#define GET_MNEMONIC_SPELL_CHECKER
+#include "AGXGenAsmMatcher.inc"
+
+/// Parse "<prefix><number><suffix>" where prefix is r or u and suffix is l,
+/// h or empty.
+static bool parseOneReg(StringRef S, char &File, unsigned &Number,
+ char &Half) {
+ if (S.size() < 2 || (S[0] != 'r' && S[0] != 'u'))
+ return false;
+ File = S[0];
+ S = S.drop_front();
+ Half = 0;
+ if (S.ends_with("l") || S.ends_with("h")) {
+ Half = S.back();
+ S = S.drop_back();
+ }
+ if (S.empty() || S.getAsInteger(10, Number))
+ return false;
+ return true;
+}
+
+bool AGXAsmParser::parseRegName(StringRef Name, ParsedReg &PR,
+ unsigned &Mods) {
+ SmallVector<StringRef, 4> Parts;
+ Name.split(Parts, '.');
+ Mods = 0;
+ for (StringRef Flag : drop_begin(Parts)) {
+ unsigned F = StringSwitch<unsigned>(Flag)
+ .Case("cache", AGX::SRC_MOD_CACHE)
+ .Case("discard", AGX::SRC_MOD_DISCARD)
+ .Case("abs", AGX::SRC_MOD_ABS)
+ .Case("neg", AGX::SRC_MOD_NEG)
+ .Case("sx", AGX::SRC_MOD_SX)
+ .Default(0);
+ if (!F)
+ return false;
+ Mods |= F;
+ }
+
+ SmallVector<StringRef, 4> Elts;
+ Parts[0].split(Elts, '_');
+ char File = 0, Half = 0;
+ unsigned Number = 0;
+ if (!parseOneReg(Elts[0], File, Number, Half))
+ return false;
+ unsigned Limit = File == 'r' ? 128 : 256;
+ if (Number >= Limit)
+ return false;
+ unsigned First = Number * 2 + (Half == 'h' ? 1 : 0);
+ bool Is16 = Half != 0;
+
+ // Tuples must name consecutive registers of the same size.
+ for (unsigned I = 1; I < Elts.size(); ++I) {
+ char F2 = 0, H2 = 0;
+ unsigned N2 = 0;
+ if (!parseOneReg(Elts[I], F2, N2, H2) || F2 != File || (H2 != 0) != Is16)
+ return false;
+ unsigned Idx = N2 * 2 + (H2 == 'h' ? 1 : 0);
+ if (Idx != First + I * (Is16 ? 1 : 2))
+ return false;
+ }
+
+ PR.HalfIndex = First;
+ PR.Count = Elts.size();
+ if (File == 'u') {
+ if (Is16 && PR.Count == 1)
+ PR.Kind = AGX::RegKind::UGPR16;
+ else if (!Is16 && PR.Count == 1)
+ PR.Kind = AGX::RegKind::UGPR32;
+ else if (!Is16 && PR.Count == 2)
+ PR.Kind = AGX::RegKind::UGPR64;
+ else
+ return false;
+ return true;
+ }
+ if (Is16) {
+ PR.Kind = AGX::RegKind::GPR16;
+ return PR.Count <= 4;
+ }
+ switch (PR.Count) {
+ case 1:
+ PR.Kind = AGX::RegKind::GPR32;
+ return true;
+ case 2:
+ PR.Kind = AGX::RegKind::GPR64;
+ return true;
+ case 3:
+ PR.Kind = AGX::RegKind::GPR96;
+ return true;
+ case 4:
+ PR.Kind = AGX::RegKind::GPR128;
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool AGXAsmParser::parseRegister(MCRegister &Reg, SMLoc &StartLoc,
+ SMLoc &EndLoc) {
+ if (!tryParseRegister(Reg, StartLoc, EndLoc).isSuccess())
+ return Error(StartLoc, "invalid register");
+ return false;
+}
+
+ParseStatus AGXAsmParser::tryParseRegister(MCRegister &Reg, SMLoc &StartLoc,
+ SMLoc &EndLoc) {
+ const AsmToken &Tok = getParser().getTok();
+ StartLoc = Tok.getLoc();
+ EndLoc = Tok.getEndLoc();
+ if (Tok.isNot(AsmToken::Identifier))
+ return ParseStatus::NoMatch;
+ ParsedReg PR;
+ unsigned Mods;
+ if (!parseRegName(Tok.getIdentifier(), PR, Mods) || Mods ||
+ (PR.Kind == AGX::RegKind::GPR16 && PR.Count != 1))
+ return ParseStatus::NoMatch;
+ Reg = AGX::getRegister(PR.Kind, PR.HalfIndex, MRI());
+ if (!Reg)
+ return ParseStatus::NoMatch;
+ Parser.Lex();
+ return ParseStatus::Success;
+}
+
+ParseStatus AGXAsmParser::parseRegOperand(OperandVector &Operands,
+ bool AllowFlags) {
+ const AsmToken &Tok = getParser().getTok();
+ if (Tok.isNot(AsmToken::Identifier))
+ return ParseStatus::NoMatch;
+ ParsedReg PR;
+ unsigned Mods;
+ if (!parseRegName(Tok.getIdentifier(), PR, Mods))
+ return ParseStatus::NoMatch;
+ if (Mods && !AllowFlags)
+ return Error(Tok.getLoc(), "unexpected register flags");
+ // Tuples of 16-bit registers only appear as memory operands, which are
+ // parsed by parseMemReg.
+ if (PR.Kind == AGX::RegKind::GPR16 && PR.Count != 1)
+ return Error(Tok.getLoc(), "unexpected register tuple");
+ MCRegister Reg = AGX::getRegister(PR.Kind, PR.HalfIndex, MRI());
+ if (!Reg)
+ return Error(Tok.getLoc(), "invalid register");
+ SMLoc S = Tok.getLoc(), E = Tok.getEndLoc();
+ Parser.Lex();
+ Operands.push_back(AGXOperand::createReg(Reg, PR.Kind, Mods, S, E));
+ return ParseStatus::Success;
+}
+
+ParseStatus AGXAsmParser::parseIntOperand(OperandVector &Operands,
+ AGXOperand::ImmKindTy IK) {
+ SMLoc S = getParser().getTok().getLoc();
+ if (getParser().getTok().isNot(AsmToken::Integer) &&
+ getParser().getTok().isNot(AsmToken::Minus))
+ return ParseStatus::NoMatch;
+ const MCExpr *Ex;
+ SMLoc E;
+ if (getParser().parseExpression(Ex, E))
+ return ParseStatus::Failure;
+ int64_t V;
+ if (!Ex->evaluateAsAbsolute(V))
+ return Error(S, "expected an absolute expression");
+ Operands.push_back(AGXOperand::createImm(V, IK, S, E));
+ return ParseStatus::Success;
+}
+
+ParseStatus AGXAsmParser::parseSrc(OperandVector &Operands) {
+ ParseStatus Res = parseRegOperand(Operands, /*AllowFlags=*/true);
+ if (!Res.isNoMatch())
+ return Res;
+
+ // Integer or floating point immediate.
+ SMLoc S = getParser().getTok().getLoc();
+ bool Neg = false;
+ if (getParser().getTok().is(AsmToken::Minus) &&
+ getLexer().peekTok().is(AsmToken::Real)) {
+ Neg = true;
+ Parser.Lex();
+ }
+ if (getParser().getTok().is(AsmToken::Real)) {
+ double V;
+ if (getParser().getTok().getString().getAsDouble(V))
+ return Error(S, "invalid floating point immediate");
+ SMLoc E = getParser().getTok().getEndLoc();
+ Parser.Lex();
+ Operands.push_back(AGXOperand::createFPImm(Neg ? -V : V, S, E));
+ return ParseStatus::Success;
+ }
+ return parseIntOperand(Operands, AGXOperand::IK_Int);
+}
+
+ParseStatus AGXAsmParser::parseDst(OperandVector &Operands) {
+ return parseRegOperand(Operands, /*AllowFlags=*/true);
+}
+
+ParseStatus AGXAsmParser::parseImm(OperandVector &Operands) {
+ return parseIntOperand(Operands, AGXOperand::IK_Int);
+}
+
+ParseStatus AGXAsmParser::parseShift(OperandVector &Operands) {
+ const AsmToken &Tok = getParser().getTok();
+ if (Tok.isNot(AsmToken::Identifier) || Tok.getIdentifier() != "lsl")
+ return ParseStatus::NoMatch;
+ SMLoc S = Tok.getLoc();
+ Parser.Lex();
+ int64_t V;
+ if (getParser().parseAbsoluteExpression(V))
+ return ParseStatus::Failure;
+ if (V < 0 || V > 7)
+ return Error(S, "shift out of range");
+ Operands.push_back(
+ AGXOperand::createImm(V, AGXOperand::IK_Shift, S, getLexer().getLoc()));
+ return ParseStatus::Success;
+}
+
+ParseStatus AGXAsmParser::parseBitfieldMask(OperandVector &Operands) {
+ const AsmToken &Tok = getParser().getTok();
+ if (Tok.isNot(AsmToken::Identifier) || Tok.getIdentifier() != "mask")
+ return ParseStatus::NoMatch;
+ SMLoc S = Tok.getLoc();
+ Parser.Lex();
+ int64_t V;
+ if (getParser().parseAbsoluteExpression(V))
+ return ParseStatus::Failure;
+ uint64_t Plus1 = uint64_t(V) + 1;
+ if (V <= 0 || !isPowerOf2_64(Plus1) || Log2_64(Plus1) > 32)
+ return Error(S, "mask must be 2^n - 1 with 0 < n <= 32");
+ unsigned M = Log2_64(Plus1);
+ Operands.push_back(AGXOperand::createImm(M == 32 ? 0 : M,
+ AGXOperand::IK_BitfieldMask, S,
+ getLexer().getLoc()));
+ return ParseStatus::Success;
+}
+
+ParseStatus AGXAsmParser::parseTruthTable(OperandVector &Operands) {
+ const AsmToken &Tok = getParser().getTok();
+ if (Tok.isNot(AsmToken::Integer))
+ return ParseStatus::NoMatch;
+ StringRef Str = Tok.getString();
+ if (Str.size() != 4 || Str.find_first_not_of("01") != StringRef::npos)
+ return Error(Tok.getLoc(), "truth table must be four binary digits");
+ unsigned TT = 0;
+ for (unsigned I = 0; I < 4; ++I)
+ TT |= unsigned(Str[I] - '0') << I;
+ SMLoc S = Tok.getLoc(), E = Tok.getEndLoc();
+ Parser.Lex();
+ Operands.push_back(
+ AGXOperand::createImm(TT, AGXOperand::IK_TruthTable, S, E));
+ return ParseStatus::Success;
+}
+
+ParseStatus
+AGXAsmParser::parseNamedEnum(OperandVector &Operands, AGXOperand::ImmKindTy IK,
+ std::optional<unsigned> (*Lookup)(StringRef)) {
+ const AsmToken &Tok = getParser().getTok();
+ SMLoc S = Tok.getLoc(), E = Tok.getEndLoc();
+ if (Tok.is(AsmToken::Integer)) {
+ int64_t V = Tok.getIntVal();
+ Parser.Lex();
+ Operands.push_back(AGXOperand::createImm(V, IK, S, E));
+ return ParseStatus::Success;
+ }
+ if (Tok.isNot(AsmToken::Identifier))
+ return ParseStatus::NoMatch;
+ std::optional<unsigned> V = Lookup(Tok.getIdentifier());
+ if (!V)
+ return ParseStatus::NoMatch;
+ Parser.Lex();
+ Operands.push_back(AGXOperand::createImm(*V, IK, S, E));
+ return ParseStatus::Success;
+}
+
+ParseStatus AGXAsmParser::parseICond(OperandVector &Operands) {
+ return parseNamedEnum(Operands, AGXOperand::IK_ICond, AGX::parseICond);
+}
+
+ParseStatus AGXAsmParser::parseFCond(OperandVector &Operands) {
+ return parseNamedEnum(Operands, AGXOperand::IK_FCond, AGX::parseFCond);
+}
+
+ParseStatus AGXAsmParser::parseConvertMode(OperandVector &Operands) {
+ return parseNamedEnum(Operands, AGXOperand::IK_ConvertMode,
+ AGX::parseConvertMode);
+}
+
+ParseStatus AGXAsmParser::parseRoundMode(OperandVector &Operands) {
+ return parseNamedEnum(Operands, AGXOperand::IK_RoundMode,
+ AGX::parseRoundMode);
+}
+
+ParseStatus AGXAsmParser::parseMemFormat(OperandVector &Operands) {
+ return parseNamedEnum(Operands, AGXOperand::IK_MemFormat,
+ AGX::parseMemFormat);
+}
+
+ParseStatus AGXAsmParser::parseMemMask(OperandVector &Operands) {
+ return parseNamedEnum(Operands, AGXOperand::IK_MemMask, AGX::parseMemMask);
+}
+
+static std::optional<unsigned> parseSignednessName(StringRef Name) {
+ if (Name == "signed")
+ return 0;
+ if (Name == "unsigned")
+ return 1;
+ return std::nullopt;
+}
+
+ParseStatus AGXAsmParser::parseSignedness(OperandVector &Operands) {
+ return parseNamedEnum(Operands, AGXOperand::IK_Signedness,
+ parseSignednessName);
+}
+
+ParseStatus AGXAsmParser::parseAtomicOp(OperandVector &Operands) {
+ return parseNamedEnum(Operands, AGXOperand::IK_AtomicOp, AGX::parseAtomicOp);
+}
+
+ParseStatus AGXAsmParser::parseSpecialReg(OperandVector &Operands) {
+ const AsmToken &Tok = getParser().getTok();
+ if (Tok.isNot(AsmToken::Identifier) || !Tok.getIdentifier().starts_with("sr"))
+ return ParseStatus::NoMatch;
+ unsigned SR;
+ if (Tok.getIdentifier().drop_front(2).getAsInteger(10, SR) || SR > 255)
+ return Error(Tok.getLoc(), "invalid special register");
+ SMLoc S = Tok.getLoc(), E = Tok.getEndLoc();
+ Parser.Lex();
+ // An optional "(name)" annotation, as printed.
+ if (getParser().getTok().is(AsmToken::LParen)) {
+ while (getParser().getTok().isNot(AsmToken::RParen) &&
+ getParser().getTok().isNot(AsmToken::EndOfStatement))
+ Parser.Lex();
+ if (getParser().getTok().is(AsmToken::RParen))
+ Parser.Lex();
+ }
+ Operands.push_back(
+ AGXOperand::createImm(SR, AGXOperand::IK_SpecialReg, S, E));
+ return ParseStatus::Success;
+}
+
+ParseStatus AGXAsmParser::parseNestCounter(OperandVector &Operands) {
+ const AsmToken &Tok = getParser().getTok();
+ if (Tok.isNot(AsmToken::Identifier))
+ return ParseStatus::NoMatch;
+ StringRef Id = Tok.getIdentifier();
+ if (Id != "r0l" && Id != "r0l.cache")
+ return ParseStatus::NoMatch;
+ SMLoc S = Tok.getLoc(), E = Tok.getEndLoc();
+ Parser.Lex();
+ Operands.push_back(AGXOperand::createImm(Id == "r0l.cache" ? 1 : 0,
+ AGXOperand::IK_NestCounter, S, E));
+ return ParseStatus::Success;
+}
+
+ParseStatus AGXAsmParser::parseMemReg(OperandVector &Operands) {
+ const AsmToken &Tok = getParser().getTok();
+ if (Tok.isNot(AsmToken::Identifier))
+ return ParseStatus::NoMatch;
+ SMLoc S = Tok.getLoc(), E = Tok.getEndLoc();
+ if (Tok.getIdentifier() == "_") {
+ Parser.Lex();
+ Operands.push_back(AGXOperand::createReg(MCRegister(), AGX::RegKind::GPR32,
+ 0, S, E));
+ return ParseStatus::Success;
+ }
+ ParsedReg PR;
+ unsigned Mods;
+ if (!parseRegName(Tok.getIdentifier(), PR, Mods) || Mods)
+ return ParseStatus::NoMatch;
+ // Register tuples are kept whole; memory instructions encode their first
+ // register. 16-bit tuples are represented by their first half register.
+ AGX::RegKind Kind = PR.Kind;
+ MCRegister Reg = AGX::getRegister(Kind, PR.HalfIndex, MRI());
+ if (!Reg)
+ return Error(S, "invalid register");
+ Parser.Lex();
+ Operands.push_back(AGXOperand::createReg(Reg, Kind, 0, S, E));
+ return ParseStatus::Success;
+}
+
+ParseStatus AGXAsmParser::parseMemIndex(OperandVector &Operands) {
+ ParseStatus Res = parseRegOperand(Operands, /*AllowFlags=*/false);
+ if (!Res.isNoMatch())
+ return Res;
+ return parseIntOperand(Operands, AGXOperand::IK_Int);
+}
+
+ParseStatus AGXAsmParser::parseBrTarget(OperandVector &Operands) {
+ SMLoc S = getParser().getTok().getLoc();
+ const MCExpr *Ex;
+ SMLoc E;
+ if (getParser().parseExpression(Ex, E))
+ return ParseStatus::Failure;
+ int64_t V;
+ if (Ex->evaluateAsAbsolute(V))
+ Operands.push_back(AGXOperand::createImm(V, AGXOperand::IK_Int, S, E));
+ else
+ Operands.push_back(AGXOperand::createExpr(Ex, S, E));
+ return ParseStatus::Success;
+}
+
+bool AGXAsmParser::parseInstruction(ParseInstructionInfo &Info, StringRef Name,
+ SMLoc NameLoc, OperandVector &Operands) {
+ Operands.push_back(AGXOperand::createToken(Name, NameLoc));
+ if (getLexer().is(AsmToken::EndOfStatement)) {
+ Parser.Lex();
+ return false;
+ }
+
+ while (true) {
+ ParseStatus Res = MatchOperandParserImpl(Operands, Name);
+ if (Res.isFailure())
+ return true;
+ if (Res.isNoMatch()) {
+ // Fall back to a register or an expression.
+ Res = parseRegOperand(Operands, /*AllowFlags=*/true);
+ if (Res.isFailure())
+ return true;
+ if (Res.isNoMatch() &&
+ parseIntOperand(Operands, AGXOperand::IK_Int).isNoMatch())
+ return Error(getLexer().getLoc(), "unknown operand");
+ }
+ if (getLexer().is(AsmToken::EndOfStatement))
+ break;
+ if (getLexer().isNot(AsmToken::Comma))
+ return Error(getLexer().getLoc(), "unexpected token in operand list");
+ Parser.Lex();
+ }
+ Parser.Lex();
+ return false;
+}
+
+bool AGXAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
+ OperandVector &Operands,
+ MCStreamer &Out,
+ uint64_t &ErrorInfo,
+ bool MatchingInlineAsm) {
+ MCInst Inst;
+ switch (MatchInstructionImpl(Operands, Inst, ErrorInfo, MatchingInlineAsm)) {
+ case Match_Success:
+ Inst.setLoc(IDLoc);
+ Out.emitInstruction(Inst, getSTI());
+ Opcode = Inst.getOpcode();
+ return false;
+ case Match_MissingFeature:
+ return Error(IDLoc, "instruction not supported by the target encoding");
+ case Match_MnemonicFail: {
+ FeatureBitset FBS = ComputeAvailableFeatures(getSTI().getFeatureBits());
+ std::string Suggestion = AGXMnemonicSpellCheck(
+ static_cast<AGXOperand &>(*Operands[0]).getToken(), FBS, 0);
+ return Error(IDLoc, "invalid instruction" + Suggestion);
+ }
+ case Match_InvalidOperand: {
+ SMLoc ErrorLoc = IDLoc;
+ if (ErrorInfo != ~0ULL) {
+ if (ErrorInfo >= Operands.size())
+ return Error(IDLoc, "too few operands for instruction");
+ ErrorLoc = Operands[ErrorInfo]->getStartLoc();
+ if (ErrorLoc == SMLoc())
+ ErrorLoc = IDLoc;
+ }
+ return Error(ErrorLoc, "invalid operand for instruction");
+ }
+ default:
+ return Error(IDLoc, "invalid instruction");
+ }
+}
+
+extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAGXAsmParser() {
+ RegisterMCAsmParser<AGXAsmParser> X(getTheAGXTarget());
+}
diff --git a/llvm/lib/Target/AGX/AsmParser/CMakeLists.txt b/llvm/lib/Target/AGX/AsmParser/CMakeLists.txt
new file mode 100644
index 0000000000000..30d8d2c1790ad
--- /dev/null
+++ b/llvm/lib/Target/AGX/AsmParser/CMakeLists.txt
@@ -0,0 +1,15 @@
+add_llvm_component_library(LLVMAGXAsmParser
+ AGXAsmParser.cpp
+
+ LINK_COMPONENTS
+ AGXDesc
+ AGXInfo
+ AGXUtils
+ MC
+ MCParser
+ Support
+ TargetParser
+
+ ADD_TO_COMPONENT
+ AGX
+ )
diff --git a/llvm/lib/Target/AGX/CMakeLists.txt b/llvm/lib/Target/AGX/CMakeLists.txt
new file mode 100644
index 0000000000000..4761716dc456b
--- /dev/null
+++ b/llvm/lib/Target/AGX/CMakeLists.txt
@@ -0,0 +1,65 @@
+add_llvm_component_group(AGX)
+
+set(LLVM_TARGET_DEFINITIONS AGX.td)
+
+tablegen(LLVM AGXGenAsmMatcher.inc -gen-asm-matcher)
+tablegen(LLVM AGXGenAsmWriter.inc -gen-asm-writer)
+tablegen(LLVM AGXGenCallingConv.inc -gen-callingconv)
+tablegen(LLVM AGXGenDAGISel.inc -gen-dag-isel)
+tablegen(LLVM AGXGenDisassemblerTables.inc -gen-disassembler)
+tablegen(LLVM AGXGenInstrInfo.inc -gen-instr-info)
+tablegen(LLVM AGXGenMCCodeEmitter.inc -gen-emitter)
+tablegen(LLVM AGXGenRegisterInfo.inc -gen-register-info)
+tablegen(LLVM AGXGenSDNodeInfo.inc -gen-sd-node-info)
+tablegen(LLVM AGXGenSearchableTables.inc -gen-searchable-tables)
+tablegen(LLVM AGXGenSubtargetInfo.inc -gen-subtarget)
+
+add_public_tablegen_target(AGXCommonTableGen)
+
+add_llvm_target(AGXCodeGen
+ AGXAnnotateControlFlow.cpp
+ AGXAsmPrinter.cpp
+ AGXCodeGenPrepare.cpp
+ AGXFoldOperands.cpp
+ AGXFrameLowering.cpp
+ AGXISelDAGToDAG.cpp
+ AGXISelLowering.cpp
+ AGXInsertWaits.cpp
+ AGXInstrInfo.cpp
+ AGXLowerControlFlow.cpp
+ AGXMachineFunctionInfo.cpp
+ AGXMCInstLower.cpp
+ AGXRegisterInfo.cpp
+ AGXSelectionDAGInfo.cpp
+ AGXSubtarget.cpp
+ AGXTargetMachine.cpp
+ AGXTargetTransformInfo.cpp
+
+ LINK_COMPONENTS
+ AGXDesc
+ AGXInfo
+ AGXUtils
+ Analysis
+ AsmPrinter
+ CodeGen
+ CodeGenTypes
+ Core
+ IPO
+ MC
+ Passes
+ Scalar
+ SelectionDAG
+ Support
+ Target
+ TargetParser
+ TransformUtils
+
+ ADD_TO_COMPONENT
+ AGX
+ )
+
+add_subdirectory(AsmParser)
+add_subdirectory(Disassembler)
+add_subdirectory(MCTargetDesc)
+add_subdirectory(TargetInfo)
+add_subdirectory(Utils)
diff --git a/llvm/lib/Target/AGX/Disassembler/AGXDisassembler.cpp b/llvm/lib/Target/AGX/Disassembler/AGXDisassembler.cpp
new file mode 100644
index 0000000000000..f8e7ca8d561c0
--- /dev/null
+++ b/llvm/lib/Target/AGX/Disassembler/AGXDisassembler.cpp
@@ -0,0 +1,423 @@
+//===-- AGXDisassembler.cpp - Disassembler for AGX ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// AGX instructions are 2 to 12 bytes long. The TableGen'erated decoder has one
+// table per long form size. Tables are tried from the shortest; an
+// instruction with a short form is re-decoded with the bytes beyond the short
+// form cleared when its length bit is clear.
+//
+//===----------------------------------------------------------------------===//
+
+#include "MCTargetDesc/AGXMCTargetDesc.h"
+#include "TargetInfo/AGXTargetInfo.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCDecoder.h"
+#include "llvm/MC/MCDecoderOps.h"
+#include "llvm/MC/MCDisassembler/MCDisassembler.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/MathExtras.h"
+#include <bitset>
+
+using namespace llvm;
+using namespace llvm::MCD;
+
+#define DEBUG_TYPE "agx-disassembler"
+
+using DecodeStatus = MCDisassembler::DecodeStatus;
+
+namespace {
+
+class AGXDisassembler : public MCDisassembler {
+ std::unique_ptr<const MCInstrInfo> MCII;
+
+public:
+ AGXDisassembler(const MCSubtargetInfo &STI, MCContext &Ctx,
+ const MCInstrInfo *MCII)
+ : MCDisassembler(STI, Ctx), MCII(MCII) {}
+
+ DecodeStatus getInstruction(MCInst &MI, uint64_t &Size,
+ ArrayRef<uint8_t> Bytes, uint64_t Address,
+ raw_ostream &CStream) const override;
+
+ const MCRegisterInfo &getRegInfo() const {
+ return *getContext().getRegisterInfo();
+ }
+
+private:
+ DecodeStatus tryDecode(MCInst &MI, uint64_t &Size, ArrayRef<uint8_t> Bytes,
+ unsigned TableBytes, unsigned ValidBytes,
+ uint64_t Address) const;
+};
+
+} // namespace
+
+static const MCRegisterInfo &getMRI(const MCDisassembler *D) {
+ return static_cast<const AGXDisassembler *>(D)->getRegInfo();
+}
+
+static DecodeStatus addReg(MCInst &MI, AGX::RegKind Kind, unsigned HalfIndex,
+ const MCDisassembler *D) {
+ MCRegister R = AGX::getRegister(Kind, HalfIndex, getMRI(D));
+ if (!R)
+ return MCDisassembler::Fail;
+ MI.addOperand(MCOperand::createReg(R));
+ return MCDisassembler::Success;
+}
+
+static DecodeStatus decodeImm(MCInst &MI, uint64_t Imm, uint64_t,
+ const MCDisassembler *) {
+ MI.addOperand(MCOperand::createImm(Imm));
+ return MCDisassembler::Success;
+}
+
+/// Decode an ALU source in the common layout ([5:0] value, [9:6] type,
+/// [11:10] extension) into a modifier and a value operand.
+static DecodeStatus decodeALUSrc(MCInst &MI, uint64_t M, unsigned ExtraMods,
+ unsigned MaxSize, const MCDisassembler *D,
+ bool Allow32BitUniform = false) {
+ unsigned Type = (M >> 6) & 0xf;
+ unsigned Value = (M & 63) | (((M >> 10) & 3) << 6);
+ if (Type == 0) {
+ MI.addOperand(MCOperand::createImm(ExtraMods));
+ MI.addOperand(MCOperand::createImm(Value));
+ return MCDisassembler::Success;
+ }
+ if ((Type & 0b1100) == 0b0100) {
+ unsigned Idx = Value | ((Type & 1) << 8);
+ bool Is32 = Type & 2;
+ if (Is32 && MaxSize < 32 && !Allow32BitUniform)
+ return MCDisassembler::Fail;
+ MI.addOperand(MCOperand::createImm(ExtraMods));
+ return addReg(MI, Is32 ? AGX::RegKind::UGPR32 : AGX::RegKind::UGPR16, Idx,
+ D);
+ }
+ unsigned Hint = Type & 3;
+ if (Hint == 0)
+ return MCDisassembler::Fail;
+ unsigned Mods = ExtraMods | (Hint == 2 ? AGX::SRC_MOD_CACHE
+ : Hint == 3 ? AGX::SRC_MOD_DISCARD
+ : 0);
+ AGX::RegKind Kind;
+ switch (Type >> 2) {
+ case 0:
+ Kind = AGX::RegKind::GPR16;
+ break;
+ case 2:
+ if (MaxSize < 32)
+ return MCDisassembler::Fail;
+ Kind = AGX::RegKind::GPR32;
+ break;
+ case 3:
+ if (MaxSize < 64)
+ return MCDisassembler::Fail;
+ Kind = AGX::RegKind::GPR64;
+ break;
+ default:
+ return MCDisassembler::Fail;
+ }
+ MI.addOperand(MCOperand::createImm(Mods));
+ return addReg(MI, Kind, Value, D);
+}
+
+static unsigned decodeFPMods(unsigned FM) {
+ return ((FM & 1) ? AGX::SRC_MOD_ABS : 0) | ((FM & 2) ? AGX::SRC_MOD_NEG : 0);
+}
+
+static DecodeStatus decodeSrc(MCInst &MI, uint64_t V, uint64_t,
+ const MCDisassembler *D) {
+ return decodeALUSrc(MI, V, 0, 32, D);
+}
+
+[[maybe_unused]] static DecodeStatus decodeSrc64(MCInst &MI, uint64_t V, uint64_t,
+ const MCDisassembler *D) {
+ return decodeALUSrc(MI, V, 0, 64, D);
+}
+
+static DecodeStatus decodeSrc16(MCInst &MI, uint64_t V, uint64_t,
+ const MCDisassembler *D) {
+ return decodeALUSrc(MI, V, 0, 16, D, /*Allow32BitUniform=*/true);
+}
+
+static DecodeStatus decodeSrcSX(MCInst &MI, uint64_t V, uint64_t,
+ const MCDisassembler *D) {
+ uint64_t M = (V & 0x3ff) | (((V >> 11) & 3) << 10);
+ return decodeALUSrc(MI, M, (V & (1u << 10)) ? AGX::SRC_MOD_SX : 0, 32, D);
+}
+
+static DecodeStatus decodeSrcSX64(MCInst &MI, uint64_t V, uint64_t,
+ const MCDisassembler *D) {
+ uint64_t M = (V & 0x3ff) | (((V >> 11) & 3) << 10);
+ return decodeALUSrc(MI, M, (V & (1u << 10)) ? AGX::SRC_MOD_SX : 0, 64, D);
+}
+
+static DecodeStatus decodeSrcF(MCInst &MI, uint64_t V, uint64_t,
+ const MCDisassembler *D) {
+ uint64_t M = (V & 0x3ff) | (((V >> 12) & 3) << 10);
+ return decodeALUSrc(MI, M, decodeFPMods((V >> 10) & 3), 32, D);
+}
+
+static DecodeStatus decodeSrcF16(MCInst &MI, uint64_t V, uint64_t,
+ const MCDisassembler *D) {
+ uint64_t M = (V & 0x1ff) | (((V >> 11) & 3) << 10);
+ // The 32-bit uniform flag is below the omitted GPR size bit.
+ return decodeALUSrc(MI, M, decodeFPMods((V >> 9) & 3), 16, D,
+ /*Allow32BitUniform=*/true);
+}
+
+static DecodeStatus decodeCmpselSrc(MCInst &MI, uint64_t V, uint64_t,
+ const MCDisassembler *D) {
+ unsigned Type = (V >> 6) & 7;
+ unsigned Value = (V & 63) | (((V >> 9) & 3) << 6);
+ // The access size follows the destination, operand 0.
+ AGX::RegInfo Dst = AGX::getRegInfo(MI.getOperand(0).getReg(), getMRI(D));
+ bool Is32 = Dst.getSizeInBits() >= 32;
+ if (Type == 0b100) {
+ MI.addOperand(MCOperand::createImm(0));
+ MI.addOperand(MCOperand::createImm(Value));
+ return MCDisassembler::Success;
+ }
+ if ((Type >> 1) == 0b11) {
+ MI.addOperand(MCOperand::createImm(0));
+ return addReg(MI, Is32 ? AGX::RegKind::UGPR32 : AGX::RegKind::UGPR16,
+ Value | ((Type & 1) << 8), D);
+ }
+ if (Type == 0 || (Type >> 2))
+ return MCDisassembler::Fail;
+ unsigned Mods = Type == 2 ? AGX::SRC_MOD_CACHE
+ : Type == 3 ? AGX::SRC_MOD_DISCARD
+ : 0;
+ MI.addOperand(MCOperand::createImm(Mods));
+ return addReg(MI, Is32 ? AGX::RegKind::GPR32 : AGX::RegKind::GPR16, Value, D);
+}
+
+static DecodeStatus decodeDstImpl(MCInst &MI, uint64_t V, bool Allow64,
+ int ForcedSize, const MCDisassembler *D) {
+ bool Cache = V & 1;
+ bool Is32 = ForcedSize ? ForcedSize == 32 : (V >> 1) & 1;
+ unsigned Value = (V >> 2) & 0xff;
+ AGX::RegKind Kind = AGX::RegKind::GPR16;
+ if (Is32) {
+ if ((Value & 1) && Allow64) {
+ Kind = AGX::RegKind::GPR64;
+ Value &= ~1u;
+ } else {
+ Kind = AGX::RegKind::GPR32;
+ }
+ }
+ if (addReg(MI, Kind, Value, D) == MCDisassembler::Fail)
+ return MCDisassembler::Fail;
+ MI.addOperand(MCOperand::createImm(Cache));
+ return MCDisassembler::Success;
+}
+
+static DecodeStatus decodeDst(MCInst &MI, uint64_t V, uint64_t,
+ const MCDisassembler *D) {
+ return decodeDstImpl(MI, V, false, 0, D);
+}
+
+static DecodeStatus decodeDst64(MCInst &MI, uint64_t V, uint64_t,
+ const MCDisassembler *D) {
+ return decodeDstImpl(MI, V, true, 0, D);
+}
+
+static DecodeStatus decodeDst16(MCInst &MI, uint64_t V, uint64_t,
+ const MCDisassembler *D) {
+ return decodeDstImpl(MI, V, false, 16, D);
+}
+
+static DecodeStatus decodeDst32(MCInst &MI, uint64_t V, uint64_t,
+ const MCDisassembler *D) {
+ return decodeDstImpl(MI, V, false, 32, D);
+}
+
+static DecodeStatus decodeMemReg(MCInst &MI, uint64_t V, uint64_t,
+ const MCDisassembler *D) {
+ bool Is32 = (V >> 8) & 1;
+ return addReg(MI, Is32 ? AGX::RegKind::GPR32 : AGX::RegKind::GPR16, V & 0xff,
+ D);
+}
+
+static DecodeStatus decodeAtomicDst(MCInst &MI, uint64_t V, uint64_t,
+ const MCDisassembler *D) {
+ if (!((V >> 8) & 1)) {
+ if (V & 0xff)
+ return MCDisassembler::Fail;
+ MI.addOperand(MCOperand::createReg(MCRegister()));
+ return MCDisassembler::Success;
+ }
+ return addReg(MI, AGX::RegKind::GPR32, V & 0xff, D);
+}
+
+static DecodeStatus decodeAtomicSrc(MCInst &MI, uint64_t V, uint64_t,
+ const MCDisassembler *D) {
+ // Operand 1 is the atomic operation; cmpxchg takes a register pair.
+ bool Pair = MI.getNumOperands() > 1 && MI.getOperand(1).isImm() &&
+ MI.getOperand(1).getImm() == 3;
+ return addReg(MI, Pair ? AGX::RegKind::GPR64 : AGX::RegKind::GPR32, V & 0xff,
+ D);
+}
+
+static DecodeStatus decodeMemBase(MCInst &MI, uint64_t V, uint64_t,
+ const MCDisassembler *D) {
+ bool Uniform = (V >> 8) & 1;
+ return addReg(MI, Uniform ? AGX::RegKind::UGPR64 : AGX::RegKind::GPR64,
+ V & 0xff, D);
+}
+
+static DecodeStatus decodeMemIndex(MCInst &MI, uint64_t V, uint64_t,
+ const MCDisassembler *D) {
+ if ((V >> 16) & 1) {
+ MI.addOperand(MCOperand::createImm(int16_t(V & 0xffff)));
+ return MCDisassembler::Success;
+ }
+ if (V & 0xff00)
+ return MCDisassembler::Fail;
+ return addReg(MI, AGX::RegKind::GPR32, V & 0xff, D);
+}
+
+static DecodeStatus decodeLocalBase(MCInst &MI, uint64_t V, uint64_t,
+ const MCDisassembler *D) {
+ unsigned Flags = (V >> 8) & 3;
+ unsigned Value = V & 0xff;
+ if (Flags == 0)
+ return addReg(MI, AGX::RegKind::GPR16, Value, D);
+ if (Flags == 2) {
+ if (Value)
+ return MCDisassembler::Fail;
+ MI.addOperand(MCOperand::createImm(0));
+ return MCDisassembler::Success;
+ }
+ return addReg(MI, AGX::RegKind::UGPR16, Value | ((Flags >> 1) << 8), D);
+}
+
+static DecodeStatus decodeLocalIndex(MCInst &MI, uint64_t V, uint64_t,
+ const MCDisassembler *D) {
+ if ((V >> 16) & 1) {
+ MI.addOperand(MCOperand::createImm(int16_t(V & 0xffff)));
+ return MCDisassembler::Success;
+ }
+ if (V & 0xff00)
+ return MCDisassembler::Fail;
+ return addReg(MI, AGX::RegKind::GPR16, V & 0xff, D);
+}
+
+static DecodeStatus decodeBrTarget(MCInst &MI, uint64_t V, uint64_t,
+ const MCDisassembler *) {
+ MI.addOperand(MCOperand::createImm(SignExtend64<32>(V)));
+ return MCDisassembler::Success;
+}
+
+#include "AGXGenDisassemblerTables.inc"
+
+
+static const uint8_t *getTable(unsigned Bytes) {
+ switch (Bytes) {
+ case 2:
+ return DecoderTableG1316;
+ case 4:
+ return DecoderTableG1332;
+ case 6:
+ return DecoderTableG1348;
+ case 8:
+ return DecoderTableG1364;
+ case 10:
+ return DecoderTableG1380;
+ default:
+ return nullptr;
+ }
+}
+
+static std::bitset<96> readBytes(ArrayRef<uint8_t> Bytes, unsigned N) {
+ std::bitset<96> Insn;
+ for (unsigned I = 0; I < N && I < Bytes.size(); ++I)
+ Insn |= std::bitset<96>(Bytes[I]) << (8 * I);
+ return Insn;
+}
+
+DecodeStatus AGXDisassembler::tryDecode(MCInst &MI, uint64_t &Size,
+ ArrayRef<uint8_t> Bytes,
+ unsigned TableBytes,
+ unsigned ValidBytes,
+ uint64_t Address) const {
+ const uint8_t *Table = getTable(TableBytes);
+ std::bitset<96> Insn = readBytes(Bytes, std::min(TableBytes, ValidBytes));
+ MI.clear();
+ if (decodeInstruction(Table, MI, Insn, Address, this, STI) !=
+ MCDisassembler::Success)
+ return MCDisassembler::Fail;
+
+ const MCInstrDesc &Desc = MCII->get(MI.getOpcode());
+ unsigned Short = AGX::TSFlags::getShortSize(Desc.TSFlags);
+ unsigned Long = AGX::TSFlags::getLongSize(Desc.TSFlags);
+ if (Long != TableBytes)
+ return MCDisassembler::Fail;
+ if (Short == Long || Insn[AGX::TSFlags::getLengthBit(Desc.TSFlags)]) {
+ if (Long > ValidBytes)
+ return MCDisassembler::Fail;
+ Size = Long;
+ return MCDisassembler::Success;
+ }
+ if (ValidBytes < Short)
+ return MCDisassembler::Fail;
+ // Short form: the remaining bytes belong to the next instruction.
+ if (ValidBytes != Short) {
+ Insn = readBytes(Bytes, Short);
+ MI.clear();
+ if (decodeInstruction(Table, MI, Insn, Address, this, STI) !=
+ MCDisassembler::Success)
+ return MCDisassembler::Fail;
+ const MCInstrDesc &ShortDesc = MCII->get(MI.getOpcode());
+ if (AGX::TSFlags::getShortSize(ShortDesc.TSFlags) != Short)
+ return MCDisassembler::Fail;
+ }
+ Size = Short;
+ return MCDisassembler::Success;
+}
+
+DecodeStatus AGXDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
+ ArrayRef<uint8_t> Bytes,
+ uint64_t Address,
+ raw_ostream &CStream) const {
+ Size = 0;
+ if (Bytes.size() < 2)
+ return MCDisassembler::Fail;
+ unsigned Avail = std::min<size_t>(Bytes.size(), 12);
+ for (unsigned TableBytes : {2u, 4u, 6u, 8u, 10u}) {
+ // First as if all bytes up to the long form belonged to the instruction,
+ // then with only the bytes of each possible short form.
+ if (tryDecode(MI, Size, Bytes, TableBytes, Avail, Address) ==
+ MCDisassembler::Success)
+ return MCDisassembler::Success;
+ for (unsigned ShortBytes : {TableBytes - 2, TableBytes - 4}) {
+ if (ShortBytes < 2 || ShortBytes > TableBytes)
+ continue;
+ if (tryDecode(MI, Size, Bytes, TableBytes, std::min(ShortBytes, Avail),
+ Address) == MCDisassembler::Success)
+ return MCDisassembler::Success;
+ }
+ }
+ Size = 2;
+ return MCDisassembler::Fail;
+}
+
+static MCDisassembler *createAGXDisassembler(const Target &T,
+ const MCSubtargetInfo &STI,
+ MCContext &Ctx) {
+ return new AGXDisassembler(STI, Ctx, T.createMCInstrInfo());
+}
+
+extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
+LLVMInitializeAGXDisassembler() {
+ TargetRegistry::RegisterMCDisassembler(getTheAGXTarget(),
+ createAGXDisassembler);
+}
diff --git a/llvm/lib/Target/AGX/Disassembler/CMakeLists.txt b/llvm/lib/Target/AGX/Disassembler/CMakeLists.txt
new file mode 100644
index 0000000000000..a1c17106688b1
--- /dev/null
+++ b/llvm/lib/Target/AGX/Disassembler/CMakeLists.txt
@@ -0,0 +1,14 @@
+add_llvm_component_library(LLVMAGXDisassembler
+ AGXDisassembler.cpp
+
+ LINK_COMPONENTS
+ AGXDesc
+ AGXInfo
+ AGXUtils
+ MC
+ MCDisassembler
+ Support
+
+ ADD_TO_COMPONENT
+ AGX
+ )
diff --git a/llvm/lib/Target/AGX/G13Instructions.td b/llvm/lib/Target/AGX/G13Instructions.td
new file mode 100644
index 0000000000000..ba861632efbfc
--- /dev/null
+++ b/llvm/lib/Target/AGX/G13Instructions.td
@@ -0,0 +1,1043 @@
+//===-- G13Instructions.td - AGX2 encoding family ----------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Real instructions of the G13/G14 encoding family. Bit numbers count from
+// bit 0 of the first byte; instructions are little endian. The layout of each
+// instruction follows dougallj/applegpu and the Mesa asahi packer.
+//
+// In ALU instructions bits 6:0 hold the opcode (bit 6 is the saturate flag
+// where saturation exists), bits 14:7 the low destination bits and bit 15 the
+// length bit. Source N starts at bit 16 + 12 * N. The two high bits of each
+// register field are stored in the last 16 bits of the long form, in reverse
+// order: destination at bits 12-13 of that word, source N at 10 - 2 * N.
+//
+//===----------------------------------------------------------------------===//
+
+class G13ALU<string mn, dag outs, dag ins, string ops, int short,
+ int long = short>
+ : AGXReal<mn, outs, ins, ops, short, long> {
+ let IsALU = 1;
+}
+
+//===----------------------------------------------------------------------===//
+// Moves
+//===----------------------------------------------------------------------===//
+
+def MOV_IMM16_g13 : G13ALU<"mov_imm", (outs AGXDst16:$dst), (ins AGXImm:$imm),
+ "$dst, $imm", 4, 6> {
+ bits<10> dst;
+ bits<16> imm;
+ let Inst{6-0} = 0b1100010;
+ let Inst{7} = dst{0};
+ let Inst{8} = 0;
+ let Inst{14-9} = dst{7-2};
+ let Inst{31-16} = imm;
+ let Inst{45-44} = dst{9-8};
+}
+
+def MOV_IMM32_g13 : G13ALU<"mov_imm", (outs AGXDst32:$dst), (ins AGXImm:$imm),
+ "$dst, $imm", 6, 8> {
+ bits<10> dst;
+ bits<32> imm;
+ let Inst{6-0} = 0b1100010;
+ let Inst{7} = dst{0};
+ let Inst{8} = 1;
+ let Inst{14-9} = dst{7-2};
+ let Inst{47-16} = imm;
+ let Inst{61-60} = dst{9-8};
+ let Inst{62} = 0;
+}
+
+def GET_SR_g13 : G13ALU<"get_sr", (outs AGXDst:$dst), (ins AGXSpecialReg:$sr),
+ "$dst, $sr", 4> {
+ bits<10> dst;
+ bits<8> sr;
+ let Inst{6-0} = 0b1110010;
+ let Inst{14-7} = dst{7-0};
+ let Inst{15} = 0;
+ let Inst{21-16} = sr{5-0};
+ let Inst{27-26} = sr{7-6};
+ let Inst{29-28} = dst{9-8};
+}
+
+//===----------------------------------------------------------------------===//
+// Integer arithmetic
+//===----------------------------------------------------------------------===//
+
+class G13IAdd<string mn, bit neg, bit sat>
+ : G13ALU<mn#!if(sat, ".sat", ""), (outs AGXDst64:$dst),
+ (ins AGXSrcSX64:$src0, AGXSrcSX64:$src1, AGXShift:$shift),
+ "$dst, $src0, $src1$shift", 8> {
+ bits<10> dst;
+ bits<13> src0;
+ bits<13> src1;
+ bits<3> shift;
+ let Inst{5-0} = 0b001110;
+ let Inst{6} = sat;
+ let Inst{14-7} = dst{7-0};
+ let Inst{15} = 0;
+ let Inst{26-16} = src0{10-0};
+ let Inst{27} = neg;
+ let Inst{38-28} = src1{10-0};
+ let Inst{39} = shift{0};
+ let Inst{41-40} = src1{12-11};
+ let Inst{43-42} = src0{12-11};
+ let Inst{45-44} = dst{9-8};
+ let Inst{53-52} = shift{2-1};
+}
+
+def IADD_g13 : G13IAdd<"iadd", 0, 0>;
+def IADD_SAT_g13 : G13IAdd<"iadd", 0, 1>;
+def ISUB_g13 : G13IAdd<"isub", 1, 0>;
+def ISUB_SAT_g13 : G13IAdd<"isub", 1, 1>;
+
+class G13IMAdd<string mn, bit neg, bit sat>
+ : G13ALU<mn#!if(sat, ".sat", ""), (outs AGXDst64:$dst),
+ (ins AGXSrcSX:$src0, AGXSrcSX:$src1, AGXSrcSX64:$src2,
+ AGXShift:$shift),
+ "$dst, $src0, $src1, $src2$shift", 8> {
+ bits<10> dst;
+ bits<13> src0;
+ bits<13> src1;
+ bits<13> src2;
+ bits<3> shift;
+ let Inst{5-0} = 0b011110;
+ let Inst{6} = sat;
+ let Inst{14-7} = dst{7-0};
+ let Inst{15} = 0;
+ let Inst{26-16} = src0{10-0};
+ let Inst{27} = neg;
+ let Inst{38-28} = src1{10-0};
+ let Inst{39} = shift{0};
+ let Inst{50-40} = src2{10-0};
+ let Inst{53-52} = shift{2-1};
+ let Inst{55-54} = src2{12-11};
+ let Inst{57-56} = src1{12-11};
+ let Inst{59-58} = src0{12-11};
+ let Inst{61-60} = dst{9-8};
+}
+
+def IMADD_g13 : G13IMAdd<"imadd", 0, 0>;
+def IMADD_SAT_g13 : G13IMAdd<"imadd", 0, 1>;
+def IMSUB_g13 : G13IMAdd<"imsub", 1, 0>;
+def IMSUB_SAT_g13 : G13IMAdd<"imsub", 1, 1>;
+
+//===----------------------------------------------------------------------===//
+// Shifts and bit fields
+//===----------------------------------------------------------------------===//
+
+class G13Bitfield<string mn, bit sel, bits<2> sub>
+ : G13ALU<mn, (outs AGXDst:$dst),
+ (ins AGXSrc:$src0, AGXSrc:$src1, AGXSrc:$src2,
+ AGXBitfieldMask:$mask),
+ "$dst, $src0, $src1, $src2$mask", 8> {
+ bits<10> dst;
+ bits<12> src0;
+ bits<12> src1;
+ bits<12> src2;
+ bits<5> mask;
+ let Inst{6-0} = 0b0101110;
+ let Inst{14-7} = dst{7-0};
+ let Inst{15} = sel;
+ let Inst{25-16} = src0{9-0};
+ let Inst{27-26} = sub;
+ let Inst{37-28} = src1{9-0};
+ let Inst{39-38} = mask{1-0};
+ let Inst{49-40} = src2{9-0};
+ let Inst{51-50} = mask{3-2};
+ let Inst{55-54} = src2{11-10};
+ let Inst{57-56} = src1{11-10};
+ let Inst{59-58} = src0{11-10};
+ let Inst{61-60} = dst{9-8};
+ let Inst{63} = mask{4};
+}
+
+def BFI_g13 : G13Bitfield<"bfi", 0, 0>;
+def BFEIL_g13 : G13Bitfield<"bfeil", 1, 0>;
+def EXTR_g13 : G13Bitfield<"extr", 0, 1>;
+def SHLHI_g13 : G13Bitfield<"shlhi", 0, 2>;
+def SHRHI_g13 : G13Bitfield<"shrhi", 1, 2>;
+
+class G13Shift<string mn, bits<2> sub>
+ : G13ALU<mn, (outs AGXDst:$dst), (ins AGXSrc:$src0, AGXSrc:$src1),
+ "$dst, $src0, $src1", 8> {
+ bits<10> dst;
+ bits<12> src0;
+ bits<12> src1;
+ let Inst{6-0} = 0b0101110;
+ let Inst{14-7} = dst{7-0};
+ let Inst{15} = 1;
+ let Inst{25-16} = src0{9-0};
+ let Inst{27-26} = sub;
+ let Inst{37-28} = src1{9-0};
+ let Inst{57-56} = src1{11-10};
+ let Inst{59-58} = src0{11-10};
+ let Inst{61-60} = dst{9-8};
+}
+
+def ASR_g13 : G13Shift<"asr", 1>;
+def ASRH_g13 : G13Shift<"asrh", 3>;
+
+//===----------------------------------------------------------------------===//
+// Bitwise operations
+//===----------------------------------------------------------------------===//
+
+// D = truth_table(A, B). The truth table bit i gives the result for the
+// operand bits (a, b) = (i & 1, i >> 1).
+def BITOP_g13 : G13ALU<"bitop", (outs AGXDst:$dst),
+ (ins AGXTruthTable:$tt, AGXSrc:$src0, AGXSrc:$src1),
+ "$tt, $dst, $src0, $src1", 6> {
+ bits<10> dst;
+ bits<4> tt;
+ bits<12> src0;
+ bits<12> src1;
+ let Inst{6-0} = 0b1111110;
+ let Inst{14-7} = dst{7-0};
+ let Inst{15} = 0;
+ let Inst{25-16} = src0{9-0};
+ let Inst{27-26} = tt{1-0};
+ let Inst{37-28} = src1{9-0};
+ let Inst{39-38} = tt{3-2};
+ let Inst{41-40} = src1{11-10};
+ let Inst{43-42} = src0{11-10};
+ let Inst{45-44} = dst{9-8};
+}
+
+// Truth table aliases, in the applegpu spelling.
+class AGXBitopAlias<string mn, int tt> {
+ string Mnemonic = mn;
+ int TT = tt;
+}
+
+foreach Alias = [AGXBitopAlias<"and", 0b1000>, AGXBitopAlias<"or", 0b1110>,
+ AGXBitopAlias<"xor", 0b0110>, AGXBitopAlias<"nand", 0b0111>,
+ AGXBitopAlias<"nor", 0b0001>, AGXBitopAlias<"xnor", 0b1001>] in
+def : InstAlias<Alias.Mnemonic#" $dst, $src0, $src1",
+ (BITOP_g13 AGXDst:$dst, Alias.TT, AGXSrc:$src0,
+ AGXSrc:$src1)>;
+
+class G13IUnary<string mn, bits<2> op>
+ : G13ALU<mn, (outs AGXDst:$dst), (ins AGXSrc:$src0), "$dst, $src0", 6> {
+ bits<10> dst;
+ bits<12> src0;
+ let Inst{6-0} = 0b0111110;
+ let Inst{14-7} = dst{7-0};
+ let Inst{15} = 0;
+ let Inst{25-16} = src0{9-0};
+ let Inst{27-26} = op;
+ let Inst{37-28} = 0;
+ let Inst{39-38} = 0;
+ let Inst{41-40} = 0;
+ let Inst{43-42} = src0{11-10};
+ let Inst{45-44} = dst{9-8};
+}
+
+def BITREV_g13 : G13IUnary<"bitrev", 1>;
+def POPCOUNT_g13 : G13IUnary<"popcount", 2>;
+def FFS_g13 : G13IUnary<"ffs", 3>;
+
+def INTL_g13 : G13ALU<"intl", (outs AGXDst:$dst),
+ (ins AGXSrc:$src0, AGXSrc:$src1),
+ "$dst, $src0, $src1", 6> {
+ bits<10> dst;
+ bits<12> src0;
+ bits<12> src1;
+ let Inst{6-0} = 0b0111110;
+ let Inst{14-7} = dst{7-0};
+ let Inst{15} = 0;
+ let Inst{25-16} = src0{9-0};
+ let Inst{27-26} = 0;
+ let Inst{37-28} = src1{9-0};
+ let Inst{39-38} = 0;
+ let Inst{41-40} = src1{11-10};
+ let Inst{43-42} = src0{11-10};
+ let Inst{45-44} = dst{9-8};
+}
+
+def CONVERT_g13 : G13ALU<"convert", (outs AGXDst:$dst),
+ (ins AGXConvertMode:$mode, AGXSrc:$src0,
+ AGXRoundMode:$round),
+ "$mode, $dst, $src0, $round", 6> {
+ bits<10> dst;
+ bits<6> mode;
+ bits<12> src0;
+ bits<2> round;
+ let Inst{6-0} = 0b0111110;
+ let Inst{14-7} = dst{7-0};
+ let Inst{15} = 1;
+ let Inst{21-16} = mode;
+ let Inst{25-22} = 0;
+ let Inst{27-26} = round;
+ let Inst{37-28} = src0{9-0};
+ let Inst{39-38} = 0;
+ let Inst{41-40} = src0{11-10};
+ let Inst{43-42} = 0;
+ let Inst{45-44} = dst{9-8};
+}
+
+//===----------------------------------------------------------------------===//
+// Floating point arithmetic
+//===----------------------------------------------------------------------===//
+
+class G13FBin32<string mn, bits<6> op, bit sat>
+ : G13ALU<mn#!if(sat, ".sat", ""), (outs AGXDst:$dst),
+ (ins AGXSrcF:$src0, AGXSrcF:$src1), "$dst, $src0, $src1", 4, 6> {
+ bits<10> dst;
+ bits<14> src0;
+ bits<14> src1;
+ let Inst{5-0} = op;
+ let Inst{6} = sat;
+ let Inst{14-7} = dst{7-0};
+ let Inst{27-16} = src0{11-0};
+ let Inst{39-28} = src1{11-0};
+ let Inst{41-40} = src1{13-12};
+ let Inst{43-42} = src0{13-12};
+ let Inst{45-44} = dst{9-8};
+}
+
+class G13FBin16<string mn, bits<6> op, bit sat>
+ : G13ALU<mn#!if(sat, ".sat", ""), (outs AGXDst16:$dst),
+ (ins AGXSrcF16:$src0, AGXSrcF16:$src1), "$dst, $src0, $src1",
+ 4, 6> {
+ bits<10> dst;
+ bits<13> src0;
+ bits<13> src1;
+ let Inst{5-0} = op;
+ let Inst{6} = sat;
+ let Inst{7} = dst{0};
+ let Inst{8} = 0;
+ let Inst{14-9} = dst{7-2};
+ let Inst{26-16} = src0{10-0};
+ let Inst{38-28} = src1{10-0};
+ let Inst{41-40} = src1{12-11};
+ let Inst{43-42} = src0{12-11};
+ let Inst{45-44} = dst{9-8};
+}
+
+foreach Sat = [0, 1] in {
+ defvar S = !if(Sat, "_SAT", "");
+ def FADD32#S#_g13 : G13FBin32<"fadd32", 0b101010, Sat>;
+ def FMUL32#S#_g13 : G13FBin32<"fmul32", 0b011010, Sat>;
+ def FADD16#S#_g13 : G13FBin16<"fadd16", 0b100110, Sat>;
+ def FMUL16#S#_g13 : G13FBin16<"fmul16", 0b010110, Sat>;
+}
+
+class G13FMA32<bit sat>
+ : G13ALU<"fmadd32"#!if(sat, ".sat", ""), (outs AGXDst:$dst),
+ (ins AGXSrcF:$src0, AGXSrcF:$src1, AGXSrcF:$src2),
+ "$dst, $src0, $src1, $src2", 6, 8> {
+ bits<10> dst;
+ bits<14> src0;
+ bits<14> src1;
+ bits<14> src2;
+ let Inst{5-0} = 0b111010;
+ let Inst{6} = sat;
+ let Inst{14-7} = dst{7-0};
+ let Inst{27-16} = src0{11-0};
+ let Inst{39-28} = src1{11-0};
+ let Inst{51-40} = src2{11-0};
+ let Inst{55-54} = src2{13-12};
+ let Inst{57-56} = src1{13-12};
+ let Inst{59-58} = src0{13-12};
+ let Inst{61-60} = dst{9-8};
+}
+
+class G13FMA16<bit sat>
+ : G13ALU<"fmadd16"#!if(sat, ".sat", ""), (outs AGXDst16:$dst),
+ (ins AGXSrcF16:$src0, AGXSrcF16:$src1, AGXSrcF16:$src2),
+ "$dst, $src0, $src1, $src2", 6, 8> {
+ bits<10> dst;
+ bits<13> src0;
+ bits<13> src1;
+ bits<13> src2;
+ let Inst{5-0} = 0b110110;
+ let Inst{6} = sat;
+ let Inst{7} = dst{0};
+ let Inst{8} = 0;
+ let Inst{14-9} = dst{7-2};
+ let Inst{26-16} = src0{10-0};
+ let Inst{38-28} = src1{10-0};
+ let Inst{50-40} = src2{10-0};
+ let Inst{55-54} = src2{12-11};
+ let Inst{57-56} = src1{12-11};
+ let Inst{59-58} = src0{12-11};
+ let Inst{61-60} = dst{9-8};
+}
+
+def FMADD32_g13 : G13FMA32<0>;
+def FMADD32_SAT_g13 : G13FMA32<1>;
+def FMADD16_g13 : G13FMA16<0>;
+def FMADD16_SAT_g13 : G13FMA16<1>;
+
+class G13FUnary<string mn, bits<6> op, bit sat>
+ : G13ALU<mn#!if(sat, ".sat", ""), (outs AGXDst:$dst), (ins AGXSrcF:$src0),
+ "$dst, $src0", 4, 6> {
+ bits<10> dst;
+ bits<14> src0;
+ let Inst{5-0} = 0b001010;
+ let Inst{6} = sat;
+ let Inst{14-7} = dst{7-0};
+ let Inst{27-16} = src0{11-0};
+ let Inst{33-28} = op;
+ let Inst{41-34} = 0;
+ let Inst{43-42} = src0{13-12};
+ let Inst{45-44} = dst{9-8};
+}
+
+foreach Sat = [0, 1] in {
+ defvar S = !if(Sat, "_SAT", "");
+ def FLOOR#S#_g13 : G13FUnary<"floor", 0b000000, Sat>;
+ def RSQRT_SPECIAL#S#_g13 : G13FUnary<"rsqrt_special", 0b000001, Sat>;
+ def DFDX#S#_g13 : G13FUnary<"dfdx", 0b000100, Sat>;
+ def DFDY#S#_g13 : G13FUnary<"dfdy", 0b000110, Sat>;
+ def RCP#S#_g13 : G13FUnary<"rcp", 0b001000, Sat>;
+ def RSQRT#S#_g13 : G13FUnary<"rsqrt", 0b001001, Sat>;
+ def SIN_PT_1#S#_g13 : G13FUnary<"sin_pt_1", 0b001010, Sat>;
+ def LOG2#S#_g13 : G13FUnary<"log2", 0b001100, Sat>;
+ def EXP2#S#_g13 : G13FUnary<"exp2", 0b001101, Sat>;
+ def SIN_PT_2#S#_g13 : G13FUnary<"sin_pt_2", 0b001110, Sat>;
+ def CEIL#S#_g13 : G13FUnary<"ceil", 0b010000, Sat>;
+ def TRUNC#S#_g13 : G13FUnary<"trunc", 0b100000, Sat>;
+ def RINT#S#_g13 : G13FUnary<"rint", 0b110000, Sat>;
+}
+
+//===----------------------------------------------------------------------===//
+// Compare and select
+//===----------------------------------------------------------------------===//
+
+def ICMPSEL_g13 : G13ALU<"icmpsel", (outs AGXDst:$dst),
+ (ins AGXICond3:$cc, AGXSrc:$src0, AGXSrc:$src1,
+ AGXCmpselSrc:$src2, AGXCmpselSrc:$src3),
+ "$cc, $dst, $src0, $src1, $src2, $src3", 8, 10> {
+ bits<10> dst;
+ bits<3> cc;
+ bits<12> src0;
+ bits<12> src1;
+ bits<11> src2;
+ bits<11> src3;
+ let Inst{6-0} = 0b0010010;
+ let Inst{14-7} = dst{7-0};
+ let Inst{25-16} = src0{9-0};
+ let Inst{37-28} = src1{9-0};
+ let Inst{48-40} = src2{8-0};
+ let Inst{60-52} = src3{8-0};
+ let Inst{63-61} = cc;
+ let Inst{69-68} = src3{10-9};
+ let Inst{71-70} = src2{10-9};
+ let Inst{73-72} = src1{11-10};
+ let Inst{75-74} = src0{11-10};
+ let Inst{77-76} = dst{9-8};
+}
+
+def FCMPSEL_g13 : G13ALU<"fcmpsel", (outs AGXDst:$dst),
+ (ins AGXFCond3:$cc, AGXSrcF:$src0, AGXSrcF:$src1,
+ AGXCmpselSrc:$src2, AGXCmpselSrc:$src3),
+ "$cc, $dst, $src0, $src1, $src2, $src3", 8, 10> {
+ bits<10> dst;
+ bits<3> cc;
+ bits<14> src0;
+ bits<14> src1;
+ bits<11> src2;
+ bits<11> src3;
+ let Inst{6-0} = 0b0000010;
+ let Inst{14-7} = dst{7-0};
+ let Inst{27-16} = src0{11-0};
+ let Inst{39-28} = src1{11-0};
+ let Inst{48-40} = src2{8-0};
+ let Inst{60-52} = src3{8-0};
+ let Inst{63-61} = cc;
+ let Inst{69-68} = src3{10-9};
+ let Inst{71-70} = src2{10-9};
+ let Inst{73-72} = src1{13-12};
+ let Inst{75-74} = src0{13-12};
+ let Inst{77-76} = dst{9-8};
+}
+
+//===----------------------------------------------------------------------===//
+// Ballots
+//===----------------------------------------------------------------------===//
+
+class G13ICmpBallot<string mn, bit simd>
+ : G13ALU<mn, (outs AGXDst:$dst),
+ (ins AGXICond:$cc, AGXSrc:$src0, AGXSrc:$src1),
+ "$dst, $cc, $src0, $src1", 8> {
+ bits<10> dst;
+ bits<4> cc;
+ bits<12> src0;
+ bits<12> src1;
+ let Inst{6-0} = 0b0110010;
+ let Inst{14-7} = dst{7-0};
+ let Inst{15} = 0;
+ let Inst{25-16} = src0{9-0};
+ let Inst{27-26} = 0;
+ let Inst{37-28} = src1{9-0};
+ let Inst{39-38} = 0;
+ let Inst{41-40} = src1{11-10};
+ let Inst{43-42} = src0{11-10};
+ let Inst{45-44} = dst{9-8};
+ let Inst{47} = cc{3};
+ let Inst{48} = simd;
+ let Inst{60-49} = 0;
+ let Inst{63-61} = cc{2-0};
+}
+
+class G13FCmpBallot<string mn, bit simd>
+ : G13ALU<mn, (outs AGXDst:$dst),
+ (ins AGXFCond:$cc, AGXSrcF:$src0, AGXSrcF:$src1),
+ "$dst, $cc, $src0, $src1", 8> {
+ bits<10> dst;
+ bits<4> cc;
+ bits<14> src0;
+ bits<14> src1;
+ let Inst{6-0} = 0b0100010;
+ let Inst{14-7} = dst{7-0};
+ let Inst{15} = 0;
+ let Inst{27-16} = src0{11-0};
+ let Inst{39-28} = src1{11-0};
+ let Inst{41-40} = src1{13-12};
+ let Inst{43-42} = src0{13-12};
+ let Inst{45-44} = dst{9-8};
+ let Inst{47} = cc{3};
+ let Inst{48} = simd;
+ let Inst{60-49} = 0;
+ let Inst{63-61} = cc{2-0};
+}
+
+def ICMP_BALLOT_g13 : G13ICmpBallot<"icmp_ballot", 1>;
+def ICMP_QUAD_BALLOT_g13 : G13ICmpBallot<"icmp_quad_ballot", 0>;
+def FCMP_BALLOT_g13 : G13FCmpBallot<"fcmp_ballot", 1>;
+def FCMP_QUAD_BALLOT_g13 : G13FCmpBallot<"fcmp_quad_ballot", 0>;
+
+//===----------------------------------------------------------------------===//
+// SIMD-group and quad-group shuffles and reductions
+//===----------------------------------------------------------------------===//
+
+// SIMD-group shuffles take a 16-bit lane index; quad-group shuffles accept
+// up to 64-bit sources for both operands.
+class G13Shuffle<string mn, bit simd, bits<2> kind, bit rotate>
+ : G13ALU<mn, (outs AGXDst:$dst),
+ !if(simd, (ins AGXSrc:$src0, AGXSrc16:$src1),
+ (ins AGXSrc64:$src0, AGXSrc64:$src1)),
+ "$dst, $src0, $src1", 6> {
+ bits<10> dst;
+ bits<12> src0;
+ bits<12> src1;
+ let Inst{6-0} = 0b1101111;
+ let Inst{14-7} = dst{7-0};
+ let Inst{15} = 0;
+ let Inst{25-16} = src0{9-0};
+ let Inst{27-26} = {0, simd};
+ let Inst{37-28} = src1{9-0};
+ let Inst{39-38} = kind;
+ let Inst{41-40} = src1{11-10};
+ let Inst{43-42} = src0{11-10};
+ let Inst{45-44} = dst{9-8};
+ let Inst{47} = rotate;
+}
+
+class AGXGroupInfo<string name, bit simd> {
+ string Name = name;
+ string Upper = !toupper(name);
+ bit Simd = simd;
+}
+
+foreach Group = [AGXGroupInfo<"simd", 1>, AGXGroupInfo<"quad", 0>] in {
+ defvar G = Group.Name;
+ defvar U = Group.Upper;
+ def U#_SHUFFLE_g13 : G13Shuffle<G#"_shuffle", Group.Simd, 0, 0>;
+ def U#_SHUFFLE_XOR_g13 : G13Shuffle<G#"_shuffle_xor", Group.Simd, 1, 0>;
+ def U#_SHUFFLE_UP_g13 : G13Shuffle<G#"_shuffle_up", Group.Simd, 2, 0>;
+ def U#_SHUFFLE_DOWN_g13 : G13Shuffle<G#"_shuffle_down", Group.Simd, 3, 0>;
+ def U#_SHUFFLE_ROTATE_UP_g13 : G13Shuffle<G#"_shuffle_rotate_up",
+ Group.Simd, 2, 1>;
+}
+
+// Reductions and exclusive scans. opc: 0 quad reduce, 1 simd reduce,
+// 2 quad prefix, 3 simd prefix. op is the AGX::SIMDOp number.
+class G13Reduce<string mn, bits<2> opc, bits<5> op>
+ : G13ALU<mn, (outs AGXDst:$dst), (ins AGXSrc64:$src0), "$dst, $src0", 6> {
+ bits<10> dst;
+ bits<12> src0;
+ let Inst{6-0} = 0b1101111;
+ let Inst{14-7} = dst{7-0};
+ let Inst{15} = 1;
+ let Inst{25-16} = src0{9-0};
+ let Inst{27-26} = 0;
+ let Inst{28} = op{0};
+ let Inst{30-29} = opc;
+ let Inst{37-31} = 0;
+ let Inst{40-38} = op{3-1};
+ let Inst{41} = 0;
+ let Inst{43-42} = src0{11-10};
+ let Inst{45-44} = dst{9-8};
+ let Inst{47} = op{4};
+}
+
+class AGXSIMDOpInfo<string name, string mn, int op> {
+ string Name = name;
+ string Mnemonic = mn;
+ int Op = op;
+}
+
+defvar AGXSIMDOps = [AGXSIMDOpInfo<"AND", "and", 0>,
+ AGXSIMDOpInfo<"FADD", "fadd", 1>,
+ AGXSIMDOpInfo<"OR", "or", 2>,
+ AGXSIMDOpInfo<"FMUL", "fmul", 3>,
+ AGXSIMDOpInfo<"XOR", "xor", 4>,
+ AGXSIMDOpInfo<"FMIN", "min.f", 5>,
+ AGXSIMDOpInfo<"FMAX", "max.f", 7>,
+ AGXSIMDOpInfo<"IADD", "iadd", 16>,
+ AGXSIMDOpInfo<"SMIN", "min.s", 20>,
+ AGXSIMDOpInfo<"SMAX", "max.s", 22>,
+ AGXSIMDOpInfo<"UMIN", "min.u", 28>,
+ AGXSIMDOpInfo<"UMAX", "max.u", 30>];
+
+class AGXReduceKind<string name, string prefix, int opc> {
+ string Name = name;
+ string Prefix = prefix;
+ int Opc = opc;
+}
+
+defvar AGXReduceKinds = [AGXReduceKind<"QUAD_REDUCE", "quad_", 0>,
+ AGXReduceKind<"SIMD_REDUCE", "simd_", 1>,
+ AGXReduceKind<"QUAD_PREFIX", "quad_prefix_", 2>,
+ AGXReduceKind<"SIMD_PREFIX", "simd_prefix_", 3>];
+
+foreach Kind = AGXReduceKinds in
+foreach Op = AGXSIMDOps in
+def Kind.Name#_#Op.Name#_g13
+ : G13Reduce<Kind.Prefix#Op.Mnemonic, Kind.Opc, Op.Op>;
+
+def SIMD_MATRIX_FMADD32_g13
+ : G13ALU<"simd_matrix_fmadd32", (outs AGXDst:$dst),
+ (ins AGXSrcF:$src0, AGXSrcF:$src1, AGXSrcF:$src2),
+ "$dst, $src0, $src1, $src2", 8> {
+ bits<10> dst;
+ bits<14> src0;
+ bits<14> src1;
+ bits<14> src2;
+ let Inst{6-0} = 0b1101111;
+ let Inst{14-7} = dst{7-0};
+ let Inst{15} = 0;
+ let Inst{25-16} = src0{9-0};
+ let Inst{26} = 1;
+ let Inst{27} = 1;
+ let Inst{37-28} = src1{9-0};
+ let Inst{39-38} = src1{11-10};
+ let Inst{49-40} = src2{9-0};
+ let Inst{51-50} = src2{11-10};
+ let Inst{53-52} = src0{11-10};
+ let Inst{55-54} = src2{13-12};
+ let Inst{57-56} = src1{13-12};
+ let Inst{59-58} = src0{13-12};
+ let Inst{61-60} = dst{9-8};
+ let Inst{63} = 1;
+}
+
+//===----------------------------------------------------------------------===//
+// Execution mask control flow
+//===----------------------------------------------------------------------===//
+
+// cf: 0 = if, 1 = else, 2 = while. The nesting counter r0l is implicit.
+class G13ICmpCF<string mn, bits<2> cf>
+ : AGXReal<mn, (outs),
+ (ins AGXNestCounter:$nest, AGXICond:$cc, AGXSrc:$src0,
+ AGXSrc:$src1, AGXImm:$n),
+ "$nest, $cc, $src0, $src1, $n", 6> {
+ bits<1> nest;
+ bits<4> cc;
+ bits<12> src0;
+ bits<12> src1;
+ bits<2> n;
+ let IsExecMaskOp = 1;
+ let Inst{6-0} = 0b1010010;
+ let Inst{7} = nest;
+ let Inst{8} = cc{3};
+ let Inst{10-9} = cf;
+ let Inst{12-11} = n;
+ let Inst{15-13} = cc{2-0};
+ let Inst{25-16} = src0{9-0};
+ let Inst{27-26} = 0;
+ let Inst{37-28} = src1{9-0};
+ let Inst{39-38} = 0;
+ let Inst{41-40} = src1{11-10};
+ let Inst{43-42} = src0{11-10};
+ let Inst{45-44} = 0;
+}
+
+class G13FCmpCF<string mn, bits<2> cf>
+ : AGXReal<mn, (outs),
+ (ins AGXNestCounter:$nest, AGXFCond:$cc, AGXSrcF:$src0,
+ AGXSrcF:$src1, AGXImm:$n),
+ "$nest, $cc, $src0, $src1, $n", 6> {
+ bits<1> nest;
+ bits<4> cc;
+ bits<14> src0;
+ bits<14> src1;
+ bits<2> n;
+ let IsExecMaskOp = 1;
+ let Inst{6-0} = 0b1000010;
+ let Inst{7} = nest;
+ let Inst{8} = cc{3};
+ let Inst{10-9} = cf;
+ let Inst{12-11} = n;
+ let Inst{15-13} = cc{2-0};
+ let Inst{27-16} = src0{11-0};
+ let Inst{39-28} = src1{11-0};
+ let Inst{41-40} = src1{13-12};
+ let Inst{43-42} = src0{13-12};
+ let Inst{45-44} = 0;
+}
+
+def IF_ICMP_g13 : G13ICmpCF<"if_icmp", 0>;
+def ELSE_ICMP_g13 : G13ICmpCF<"else_icmp", 1>;
+def WHILE_ICMP_g13 : G13ICmpCF<"while_icmp", 2>;
+def IF_FCMP_g13 : G13FCmpCF<"if_fcmp", 0>;
+def ELSE_FCMP_g13 : G13FCmpCF<"else_fcmp", 1>;
+def WHILE_FCMP_g13 : G13FCmpCF<"while_fcmp", 2>;
+
+def POP_EXEC_g13 : AGXReal<"pop_exec", (outs),
+ (ins AGXNestCounter:$nest, AGXImm:$n),
+ "$nest, $n", 6> {
+ bits<1> nest;
+ bits<2> n;
+ let IsExecMaskOp = 1;
+ let Inst{6-0} = 0b1010010;
+ let Inst{7} = nest;
+ let Inst{8} = 0;
+ let Inst{10-9} = 3;
+ let Inst{12-11} = n;
+ let Inst{47-13} = 0;
+}
+
+let isBranch = 1, isTerminator = 1 in {
+def JMP_EXEC_ANY_g13 : AGXReal<"jmp_exec_any", (outs),
+ (ins AGXBrTarget:$target), "$target", 6> {
+ bits<32> target;
+ let Inst{15-0} = 0xC000;
+ let Inst{47-16} = target;
+}
+
+def JMP_EXEC_NONE_g13 : AGXReal<"jmp_exec_none", (outs),
+ (ins AGXBrTarget:$target), "$target", 6> {
+ bits<32> target;
+ let Inst{15-0} = 0xC020;
+ let Inst{47-16} = target;
+}
+}
+
+let isTerminator = 1, isReturn = 1, isBarrier = 1 in
+def STOP_g13 : AGXReal<"stop", (outs), (ins), "", 2> {
+ let Inst{15-0} = 0x0088;
+}
+
+let isTrap = 1, isTerminator = 1, isBarrier = 1 in
+def TRAP_g13 : AGXReal<"trap", (outs), (ins), "", 2> {
+ let Inst{15-0} = 0x0008;
+}
+
+def WAIT_g13 : AGXReal<"wait", (outs), (ins AGXImm:$slot), "$slot", 2> {
+ bits<1> slot;
+ let Inst{7-0} = 0x38;
+ let Inst{8} = slot;
+}
+
+def THREADGROUP_BARRIER_g13 : AGXReal<"threadgroup_barrier", (outs), (ins),
+ "", 2> {
+ let Inst{15-0} = 0x0068;
+}
+
+def MEMORY_BARRIER_g13 : AGXReal<"memory_barrier", (outs),
+ (ins AGXImm:$a, AGXImm:$b, AGXImm:$c),
+ "$a, $b, $c", 2> {
+ bits<2> a;
+ bits<2> b;
+ bits<4> c;
+ let Inst{7-0} = 0xF5;
+ let Inst{9-8} = b;
+ let Inst{11-10} = a;
+ let Inst{15-12} = c;
+}
+
+//===----------------------------------------------------------------------===//
+// Device memory
+//===----------------------------------------------------------------------===//
+
+// Fields shared by device memory, uniform store and stack instructions:
+// R register 15:10 + 41:40 with 32-bit flag 49; index O 23:20 + 35:32 + 63:56
+// with immediate flag 24; scoreboard slot 30; length bit 47 (always set).
+class G13DeviceMem<string mn, bits<7> op, dag outs, dag ins, string ops>
+ : AGXReal<mn, outs, ins, ops, 8> {
+ bits<1> g;
+ bits<4> fmt;
+ bits<4> mask;
+ bits<9> R;
+ bits<9> A;
+ bits<17> O;
+ bits<1> sgn;
+ bits<2> shift;
+ let IsMemory = 1;
+ let Inst{6-0} = op;
+ let Inst{9-7} = fmt{2-0};
+ let Inst{15-10} = R{5-0};
+ let Inst{19-16} = A{3-0};
+ let Inst{23-20} = O{3-0};
+ let Inst{24} = O{16};
+ let Inst{25} = sgn;
+ let Inst{26} = 1;
+ let Inst{27} = A{8};
+ let Inst{29-28} = 0;
+ let Inst{30} = g;
+ let Inst{35-32} = O{7-4};
+ let Inst{39-36} = A{7-4};
+ let Inst{41-40} = R{7-6};
+ let Inst{43-42} = shift;
+ let Inst{46-44} = 4;
+ let Inst{47} = 1;
+ let Inst{48} = fmt{3};
+ let Inst{49} = R{8};
+ let Inst{51-50} = 0;
+ let Inst{55-52} = mask;
+ let Inst{63-56} = O{15-8};
+}
+
+let mayLoad = 1, IsAsync = 1 in
+def DEVICE_LOAD_g13
+ : G13DeviceMem<"device_load", 0b0000101, (outs AGXMemReg:$R),
+ (ins AGXImm:$g, AGXMemFormat:$fmt, AGXMemMask:$mask,
+ AGXMemBase:$A, AGXMemIndex:$O, AGXSignedness:$sgn,
+ AGXShift:$shift),
+ "$g, $fmt, $mask, $R, $A, $O, $sgn$shift">;
+
+let mayStore = 1 in
+def DEVICE_STORE_g13
+ : G13DeviceMem<"device_store", 0b1000101, (outs),
+ (ins AGXImm:$g, AGXMemFormat:$fmt, AGXMemMask:$mask,
+ AGXMemReg:$R, AGXMemBase:$A, AGXMemIndex:$O,
+ AGXSignedness:$sgn, AGXShift:$shift),
+ "$g, $fmt, $mask, $R, $A, $O, $sgn$shift">;
+
+// Initialise uniform registers from general purpose registers. The index is
+// the first 16-bit uniform register written.
+let mayStore = 1, hasSideEffects = 1 in
+def UNIFORM_STORE_g13
+ : AGXReal<"uniform_store", (outs),
+ (ins AGXImm:$g, AGXMemMask:$mask, AGXMemReg:$R,
+ AGXMemIndex:$O),
+ "$g, $mask, $R, $O", 8> {
+ bits<1> g;
+ bits<4> mask;
+ bits<9> R;
+ bits<17> O;
+ let IsMemory = 1;
+ let Inst{6-0} = 0b1000101;
+ let Inst{9-7} = 1;
+ let Inst{15-10} = R{5-0};
+ let Inst{19-16} = 0;
+ let Inst{23-20} = O{3-0};
+ let Inst{24} = O{16};
+ let Inst{26-25} = 0b10;
+ let Inst{29-27} = 0b111;
+ let Inst{30} = g;
+ let Inst{35-32} = O{7-4};
+ let Inst{39-36} = 0;
+ let Inst{41-40} = R{7-6};
+ let Inst{43-42} = 0;
+ let Inst{46-44} = 0;
+ let Inst{47} = 1;
+ let Inst{48} = 0;
+ let Inst{49} = R{8};
+ let Inst{51-50} = 0;
+ let Inst{55-52} = mask;
+ let Inst{63-56} = O{15-8};
+}
+
+//===----------------------------------------------------------------------===//
+// Stack (scratch) memory
+//===----------------------------------------------------------------------===//
+
+class G13StackMem<string mn, bits<8> op, dag outs, dag ins>
+ : AGXReal<mn, outs, ins, "$g, $fmt, $mask, $R, $O", 8> {
+ bits<1> g;
+ bits<4> fmt;
+ bits<4> mask;
+ bits<9> R;
+ bits<17> O;
+ let IsMemory = 1;
+ let Inst{7-0} = op;
+ let Inst{9-8} = fmt{1-0};
+ let Inst{15-10} = R{5-0};
+ let Inst{19-16} = 0;
+ let Inst{23-20} = O{3-0};
+ let Inst{24} = O{16};
+ let Inst{25} = 0;
+ let Inst{26} = 1;
+ let Inst{29-27} = 0;
+ let Inst{30} = g;
+ let Inst{35-32} = O{7-4};
+ let Inst{38-36} = 0;
+ let Inst{41-40} = R{7-6};
+ let Inst{46-44} = 4;
+ let Inst{47} = 1;
+ let Inst{49} = R{8};
+ let Inst{51-50} = fmt{3-2};
+ let Inst{55-52} = mask;
+ let Inst{63-56} = O{15-8};
+}
+
+let mayLoad = 1, IsAsync = 1 in
+def STACK_LOAD_g13
+ : G13StackMem<"stack_load", 0x35, (outs AGXMemReg:$R),
+ (ins AGXImm:$g, AGXMemFormat:$fmt, AGXMemMask:$mask,
+ AGXMemIndex:$O)>;
+
+let mayStore = 1 in
+def STACK_STORE_g13
+ : G13StackMem<"stack_store", 0xB5, (outs),
+ (ins AGXImm:$g, AGXMemFormat:$fmt, AGXMemMask:$mask,
+ AGXMemReg:$R, AGXMemIndex:$O)>;
+
+let hasSideEffects = 1 in
+def STACK_ADJUST_g13 : AGXReal<"stack_adjust", (outs), (ins AGXImm:$size),
+ "$size", 8> {
+ bits<16> size;
+ let Inst{7-0} = 0xB5;
+ let Inst{19-16} = 1;
+ let Inst{23-20} = size{3-0};
+ let Inst{24} = 1;
+ let Inst{25} = 0;
+ let Inst{26} = 1;
+ let Inst{35-32} = size{7-4};
+ let Inst{38-36} = 2;
+ let Inst{46-44} = 0;
+ let Inst{47} = 1;
+ let Inst{55-50} = 0;
+ let Inst{63-56} = size{15-8};
+}
+
+//===----------------------------------------------------------------------===//
+// Threadgroup memory
+//===----------------------------------------------------------------------===//
+
+class G13LocalMem<string mn, bits<2> op, dag outs, dag ins>
+ : AGXReal<mn, outs, ins, "$fmt, $mask, $R, $A, $O", 8> {
+ bits<4> fmt;
+ bits<4> mask;
+ bits<9> R;
+ bits<10> A;
+ bits<17> O;
+ let IsMemory = 1;
+ let Inst{3-0} = 0b1001;
+ let Inst{4} = 0;
+ let Inst{6-5} = op;
+ let Inst{7} = 0;
+ let Inst{8} = R{8};
+ let Inst{14-9} = R{5-0};
+ let Inst{15} = 1;
+ let Inst{21-16} = A{5-0};
+ let Inst{23-22} = A{9-8};
+ let Inst{27-24} = fmt;
+ let Inst{33-28} = O{5-0};
+ let Inst{34} = O{16};
+ let Inst{35} = 0;
+ let Inst{39-36} = mask;
+ let Inst{47-40} = 0;
+ let Inst{57-48} = O{15-6};
+ let Inst{59-58} = A{7-6};
+ let Inst{61-60} = R{7-6};
+ let Inst{63-62} = 0;
+}
+
+let mayLoad = 1 in
+def THREADGROUP_LOAD_g13
+ : G13LocalMem<"threadgroup_load", 0b11, (outs AGXMemReg:$R),
+ (ins AGXMemFormat:$fmt, AGXMemMask:$mask, AGXLocalBase:$A,
+ AGXLocalIndex:$O)>;
+
+let mayStore = 1 in
+def THREADGROUP_STORE_g13
+ : G13LocalMem<"threadgroup_store", 0b01, (outs),
+ (ins AGXMemFormat:$fmt, AGXMemMask:$mask, AGXMemReg:$R,
+ AGXLocalBase:$A, AGXLocalIndex:$O)>;
+
+//===----------------------------------------------------------------------===//
+// Atomics
+//===----------------------------------------------------------------------===//
+
+let mayLoad = 1, mayStore = 1, hasSideEffects = 1, IsAsync = 1 in
+def ATOMIC_g13 : AGXReal<"atomic", (outs AGXAtomicDst:$R),
+ (ins AGXAtomicOp:$op, AGXImm:$g, AGXMemBase:$A,
+ AGXMemIndex:$O, AGXSignedness:$sgn,
+ AGXAtomicSrc:$S, AGXImm:$coherent),
+ "$op, $g, $R, $A, $O, $sgn, $S, $coherent", 8> {
+ bits<4> op;
+ bits<1> g;
+ bits<9> R;
+ bits<9> A;
+ bits<17> O;
+ bits<1> sgn;
+ bits<8> S;
+ bits<1> coherent;
+ let IsMemory = 1;
+ let Inst{5-0} = 0b010101;
+ let Inst{9-6} = op;
+ let Inst{15-10} = R{5-0};
+ let Inst{19-16} = A{3-0};
+ let Inst{23-20} = O{3-0};
+ let Inst{24} = O{16};
+ let Inst{25} = sgn;
+ let Inst{26} = 1;
+ let Inst{27} = A{8};
+ let Inst{29-28} = 0;
+ let Inst{30} = g;
+ let Inst{31} = 1;
+ let Inst{35-32} = O{7-4};
+ let Inst{39-36} = A{7-4};
+ let Inst{41-40} = R{7-6};
+ let Inst{43-42} = 0;
+ let Inst{44} = 1;
+ let Inst{45} = coherent;
+ let Inst{46} = 1;
+ let Inst{47} = R{8};
+ let Inst{55-48} = S;
+ let Inst{63-56} = O{15-8};
+}
+
+let mayLoad = 1, mayStore = 1, hasSideEffects = 1 in
+def THREADGROUP_ATOMIC_g13
+ : AGXReal<"threadgroup_atomic", (outs AGXAtomicDst:$R),
+ (ins AGXAtomicOp:$op, AGXLocalBase:$A, AGXLocalIndex:$O,
+ AGXAtomicSrc:$S),
+ "$op, $R, $A, $O, $S", 10> {
+ bits<4> op;
+ bits<9> R;
+ bits<10> A;
+ bits<17> O;
+ bits<8> S;
+ let IsMemory = 1;
+ let Inst{5-0} = 0b011001;
+ let Inst{7-6} = 0;
+ let Inst{8} = R{8};
+ let Inst{14-9} = R{5-0};
+ let Inst{15} = 1;
+ let Inst{21-16} = A{5-0};
+ let Inst{23-22} = A{9-8};
+ let Inst{27-24} = op;
+ let Inst{33-28} = O{5-0};
+ let Inst{34} = O{16};
+ let Inst{37-35} = 0b010;
+ let Inst{38} = R{8};
+ let Inst{46-39} = 0;
+ let Inst{47} = 1;
+ let Inst{57-48} = O{15-6};
+ let Inst{59-58} = A{7-6};
+ let Inst{61-60} = R{7-6};
+ let Inst{63-62} = 0;
+ let Inst{71-64} = S;
+ let Inst{79-72} = 0;
+}
diff --git a/llvm/lib/Target/AGX/MCTargetDesc/AGXAsmBackend.cpp b/llvm/lib/Target/AGX/MCTargetDesc/AGXAsmBackend.cpp
new file mode 100644
index 0000000000000..de526bd0b9e30
--- /dev/null
+++ b/llvm/lib/Target/AGX/MCTargetDesc/AGXAsmBackend.cpp
@@ -0,0 +1,110 @@
+//===-- AGXAsmBackend.cpp - AGX assembler backend -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MCTargetDesc/AGXFixupKinds.h"
+#include "MCTargetDesc/AGXMCTargetDesc.h"
+#include "llvm/MC/MCAsmBackend.h"
+#include "llvm/MC/MCAssembler.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCELFObjectWriter.h"
+#include "llvm/MC/MCObjectWriter.h"
+#include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/MC/MCValue.h"
+#include "llvm/Support/EndianStream.h"
+
+using namespace llvm;
+
+namespace {
+
+class AGXAsmBackend : public MCAsmBackend {
+ uint8_t OSABI;
+
+public:
+ explicit AGXAsmBackend(uint8_t OSABI)
+ : MCAsmBackend(llvm::endianness::little), OSABI(OSABI) {}
+
+ void applyFixup(const MCFragment &F, const MCFixup &Fixup,
+ const MCValue &Target, uint8_t *Data, uint64_t Value,
+ bool IsResolved) override;
+
+ std::unique_ptr<MCObjectTargetWriter>
+ createObjectTargetWriter() const override {
+ return createAGXELFObjectWriter(OSABI);
+ }
+
+ MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
+
+ bool writeNopData(raw_ostream &OS, uint64_t Count,
+ const MCSubtargetInfo *STI) const override;
+};
+
+} // namespace
+
+void AGXAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
+ const MCValue &Target, uint8_t *Data,
+ uint64_t Value, bool IsResolved) {
+ if (!IsResolved)
+ Asm->getWriter().recordRelocation(F, Fixup, Target, Value);
+
+ switch (Fixup.getKind()) {
+ case AGX::fixup_agx_branch32: {
+ // The offset is relative to the start of the branch, two bytes before the
+ // fixup.
+ int64_t Offset = int64_t(Value) + 2;
+ if (!isInt<32>(Offset))
+ getContext().reportError(Fixup.getLoc(), "branch target out of range");
+ support::endian::write32le(Data, uint32_t(Offset));
+ return;
+ }
+ case FK_Data_1:
+ Data[0] = uint8_t(Value);
+ return;
+ case FK_Data_2:
+ support::endian::write16le(Data, uint16_t(Value));
+ return;
+ case FK_Data_4:
+ support::endian::write32le(Data, uint32_t(Value));
+ return;
+ case FK_Data_8:
+ support::endian::write64le(Data, Value);
+ return;
+ default:
+ llvm_unreachable("unknown fixup kind");
+ }
+}
+
+MCFixupKindInfo AGXAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
+ static const MCFixupKindInfo Infos[AGX::NumTargetFixupKinds] = {
+ // name offset bits flags
+ {"fixup_agx_branch32", 0, 32, 0},
+ };
+ if (Kind < FirstTargetFixupKind)
+ return MCAsmBackend::getFixupKindInfo(Kind);
+ assert(unsigned(Kind - FirstTargetFixupKind) < AGX::NumTargetFixupKinds &&
+ "invalid kind");
+ return Infos[Kind - FirstTargetFixupKind];
+}
+
+bool AGXAsmBackend::writeNopData(raw_ostream &OS, uint64_t Count,
+ const MCSubtargetInfo *STI) const {
+ // There is no documented no-op. Pad with trap, which is never executed.
+ if (Count % 2)
+ return false;
+ for (uint64_t I = 0; I < Count; I += 2)
+ support::endian::write<uint16_t>(OS, 0x0008, llvm::endianness::little);
+ return true;
+}
+
+MCAsmBackend *llvm::createAGXAsmBackend(const Target &T,
+ const MCSubtargetInfo &STI,
+ const MCRegisterInfo &MRI,
+ const MCTargetOptions &Options) {
+ uint8_t OSABI =
+ MCELFObjectTargetWriter::getOSABI(STI.getTargetTriple().getOS());
+ return new AGXAsmBackend(OSABI);
+}
diff --git a/llvm/lib/Target/AGX/MCTargetDesc/AGXELFObjectWriter.cpp b/llvm/lib/Target/AGX/MCTargetDesc/AGXELFObjectWriter.cpp
new file mode 100644
index 0000000000000..c8c6abd371887
--- /dev/null
+++ b/llvm/lib/Target/AGX/MCTargetDesc/AGXELFObjectWriter.cpp
@@ -0,0 +1,42 @@
+//===-- AGXELFObjectWriter.cpp - AGX ELF writer ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// AGX objects are ELF64 files whose .text holds raw instructions. Branches
+// are resolved by the assembler; there are no AGX relocation types.
+//
+//===----------------------------------------------------------------------===//
+
+#include "MCTargetDesc/AGXMCTargetDesc.h"
+#include "llvm/BinaryFormat/ELF.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCELFObjectWriter.h"
+#include "llvm/MC/MCFixup.h"
+#include "llvm/MC/MCObjectWriter.h"
+
+using namespace llvm;
+
+namespace {
+class AGXELFObjectWriter : public MCELFObjectTargetWriter {
+public:
+ explicit AGXELFObjectWriter(uint8_t OSABI)
+ : MCELFObjectTargetWriter(/*Is64Bit=*/true, OSABI, ELF::EM_AGX,
+ /*HasRelocationAddend=*/true) {}
+
+protected:
+ unsigned getRelocType(const MCFixup &Fixup, const MCValue &,
+ bool) const override {
+ reportError(Fixup.getLoc(), "AGX does not support relocations");
+ return 0;
+ }
+};
+} // namespace
+
+std::unique_ptr<MCObjectTargetWriter>
+llvm::createAGXELFObjectWriter(uint8_t OSABI) {
+ return std::make_unique<AGXELFObjectWriter>(OSABI);
+}
diff --git a/llvm/lib/Target/AGX/MCTargetDesc/AGXFixupKinds.h b/llvm/lib/Target/AGX/MCTargetDesc/AGXFixupKinds.h
new file mode 100644
index 0000000000000..3e104573c6920
--- /dev/null
+++ b/llvm/lib/Target/AGX/MCTargetDesc/AGXFixupKinds.h
@@ -0,0 +1,27 @@
+//===-- AGXFixupKinds.h - AGX specific fixup entries ------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AGX_MCTARGETDESC_AGXFIXUPKINDS_H
+#define LLVM_LIB_TARGET_AGX_MCTARGETDESC_AGXFIXUPKINDS_H
+
+#include "llvm/MC/MCFixup.h"
+
+namespace llvm {
+namespace AGX {
+enum Fixups {
+ /// 32-bit branch offset, relative to the start of the branch instruction.
+ /// The field starts 2 bytes into the instruction.
+ fixup_agx_branch32 = FirstTargetFixupKind,
+
+ LastTargetFixupKind,
+ NumTargetFixupKinds = LastTargetFixupKind - FirstTargetFixupKind
+};
+} // namespace AGX
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AGX_MCTARGETDESC_AGXFIXUPKINDS_H
diff --git a/llvm/lib/Target/AGX/MCTargetDesc/AGXInstPrinter.cpp b/llvm/lib/Target/AGX/MCTargetDesc/AGXInstPrinter.cpp
new file mode 100644
index 0000000000000..96cf2c76ac6d1
--- /dev/null
+++ b/llvm/lib/Target/AGX/MCTargetDesc/AGXInstPrinter.cpp
@@ -0,0 +1,346 @@
+//===-- AGXInstPrinter.cpp - Convert AGX MCInst to assembly ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The syntax follows the applegpu disassembler (dougallj/applegpu), so that
+// output can be compared with it directly: registers r0l/r0h/r0/r0_r1,
+// uniforms u0l/u0/u0_u1, and source flags as suffixes (.cache, .discard,
+// .abs, .neg, .sx).
+//
+//===----------------------------------------------------------------------===//
+
+#include "AGXInstPrinter.h"
+#include "MCTargetDesc/AGXMCTargetDesc.h"
+#include "llvm/ADT/bit.h"
+#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "asm-printer"
+
+#define PRINT_ALIAS_INSTR
+#include "AGXGenAsmWriter.inc"
+
+void AGXInstPrinter::printInst(const MCInst *MI, uint64_t Address,
+ StringRef Annot, const MCSubtargetInfo &STI,
+ raw_ostream &O) {
+ if (!printAliasInstr(MI, Address, STI, O))
+ printInstruction(MI, Address, STI, O);
+ printAnnotation(O, Annot);
+}
+
+void AGXInstPrinter::printRegName(raw_ostream &O, MCRegister Reg) {
+ printReg(Reg, O);
+}
+
+void AGXInstPrinter::printReg(MCRegister Reg, raw_ostream &O) {
+ AGX::RegInfo Info = AGX::getRegInfo(Reg, MRI);
+ unsigned Count = 0;
+ AGX::RegKind EltKind = AGX::RegKind::GPR32;
+ switch (Info.Kind) {
+ case AGX::RegKind::GPR64:
+ Count = 2;
+ break;
+ case AGX::RegKind::GPR96:
+ Count = 3;
+ break;
+ case AGX::RegKind::GPR128:
+ Count = 4;
+ break;
+ case AGX::RegKind::UGPR64:
+ Count = 2;
+ EltKind = AGX::RegKind::UGPR32;
+ break;
+ default:
+ O << getRegisterName(Reg);
+ return;
+ }
+ for (unsigned I = 0; I < Count; ++I) {
+ if (I)
+ O << '_';
+ O << getRegisterName(
+ AGX::getRegister(EltKind, Info.HalfIndex + 2 * I, MRI));
+ }
+}
+
+static void printFlag(raw_ostream &O, unsigned Mods, unsigned Flag,
+ StringRef Name) {
+ if (Mods & Flag)
+ O << '.' << Name;
+}
+
+void AGXInstPrinter::printALUSrc(const MCInst *MI, unsigned OpNo, bool IsFloat,
+ raw_ostream &O) {
+ unsigned Mods = MI->getOperand(OpNo).getImm();
+ const MCOperand &Val = MI->getOperand(OpNo + 1);
+ if (Val.isImm()) {
+ if (IsFloat)
+ O << AGX::formatFPImm8(Val.getImm() & 0xff);
+ else
+ O << Val.getImm();
+ return;
+ }
+ if (Val.isExpr()) {
+ MAI.printExpr(O, *Val.getExpr());
+ return;
+ }
+ printReg(Val.getReg(), O);
+ printFlag(O, Mods, AGX::SRC_MOD_CACHE, "cache");
+ printFlag(O, Mods, AGX::SRC_MOD_DISCARD, "discard");
+ printFlag(O, Mods, AGX::SRC_MOD_ABS, "abs");
+ printFlag(O, Mods, AGX::SRC_MOD_NEG, "neg");
+ printFlag(O, Mods, AGX::SRC_MOD_SX, "sx");
+}
+
+#define AGX_INT_SRC_PRINTER(Kind) \
+ void AGXInstPrinter::print##Kind(const MCInst *MI, unsigned OpNo, \
+ const MCSubtargetInfo &, raw_ostream &O) { \
+ printALUSrc(MI, OpNo, /*IsFloat=*/false, O); \
+ }
+AGX_INT_SRC_PRINTER(Src)
+AGX_INT_SRC_PRINTER(Src64)
+AGX_INT_SRC_PRINTER(Src16)
+AGX_INT_SRC_PRINTER(SrcSX)
+AGX_INT_SRC_PRINTER(SrcSX64)
+AGX_INT_SRC_PRINTER(CmpselSrc)
+#undef AGX_INT_SRC_PRINTER
+
+void AGXInstPrinter::printSrcF(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &, raw_ostream &O) {
+ printALUSrc(MI, OpNo, /*IsFloat=*/true, O);
+}
+
+void AGXInstPrinter::printSrcF16(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &, raw_ostream &O) {
+ printALUSrc(MI, OpNo, /*IsFloat=*/true, O);
+}
+
+void AGXInstPrinter::printDst(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &, raw_ostream &O) {
+ printReg(MI->getOperand(OpNo).getReg(), O);
+ if (MI->getOperand(OpNo + 1).getImm())
+ O << ".cache";
+}
+
+void AGXInstPrinter::printImm(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &, raw_ostream &O) {
+ const MCOperand &Op = MI->getOperand(OpNo);
+ if (Op.isImm())
+ O << Op.getImm();
+ else if (Op.isExpr())
+ MAI.printExpr(O, *Op.getExpr());
+ else
+ O << getRegisterName(Op.getReg());
+}
+
+void AGXInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &STI,
+ raw_ostream &O) {
+ printImm(MI, OpNo, STI, O);
+}
+
+void AGXInstPrinter::printShift(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &, raw_ostream &O) {
+ if (int64_t S = MI->getOperand(OpNo).getImm())
+ O << ", lsl " << S;
+}
+
+void AGXInstPrinter::printBitfieldMask(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &,
+ raw_ostream &O) {
+ if (int64_t M = MI->getOperand(OpNo).getImm())
+ O << ", mask 0x" << utohexstr((uint64_t(1) << M) - 1, /*LowerCase=*/false);
+}
+
+void AGXInstPrinter::printTruthTable(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &, raw_ostream &O) {
+ int64_t TT = MI->getOperand(OpNo).getImm();
+ for (unsigned I = 0; I < 4; ++I)
+ O << ((TT >> I) & 1);
+}
+
+static void printEnum(raw_ostream &O, StringRef Name, int64_t Value) {
+ if (Name.empty())
+ O << Value;
+ else
+ O << Name;
+}
+
+void AGXInstPrinter::printICond(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &, raw_ostream &O) {
+ int64_t CC = MI->getOperand(OpNo).getImm();
+ printEnum(O, AGX::getICondName(CC), CC);
+}
+
+void AGXInstPrinter::printFCond(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &, raw_ostream &O) {
+ int64_t CC = MI->getOperand(OpNo).getImm();
+ printEnum(O, AGX::getFCondName(CC), CC);
+}
+
+void AGXInstPrinter::printICond3(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &STI, raw_ostream &O) {
+ printICond(MI, OpNo, STI, O);
+}
+
+void AGXInstPrinter::printFCond3(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &STI, raw_ostream &O) {
+ printFCond(MI, OpNo, STI, O);
+}
+
+void AGXInstPrinter::printConvertMode(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &, raw_ostream &O) {
+ int64_t V = MI->getOperand(OpNo).getImm();
+ printEnum(O, AGX::getConvertModeName(V), V);
+}
+
+void AGXInstPrinter::printRoundMode(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &, raw_ostream &O) {
+ int64_t V = MI->getOperand(OpNo).getImm();
+ printEnum(O, AGX::getRoundModeName(V), V);
+}
+
+void AGXInstPrinter::printMemFormat(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &, raw_ostream &O) {
+ int64_t V = MI->getOperand(OpNo).getImm();
+ printEnum(O, AGX::getMemFormatName(V), V);
+}
+
+void AGXInstPrinter::printMemMask(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &, raw_ostream &O) {
+ int64_t V = MI->getOperand(OpNo).getImm();
+ std::string Name = AGX::getMemMaskName(V);
+ printEnum(O, Name, V);
+}
+
+void AGXInstPrinter::printSignedness(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &, raw_ostream &O) {
+ O << (MI->getOperand(OpNo).getImm() ? "unsigned" : "signed");
+}
+
+void AGXInstPrinter::printAtomicOp(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &, raw_ostream &O) {
+ int64_t V = MI->getOperand(OpNo).getImm();
+ printEnum(O, AGX::getAtomicOpName(V), V);
+}
+
+void AGXInstPrinter::printSpecialReg(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &, raw_ostream &O) {
+ int64_t SR = MI->getOperand(OpNo).getImm();
+ O << "sr" << SR;
+ StringRef Name = AGX::getSpecialRegName(SR);
+ if (!Name.empty())
+ O << " (" << Name << ')';
+}
+
+void AGXInstPrinter::printRegisterTuple(MCRegister First, unsigned Count,
+ raw_ostream &O) {
+ AGX::RegInfo Info = AGX::getRegInfo(First, MRI);
+ bool Is16 = Info.Kind == AGX::RegKind::GPR16;
+ unsigned Step = Is16 ? 1 : 2;
+ AGX::RegKind Kind = Is16 ? AGX::RegKind::GPR16 : AGX::RegKind::GPR32;
+ for (unsigned I = 0; I < std::max(Count, 1u); ++I) {
+ if (I)
+ O << '_';
+ MCRegister R = AGX::getRegister(Kind, Info.HalfIndex + I * Step, MRI);
+ if (R)
+ O << getRegisterName(R);
+ else
+ O << "<out of range>";
+ }
+}
+
+static unsigned getMemCount(const MCInst *MI) {
+ int16_t MaskIdx = AGX::getNamedOperandIdx(MI->getOpcode(), AGX::OpName::mask);
+ if (MaskIdx < 0)
+ return 1;
+ return llvm::popcount(uint64_t(MI->getOperand(MaskIdx).getImm()));
+}
+
+void AGXInstPrinter::printMemReg(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &, raw_ostream &O) {
+ printRegisterTuple(MI->getOperand(OpNo).getReg(), getMemCount(MI), O);
+}
+
+void AGXInstPrinter::printAtomicDst(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &, raw_ostream &O) {
+ MCRegister R = MI->getOperand(OpNo).getReg();
+ if (!R)
+ O << '_';
+ else
+ O << getRegisterName(R);
+}
+
+void AGXInstPrinter::printAtomicSrc(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &, raw_ostream &O) {
+ printReg(MI->getOperand(OpNo).getReg(), O);
+}
+
+void AGXInstPrinter::printMemBase(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &, raw_ostream &O) {
+ printReg(MI->getOperand(OpNo).getReg(), O);
+}
+
+void AGXInstPrinter::printMemIndex(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &, raw_ostream &O) {
+ const MCOperand &Op = MI->getOperand(OpNo);
+ if (Op.isReg())
+ O << getRegisterName(Op.getReg());
+ else
+ O << int16_t(Op.getImm());
+}
+
+void AGXInstPrinter::printLocalBase(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &, raw_ostream &O) {
+ const MCOperand &Op = MI->getOperand(OpNo);
+ if (Op.isReg())
+ O << getRegisterName(Op.getReg());
+ else
+ O << Op.getImm();
+}
+
+void AGXInstPrinter::printLocalIndex(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &, raw_ostream &O) {
+ const MCOperand &Op = MI->getOperand(OpNo);
+ if (Op.isReg())
+ O << getRegisterName(Op.getReg());
+ else
+ O << int16_t(Op.getImm());
+}
+
+void AGXInstPrinter::printNestCounter(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &, raw_ostream &O) {
+ O << "r0l";
+ if (MI->getOperand(OpNo).getImm())
+ O << ".cache";
+}
+
+void AGXInstPrinter::printBrTarget(const MCInst *MI, uint64_t Address,
+ unsigned OpNo, const MCSubtargetInfo &,
+ raw_ostream &O) {
+ const MCOperand &Op = MI->getOperand(OpNo);
+ if (Op.isExpr()) {
+ MAI.printExpr(O, *Op.getExpr());
+ return;
+ }
+ int64_t Offset = int32_t(Op.getImm());
+ if (PrintBranchImmAsAddress) {
+ O << formatHex(uint64_t(Address + Offset));
+ return;
+ }
+ if (Offset >= 0)
+ O << "pc+" << Offset;
+ else
+ O << "pc-" << -Offset;
+}
diff --git a/llvm/lib/Target/AGX/MCTargetDesc/AGXInstPrinter.h b/llvm/lib/Target/AGX/MCTargetDesc/AGXInstPrinter.h
new file mode 100644
index 0000000000000..7803eef10c56e
--- /dev/null
+++ b/llvm/lib/Target/AGX/MCTargetDesc/AGXInstPrinter.h
@@ -0,0 +1,89 @@
+//===-- AGXInstPrinter.h - Convert AGX MCInst to assembly -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AGX_MCTARGETDESC_AGXINSTPRINTER_H
+#define LLVM_LIB_TARGET_AGX_MCTARGETDESC_AGXINSTPRINTER_H
+
+#include "llvm/MC/MCInstPrinter.h"
+
+namespace llvm {
+
+class AGXInstPrinter : public MCInstPrinter {
+public:
+ AGXInstPrinter(const MCAsmInfo &MAI, const MCInstrInfo &MII,
+ const MCRegisterInfo &MRI)
+ : MCInstPrinter(MAI, MII, MRI) {}
+
+ void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
+ const MCSubtargetInfo &STI, raw_ostream &O) override;
+ void printRegName(raw_ostream &O, MCRegister Reg) override;
+
+ // Autogenerated by tblgen.
+ std::pair<const char *, uint64_t>
+ getMnemonic(const MCInst &MI) const override;
+ void printInstruction(const MCInst *MI, uint64_t Address,
+ const MCSubtargetInfo &STI, raw_ostream &O);
+ bool printAliasInstr(const MCInst *MI, uint64_t Address,
+ const MCSubtargetInfo &STI, raw_ostream &O);
+ void printCustomAliasOperand(const MCInst *MI, uint64_t Address,
+ unsigned OpIdx, unsigned PrintMethodIdx,
+ const MCSubtargetInfo &STI, raw_ostream &O);
+ static const char *getRegisterName(MCRegister Reg);
+
+#define AGX_SRC_PRINTER(Kind) \
+ void print##Kind(const MCInst *MI, unsigned OpNo, \
+ const MCSubtargetInfo &STI, raw_ostream &O);
+ AGX_SRC_PRINTER(Src)
+ AGX_SRC_PRINTER(Src64)
+ AGX_SRC_PRINTER(Src16)
+ AGX_SRC_PRINTER(SrcSX)
+ AGX_SRC_PRINTER(SrcSX64)
+ AGX_SRC_PRINTER(SrcF)
+ AGX_SRC_PRINTER(SrcF16)
+ AGX_SRC_PRINTER(CmpselSrc)
+ AGX_SRC_PRINTER(Dst)
+ AGX_SRC_PRINTER(Imm)
+ AGX_SRC_PRINTER(Shift)
+ AGX_SRC_PRINTER(BitfieldMask)
+ AGX_SRC_PRINTER(TruthTable)
+ AGX_SRC_PRINTER(ICond)
+ AGX_SRC_PRINTER(FCond)
+ AGX_SRC_PRINTER(ICond3)
+ AGX_SRC_PRINTER(FCond3)
+ AGX_SRC_PRINTER(ConvertMode)
+ AGX_SRC_PRINTER(RoundMode)
+ AGX_SRC_PRINTER(MemFormat)
+ AGX_SRC_PRINTER(MemMask)
+ AGX_SRC_PRINTER(Signedness)
+ AGX_SRC_PRINTER(AtomicOp)
+ AGX_SRC_PRINTER(SpecialReg)
+ AGX_SRC_PRINTER(MemReg)
+ AGX_SRC_PRINTER(AtomicDst)
+ AGX_SRC_PRINTER(MemBase)
+ AGX_SRC_PRINTER(MemIndex)
+ AGX_SRC_PRINTER(LocalBase)
+ AGX_SRC_PRINTER(LocalIndex)
+ AGX_SRC_PRINTER(AtomicSrc)
+ AGX_SRC_PRINTER(NestCounter)
+#undef AGX_SRC_PRINTER
+
+ void printOperand(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &STI, raw_ostream &O);
+ void printBrTarget(const MCInst *MI, uint64_t Address, unsigned OpNo,
+ const MCSubtargetInfo &STI, raw_ostream &O);
+
+private:
+ void printALUSrc(const MCInst *MI, unsigned OpNo, bool IsFloat,
+ raw_ostream &O);
+ void printReg(MCRegister Reg, raw_ostream &O);
+ void printRegisterTuple(MCRegister First, unsigned Count, raw_ostream &O);
+};
+
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AGX_MCTARGETDESC_AGXINSTPRINTER_H
diff --git a/llvm/lib/Target/AGX/MCTargetDesc/AGXMCAsmInfo.cpp b/llvm/lib/Target/AGX/MCTargetDesc/AGXMCAsmInfo.cpp
new file mode 100644
index 0000000000000..26441eb4e2aa8
--- /dev/null
+++ b/llvm/lib/Target/AGX/MCTargetDesc/AGXMCAsmInfo.cpp
@@ -0,0 +1,30 @@
+//===-- AGXMCAsmInfo.cpp - AGX asm properties -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AGXMCAsmInfo.h"
+#include "llvm/TargetParser/Triple.h"
+
+using namespace llvm;
+
+void AGXMCAsmInfo::anchor() {}
+
+AGXMCAsmInfo::AGXMCAsmInfo(const Triple &TT, const MCTargetOptions &Options)
+ : MCAsmInfoELF(Options) {
+ CodePointerSize = 8;
+ CalleeSaveStackSlotSize = 4;
+ IsLittleEndian = true;
+ CommentString = "//";
+ InternalSymbolPrefix = ".L";
+ // Instructions are 2 to 12 bytes long, in units of 2 bytes.
+ MinInstAlignment = 2;
+ MaxInstLength = 12;
+ SupportsDebugInformation = true;
+ UsesELFSectionDirectiveForBSS = true;
+ HasDotTypeDotSizeDirective = true;
+ HasSingleParameterDotFile = true;
+}
diff --git a/llvm/lib/Target/AGX/MCTargetDesc/AGXMCAsmInfo.h b/llvm/lib/Target/AGX/MCTargetDesc/AGXMCAsmInfo.h
new file mode 100644
index 0000000000000..1a4f7d5187855
--- /dev/null
+++ b/llvm/lib/Target/AGX/MCTargetDesc/AGXMCAsmInfo.h
@@ -0,0 +1,26 @@
+//===-- AGXMCAsmInfo.h - AGX asm properties ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AGX_MCTARGETDESC_AGXMCASMINFO_H
+#define LLVM_LIB_TARGET_AGX_MCTARGETDESC_AGXMCASMINFO_H
+
+#include "llvm/MC/MCAsmInfoELF.h"
+
+namespace llvm {
+class Triple;
+
+class AGXMCAsmInfo : public MCAsmInfoELF {
+ void anchor() override;
+
+public:
+ explicit AGXMCAsmInfo(const Triple &TT, const MCTargetOptions &Options);
+};
+
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AGX_MCTARGETDESC_AGXMCASMINFO_H
diff --git a/llvm/lib/Target/AGX/MCTargetDesc/AGXMCCodeEmitter.cpp b/llvm/lib/Target/AGX/MCTargetDesc/AGXMCCodeEmitter.cpp
new file mode 100644
index 0000000000000..399e2920329aa
--- /dev/null
+++ b/llvm/lib/Target/AGX/MCTargetDesc/AGXMCCodeEmitter.cpp
@@ -0,0 +1,386 @@
+//===-- AGXMCCodeEmitter.cpp - Convert AGX code to machine code -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The TableGen'erated encoder produces the long form of every instruction.
+// Instructions with a short and a long form are emitted in the short form
+// when all bits beyond it are zero; otherwise the length bit is set.
+//
+//===----------------------------------------------------------------------===//
+
+#include "MCTargetDesc/AGXFixupKinds.h"
+#include "MCTargetDesc/AGXMCTargetDesc.h"
+#include "llvm/ADT/APInt.h"
+#include "llvm/MC/MCCodeEmitter.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCFixup.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/Support/ErrorHandling.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "mccodeemitter"
+
+namespace {
+
+class AGXMCCodeEmitter : public MCCodeEmitter {
+ const MCInstrInfo &MCII;
+ MCContext &Ctx;
+
+public:
+ AGXMCCodeEmitter(const MCInstrInfo &MCII, MCContext &Ctx)
+ : MCII(MCII), Ctx(Ctx) {}
+
+ void encodeInstruction(const MCInst &MI, SmallVectorImpl<char> &CB,
+ SmallVectorImpl<MCFixup> &Fixups,
+ const MCSubtargetInfo &STI) const override;
+
+ // TableGen'erated.
+ void getBinaryCodeForInstr(const MCInst &MI, SmallVectorImpl<MCFixup> &Fixups,
+ APInt &Inst, APInt &Scratch,
+ const MCSubtargetInfo &STI) const;
+
+ void getMachineOpValue(const MCInst &MI, const MCOperand &MO, APInt &Op,
+ SmallVectorImpl<MCFixup> &Fixups,
+ const MCSubtargetInfo &STI) const;
+
+#define AGX_ENCODER(Kind) \
+ void get##Kind##Encoding(const MCInst &MI, unsigned OpNo, APInt &Op, \
+ SmallVectorImpl<MCFixup> &Fixups, \
+ const MCSubtargetInfo &STI) const;
+ AGX_ENCODER(Src)
+ AGX_ENCODER(Src64)
+ AGX_ENCODER(Src16)
+ AGX_ENCODER(SrcSX)
+ AGX_ENCODER(SrcSX64)
+ AGX_ENCODER(SrcF)
+ AGX_ENCODER(SrcF16)
+ AGX_ENCODER(CmpselSrc)
+ AGX_ENCODER(Dst)
+ AGX_ENCODER(MemReg)
+ AGX_ENCODER(AtomicDst)
+ AGX_ENCODER(MemBase)
+ AGX_ENCODER(MemIndex)
+ AGX_ENCODER(LocalBase)
+ AGX_ENCODER(LocalIndex)
+ AGX_ENCODER(AtomicSrc)
+ AGX_ENCODER(BrTarget)
+#undef AGX_ENCODER
+
+private:
+ AGX::RegInfo regInfo(const MCOperand &MO) const {
+ return AGX::getRegInfo(MO.getReg(), *Ctx.getRegisterInfo());
+ }
+ /// The source field in the common layout: [5:0] value, [9:6] type and
+ /// [11:10] extension.
+ uint64_t encodeALUSrc(const MCInst &MI, unsigned OpNo) const;
+ void report(const MCInst &MI, const Twine &Msg) const;
+};
+
+} // namespace
+
+void AGXMCCodeEmitter::report(const MCInst &MI, const Twine &Msg) const {
+ Ctx.reportError(MI.getLoc(), "cannot encode " +
+ Twine(MCII.getName(MI.getOpcode())) + ": " +
+ Msg);
+}
+
+uint64_t AGXMCCodeEmitter::encodeALUSrc(const MCInst &MI,
+ unsigned OpNo) const {
+ unsigned Mods = MI.getOperand(OpNo).getImm();
+ const MCOperand &Val = MI.getOperand(OpNo + 1);
+ if (Val.isImm()) {
+ uint64_t V = uint64_t(Val.getImm());
+ if (V > 0xff)
+ report(MI, "ALU immediate out of range");
+ V &= 0xff;
+ return (V & 63) | ((V >> 6) << 10);
+ }
+
+ AGX::RegInfo Info = regInfo(Val);
+ unsigned Idx = Info.HalfIndex;
+ if (Info.isUniform()) {
+ if (Info.Kind == AGX::RegKind::UGPR64)
+ report(MI, "64-bit uniform registers cannot be ALU sources");
+ bool Is32 = Info.Kind == AGX::RegKind::UGPR32;
+ return (Idx & 63) | (((Idx >> 8) & 1) << 6) | (uint64_t(Is32) << 7) |
+ (1u << 8) | (((Idx >> 6) & 3) << 10);
+ }
+ if (!Info.isGPR()) {
+ report(MI, "invalid ALU source register");
+ return 0;
+ }
+ unsigned Hint = (Mods & AGX::SRC_MOD_DISCARD) ? 3
+ : (Mods & AGX::SRC_MOD_CACHE) ? 2
+ : 1;
+ unsigned Size = Info.getSizeInBits();
+ unsigned SizeFlag = Size == 64 ? 3 : Size == 32 ? 2 : 0;
+ return (Idx & 63) | (Hint << 6) | (SizeFlag << 8) | (((Idx >> 6) & 3) << 10);
+}
+
+static void setOp(APInt &Op, uint64_t V) { Op = APInt(Op.getBitWidth(), V); }
+
+void AGXMCCodeEmitter::getSrcEncoding(const MCInst &MI, unsigned OpNo,
+ APInt &Op, SmallVectorImpl<MCFixup> &,
+ const MCSubtargetInfo &) const {
+ setOp(Op, encodeALUSrc(MI, OpNo));
+}
+
+void AGXMCCodeEmitter::getSrc64Encoding(const MCInst &MI, unsigned OpNo,
+ APInt &Op, SmallVectorImpl<MCFixup> &,
+ const MCSubtargetInfo &) const {
+ setOp(Op, encodeALUSrc(MI, OpNo));
+}
+
+void AGXMCCodeEmitter::getSrc16Encoding(const MCInst &MI, unsigned OpNo,
+ APInt &Op, SmallVectorImpl<MCFixup> &,
+ const MCSubtargetInfo &) const {
+ setOp(Op, encodeALUSrc(MI, OpNo));
+}
+
+void AGXMCCodeEmitter::getSrcSXEncoding(const MCInst &MI, unsigned OpNo,
+ APInt &Op, SmallVectorImpl<MCFixup> &,
+ const MCSubtargetInfo &) const {
+ uint64_t M = encodeALUSrc(MI, OpNo);
+ unsigned SX = (MI.getOperand(OpNo).getImm() & AGX::SRC_MOD_SX) ? 1 : 0;
+ setOp(Op, (M & 0x3ff) | (SX << 10) | ((M >> 10) << 11));
+}
+
+void AGXMCCodeEmitter::getSrcSX64Encoding(const MCInst &MI, unsigned OpNo,
+ APInt &Op,
+ SmallVectorImpl<MCFixup> &Fixups,
+ const MCSubtargetInfo &STI) const {
+ getSrcSXEncoding(MI, OpNo, Op, Fixups, STI);
+}
+
+static unsigned getFPMods(unsigned Mods) {
+ return ((Mods & AGX::SRC_MOD_ABS) ? 1 : 0) |
+ ((Mods & AGX::SRC_MOD_NEG) ? 2 : 0);
+}
+
+void AGXMCCodeEmitter::getSrcFEncoding(const MCInst &MI, unsigned OpNo,
+ APInt &Op, SmallVectorImpl<MCFixup> &,
+ const MCSubtargetInfo &) const {
+ uint64_t M = encodeALUSrc(MI, OpNo);
+ unsigned FM = getFPMods(MI.getOperand(OpNo).getImm());
+ setOp(Op, (M & 0x3ff) | (FM << 10) | ((M >> 10) << 12));
+}
+
+void AGXMCCodeEmitter::getSrcF16Encoding(const MCInst &MI, unsigned OpNo,
+ APInt &Op, SmallVectorImpl<MCFixup> &,
+ const MCSubtargetInfo &) const {
+ uint64_t M = encodeALUSrc(MI, OpNo);
+ if (M & (1u << 9))
+ report(MI, "half precision sources must be 16-bit");
+ unsigned FM = getFPMods(MI.getOperand(OpNo).getImm());
+ setOp(Op, (M & 0x1ff) | (FM << 9) | ((M >> 10) << 11));
+}
+
+void AGXMCCodeEmitter::getCmpselSrcEncoding(const MCInst &MI, unsigned OpNo,
+ APInt &Op,
+ SmallVectorImpl<MCFixup> &,
+ const MCSubtargetInfo &) const {
+ unsigned Mods = MI.getOperand(OpNo).getImm();
+ const MCOperand &Val = MI.getOperand(OpNo + 1);
+ uint64_t V;
+ if (Val.isImm()) {
+ uint64_t I = uint64_t(Val.getImm()) & 0xff;
+ V = (I & 63) | (0b100 << 6) | ((I >> 6) << 9);
+ } else {
+ AGX::RegInfo Info = regInfo(Val);
+ unsigned Idx = Info.HalfIndex;
+ if (Info.isUniform()) {
+ V = (Idx & 63) | (((Idx >> 8) & 1) << 6) | (0b11 << 7) |
+ (((Idx >> 6) & 3) << 9);
+ } else {
+ unsigned Hint = (Mods & AGX::SRC_MOD_DISCARD) ? 3
+ : (Mods & AGX::SRC_MOD_CACHE) ? 2
+ : 1;
+ V = (Idx & 63) | (Hint << 6) | (((Idx >> 6) & 3) << 9);
+ }
+ }
+ setOp(Op, V);
+}
+
+void AGXMCCodeEmitter::getDstEncoding(const MCInst &MI, unsigned OpNo,
+ APInt &Op, SmallVectorImpl<MCFixup> &,
+ const MCSubtargetInfo &) const {
+ AGX::RegInfo Info = regInfo(MI.getOperand(OpNo));
+ if (!Info.isGPR())
+ report(MI, "invalid destination register");
+ bool Cache = MI.getOperand(OpNo + 1).getImm();
+ unsigned Size = Info.getSizeInBits();
+ uint64_t D = uint64_t(Cache) | (uint64_t(Size >= 32) << 1) |
+ (uint64_t(Info.HalfIndex | (Size == 64 ? 1 : 0)) << 2);
+ setOp(Op, D);
+}
+
+void AGXMCCodeEmitter::getMemRegEncoding(const MCInst &MI, unsigned OpNo,
+ APInt &Op, SmallVectorImpl<MCFixup> &,
+ const MCSubtargetInfo &) const {
+ AGX::RegInfo Info = regInfo(MI.getOperand(OpNo));
+ if (!Info.isGPR())
+ report(MI, "invalid memory register");
+ bool Is32 = Info.Kind != AGX::RegKind::GPR16;
+ setOp(Op, Info.HalfIndex | (uint64_t(Is32) << 8));
+}
+
+void AGXMCCodeEmitter::getAtomicDstEncoding(const MCInst &MI, unsigned OpNo,
+ APInt &Op,
+ SmallVectorImpl<MCFixup> &,
+ const MCSubtargetInfo &) const {
+ const MCOperand &MO = MI.getOperand(OpNo);
+ if (!MO.getReg()) {
+ setOp(Op, 0);
+ return;
+ }
+ AGX::RegInfo Info = regInfo(MO);
+ if (Info.Kind != AGX::RegKind::GPR32)
+ report(MI, "atomic results are 32-bit registers");
+ setOp(Op, Info.HalfIndex | (1u << 8));
+}
+
+void AGXMCCodeEmitter::getMemBaseEncoding(const MCInst &MI, unsigned OpNo,
+ APInt &Op, SmallVectorImpl<MCFixup> &,
+ const MCSubtargetInfo &) const {
+ AGX::RegInfo Info = regInfo(MI.getOperand(OpNo));
+ if (Info.Kind != AGX::RegKind::GPR64 && Info.Kind != AGX::RegKind::UGPR64)
+ report(MI, "memory base must be a 64-bit register pair");
+ if (Info.HalfIndex >= 0x100)
+ report(MI, "memory base register out of range");
+ setOp(Op, (Info.HalfIndex & 0xff) | (uint64_t(Info.isUniform()) << 8));
+}
+
+void AGXMCCodeEmitter::getMemIndexEncoding(const MCInst &MI, unsigned OpNo,
+ APInt &Op,
+ SmallVectorImpl<MCFixup> &,
+ const MCSubtargetInfo &) const {
+ const MCOperand &MO = MI.getOperand(OpNo);
+ if (MO.isImm()) {
+ int64_t V = MO.getImm();
+ if (!isInt<16>(V) && !isUInt<16>(V))
+ report(MI, "memory index immediate out of range");
+ setOp(Op, (uint64_t(V) & 0xffff) | (1u << 16));
+ return;
+ }
+ AGX::RegInfo Info = regInfo(MO);
+ if (Info.Kind != AGX::RegKind::GPR32)
+ report(MI, "memory index must be a 32-bit register");
+ setOp(Op, Info.HalfIndex);
+}
+
+void AGXMCCodeEmitter::getLocalBaseEncoding(const MCInst &MI, unsigned OpNo,
+ APInt &Op,
+ SmallVectorImpl<MCFixup> &,
+ const MCSubtargetInfo &) const {
+ const MCOperand &MO = MI.getOperand(OpNo);
+ if (MO.isImm()) {
+ if (MO.getImm() != 0)
+ report(MI, "threadgroup base immediate must be zero");
+ setOp(Op, 2u << 8);
+ return;
+ }
+ AGX::RegInfo Info = regInfo(MO);
+ if (Info.Kind == AGX::RegKind::UGPR16) {
+ unsigned Idx = Info.HalfIndex;
+ setOp(Op, (Idx & 0xff) | ((1u | ((Idx >> 8) << 1)) << 8));
+ return;
+ }
+ if (Info.Kind != AGX::RegKind::GPR16)
+ report(MI, "threadgroup base must be a 16-bit register");
+ setOp(Op, Info.HalfIndex);
+}
+
+void AGXMCCodeEmitter::getLocalIndexEncoding(const MCInst &MI, unsigned OpNo,
+ APInt &Op,
+ SmallVectorImpl<MCFixup> &,
+ const MCSubtargetInfo &) const {
+ const MCOperand &MO = MI.getOperand(OpNo);
+ if (MO.isImm()) {
+ setOp(Op, (uint64_t(MO.getImm()) & 0xffff) | (1u << 16));
+ return;
+ }
+ AGX::RegInfo Info = regInfo(MO);
+ if (Info.Kind != AGX::RegKind::GPR16)
+ report(MI, "threadgroup index must be a 16-bit register");
+ setOp(Op, Info.HalfIndex);
+}
+
+void AGXMCCodeEmitter::getAtomicSrcEncoding(const MCInst &MI, unsigned OpNo,
+ APInt &Op,
+ SmallVectorImpl<MCFixup> &,
+ const MCSubtargetInfo &) const {
+ AGX::RegInfo Info = regInfo(MI.getOperand(OpNo));
+ if (Info.Kind != AGX::RegKind::GPR32 && Info.Kind != AGX::RegKind::GPR64)
+ report(MI, "atomic data must be a 32-bit register or a pair");
+ setOp(Op, Info.HalfIndex);
+}
+
+void AGXMCCodeEmitter::getBrTargetEncoding(const MCInst &MI, unsigned OpNo,
+ APInt &Op,
+ SmallVectorImpl<MCFixup> &Fixups,
+ const MCSubtargetInfo &) const {
+ const MCOperand &MO = MI.getOperand(OpNo);
+ if (MO.isImm()) {
+ setOp(Op, uint32_t(MO.getImm()));
+ return;
+ }
+ // The offset field is 2 bytes into the instruction.
+ Fixups.push_back(MCFixup::create(2, MO.getExpr(),
+ MCFixupKind(AGX::fixup_agx_branch32),
+ /*PCRel=*/true));
+ setOp(Op, 0);
+}
+
+void AGXMCCodeEmitter::getMachineOpValue(const MCInst &MI, const MCOperand &MO,
+ APInt &Op,
+ SmallVectorImpl<MCFixup> &,
+ const MCSubtargetInfo &) const {
+ if (MO.isImm()) {
+ setOp(Op, uint64_t(MO.getImm()));
+ return;
+ }
+ if (MO.isReg()) {
+ setOp(Op, regInfo(MO).HalfIndex);
+ return;
+ }
+ report(MI, "unexpected operand");
+ setOp(Op, 0);
+}
+
+void AGXMCCodeEmitter::encodeInstruction(const MCInst &MI,
+ SmallVectorImpl<char> &CB,
+ SmallVectorImpl<MCFixup> &Fixups,
+ const MCSubtargetInfo &STI) const {
+ APInt Inst(96, 0), Scratch(96, 0);
+ getBinaryCodeForInstr(MI, Fixups, Inst, Scratch, STI);
+
+ const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
+ unsigned Short = AGX::TSFlags::getShortSize(Desc.TSFlags);
+ unsigned Long = AGX::TSFlags::getLongSize(Desc.TSFlags);
+ unsigned Size = Long;
+ if (Short != Long) {
+ if (Inst.lshr(Short * 8).isZero())
+ Size = Short;
+ else
+ Inst.setBit(AGX::TSFlags::getLengthBit(Desc.TSFlags));
+ }
+
+ for (unsigned I = 0; I < Size; ++I)
+ CB.push_back(char(Inst.extractBitsAsZExtValue(8, I * 8)));
+}
+
+#include "AGXGenMCCodeEmitter.inc"
+
+MCCodeEmitter *llvm::createAGXMCCodeEmitter(const MCInstrInfo &MCII,
+ MCContext &Ctx) {
+ return new AGXMCCodeEmitter(MCII, Ctx);
+}
diff --git a/llvm/lib/Target/AGX/MCTargetDesc/AGXMCTargetDesc.cpp b/llvm/lib/Target/AGX/MCTargetDesc/AGXMCTargetDesc.cpp
new file mode 100644
index 0000000000000..feb68b06481e7
--- /dev/null
+++ b/llvm/lib/Target/AGX/MCTargetDesc/AGXMCTargetDesc.cpp
@@ -0,0 +1,114 @@
+//===-- AGXMCTargetDesc.cpp - AGX target descriptions ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AGXMCTargetDesc.h"
+#include "AGXInstPrinter.h"
+#include "AGXMCAsmInfo.h"
+#include "TargetInfo/AGXTargetInfo.h"
+#include "llvm/MC/MCCodeEmitter.h"
+#include "llvm/MC/MCELFStreamer.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCInstrAnalysis.h"
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Support/Compiler.h"
+
+using namespace llvm;
+
+#define GET_INSTRINFO_MC_DESC
+#define GET_INSTRINFO_NAMED_OPS
+#define ENABLE_INSTR_PREDICATE_VERIFIER
+#include "AGXGenInstrInfo.inc"
+
+#define GET_SUBTARGETINFO_MC_DESC
+#include "AGXGenSubtargetInfo.inc"
+
+#define GET_REGINFO_MC_DESC
+#include "AGXGenRegisterInfo.inc"
+
+static MCInstrInfo *createAGXMCInstrInfo() {
+ MCInstrInfo *X = new MCInstrInfo();
+ InitAGXMCInstrInfo(X);
+ return X;
+}
+
+static MCRegisterInfo *createAGXMCRegisterInfo(const Triple &TT) {
+ MCRegisterInfo *X = new MCRegisterInfo();
+ InitAGXMCRegisterInfo(X, AGX::R1);
+ return X;
+}
+
+static MCSubtargetInfo *createAGXMCSubtargetInfo(const Triple &TT,
+ StringRef CPU, StringRef FS) {
+ if (CPU.empty())
+ CPU = "generic";
+ return createAGXMCSubtargetInfoImpl(TT, CPU, /*TuneCPU=*/CPU, FS);
+}
+
+static MCInstPrinter *createAGXMCInstPrinter(const Triple &T,
+ unsigned SyntaxVariant,
+ const MCAsmInfo &MAI,
+ const MCInstrInfo &MII,
+ const MCRegisterInfo &MRI) {
+ if (SyntaxVariant != 0)
+ return nullptr;
+ return new AGXInstPrinter(MAI, MII, MRI);
+}
+
+static MCStreamer *createAGXELFStreamer(const Triple &T, MCContext &Ctx,
+ std::unique_ptr<MCAsmBackend> &&MAB,
+ std::unique_ptr<MCObjectWriter> &&OW,
+ std::unique_ptr<MCCodeEmitter> &&CE) {
+ return createELFStreamer(Ctx, std::move(MAB), std::move(OW), std::move(CE));
+}
+
+namespace {
+
+class AGXMCInstrAnalysis : public MCInstrAnalysis {
+public:
+ explicit AGXMCInstrAnalysis(const MCInstrInfo *Info)
+ : MCInstrAnalysis(Info) {}
+
+ bool evaluateBranch(const MCInst &Inst, uint64_t Addr, uint64_t Size,
+ uint64_t &Target) const override {
+ const MCInstrDesc &Desc = Info->get(Inst.getOpcode());
+ for (unsigned I = 0, E = Inst.getNumOperands(); I != E; ++I) {
+ if (I >= Desc.getNumOperands() ||
+ Desc.operands()[I].OperandType != MCOI::OPERAND_PCREL)
+ continue;
+ if (!Inst.getOperand(I).isImm())
+ return false;
+ // Branch offsets are relative to the start of the branch.
+ Target = Addr + int32_t(Inst.getOperand(I).getImm());
+ return true;
+ }
+ return false;
+ }
+};
+
+} // namespace
+
+static MCInstrAnalysis *createAGXMCInstrAnalysis(const MCInstrInfo *Info) {
+ return new AGXMCInstrAnalysis(Info);
+}
+
+extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAGXTargetMC() {
+ Target &T = getTheAGXTarget();
+ RegisterMCAsmInfo<AGXMCAsmInfo> X(T);
+ TargetRegistry::RegisterMCInstrInfo(T, createAGXMCInstrInfo);
+ TargetRegistry::RegisterMCRegInfo(T, createAGXMCRegisterInfo);
+ TargetRegistry::RegisterMCSubtargetInfo(T, createAGXMCSubtargetInfo);
+ TargetRegistry::RegisterMCInstPrinter(T, createAGXMCInstPrinter);
+ TargetRegistry::RegisterMCCodeEmitter(T, createAGXMCCodeEmitter);
+ TargetRegistry::RegisterMCAsmBackend(T, createAGXAsmBackend);
+ TargetRegistry::RegisterELFStreamer(T, createAGXELFStreamer);
+ TargetRegistry::RegisterMCInstrAnalysis(T, createAGXMCInstrAnalysis);
+}
diff --git a/llvm/lib/Target/AGX/MCTargetDesc/AGXMCTargetDesc.h b/llvm/lib/Target/AGX/MCTargetDesc/AGXMCTargetDesc.h
new file mode 100644
index 0000000000000..3cf4df120be5c
--- /dev/null
+++ b/llvm/lib/Target/AGX/MCTargetDesc/AGXMCTargetDesc.h
@@ -0,0 +1,52 @@
+//===-- AGXMCTargetDesc.h - AGX target descriptions -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Provides AGX specific target descriptions.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AGX_MCTARGETDESC_AGXMCTARGETDESC_H
+#define LLVM_LIB_TARGET_AGX_MCTARGETDESC_AGXMCTARGETDESC_H
+
+#include "Utils/AGXBaseInfo.h"
+#include <memory>
+
+namespace llvm {
+class MCAsmBackend;
+class MCCodeEmitter;
+class MCContext;
+class MCInstrInfo;
+class MCObjectTargetWriter;
+class MCRegisterInfo;
+class MCSubtargetInfo;
+class MCTargetOptions;
+class Target;
+
+MCCodeEmitter *createAGXMCCodeEmitter(const MCInstrInfo &MCII,
+ MCContext &Ctx);
+
+MCAsmBackend *createAGXAsmBackend(const Target &T, const MCSubtargetInfo &STI,
+ const MCRegisterInfo &MRI,
+ const MCTargetOptions &Options);
+
+std::unique_ptr<MCObjectTargetWriter> createAGXELFObjectWriter(uint8_t OSABI);
+
+} // namespace llvm
+
+#define GET_REGINFO_ENUM
+#include "AGXGenRegisterInfo.inc"
+
+#define GET_INSTRINFO_ENUM
+#define GET_INSTRINFO_OPERAND_ENUM
+#define GET_INSTRINFO_MC_HELPER_DECLS
+#include "AGXGenInstrInfo.inc"
+
+#define GET_SUBTARGETINFO_ENUM
+#include "AGXGenSubtargetInfo.inc"
+
+#endif // LLVM_LIB_TARGET_AGX_MCTARGETDESC_AGXMCTARGETDESC_H
diff --git a/llvm/lib/Target/AGX/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/AGX/MCTargetDesc/CMakeLists.txt
new file mode 100644
index 0000000000000..2247ac871b814
--- /dev/null
+++ b/llvm/lib/Target/AGX/MCTargetDesc/CMakeLists.txt
@@ -0,0 +1,21 @@
+add_llvm_component_library(LLVMAGXDesc
+ AGXAsmBackend.cpp
+ AGXELFObjectWriter.cpp
+ AGXInstPrinter.cpp
+ AGXMCAsmInfo.cpp
+ AGXMCCodeEmitter.cpp
+ AGXMCTargetDesc.cpp
+
+ LINK_COMPONENTS
+ AGXInfo
+ AGXUtils
+ CodeGenTypes
+ MC
+ Support
+ TargetParser
+
+ ADD_TO_COMPONENT
+ AGX
+ )
+
+add_dependencies(LLVMAGXDesc AGXCommonTableGen)
diff --git a/llvm/lib/Target/AGX/TargetInfo/AGXTargetInfo.cpp b/llvm/lib/Target/AGX/TargetInfo/AGXTargetInfo.cpp
new file mode 100644
index 0000000000000..ff42dea0baf0d
--- /dev/null
+++ b/llvm/lib/Target/AGX/TargetInfo/AGXTargetInfo.cpp
@@ -0,0 +1,23 @@
+//===-- AGXTargetInfo.cpp - AGX target implementation ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TargetInfo/AGXTargetInfo.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Support/Compiler.h"
+
+using namespace llvm;
+
+Target &llvm::getTheAGXTarget() {
+ static Target TheAGXTarget;
+ return TheAGXTarget;
+}
+
+extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAGXTargetInfo() {
+ RegisterTarget<Triple::agx, /*HasJIT=*/false> X(
+ getTheAGXTarget(), "agx", "Apple AGX GPUs", "AGX");
+}
diff --git a/llvm/lib/Target/AGX/TargetInfo/AGXTargetInfo.h b/llvm/lib/Target/AGX/TargetInfo/AGXTargetInfo.h
new file mode 100644
index 0000000000000..d4d652a9b332b
--- /dev/null
+++ b/llvm/lib/Target/AGX/TargetInfo/AGXTargetInfo.h
@@ -0,0 +1,18 @@
+//===-- AGXTargetInfo.h - AGX target implementation -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AGX_TARGETINFO_AGXTARGETINFO_H
+#define LLVM_LIB_TARGET_AGX_TARGETINFO_AGXTARGETINFO_H
+
+namespace llvm {
+class Target;
+
+Target &getTheAGXTarget();
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AGX_TARGETINFO_AGXTARGETINFO_H
diff --git a/llvm/lib/Target/AGX/TargetInfo/CMakeLists.txt b/llvm/lib/Target/AGX/TargetInfo/CMakeLists.txt
new file mode 100644
index 0000000000000..397f0a001c813
--- /dev/null
+++ b/llvm/lib/Target/AGX/TargetInfo/CMakeLists.txt
@@ -0,0 +1,10 @@
+add_llvm_component_library(LLVMAGXInfo
+ AGXTargetInfo.cpp
+
+ LINK_COMPONENTS
+ MC
+ Support
+
+ ADD_TO_COMPONENT
+ AGX
+ )
diff --git a/llvm/lib/Target/AGX/Utils/AGXBaseInfo.cpp b/llvm/lib/Target/AGX/Utils/AGXBaseInfo.cpp
new file mode 100644
index 0000000000000..1a55c2a373cf8
--- /dev/null
+++ b/llvm/lib/Target/AGX/Utils/AGXBaseInfo.cpp
@@ -0,0 +1,340 @@
+//===-- AGXBaseInfo.cpp - AGX shared definitions --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AGXBaseInfo.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/MC/MCRegister.h"
+#include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/raw_ostream.h"
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+
+#define GET_REGINFO_ENUM
+#include "AGXGenRegisterInfo.inc"
+
+using namespace llvm;
+
+double AGX::decodeFPImm8(unsigned Imm) {
+ double Sign = (Imm & 0x80) ? -1.0 : 1.0;
+ unsigned E = (Imm >> 4) & 7;
+ unsigned F = Imm & 0xf;
+ if (E == 0)
+ return Sign * F / 64.0;
+ return Sign * double(0x10 | F) * std::ldexp(1.0, int(E) - 7);
+}
+
+std::optional<unsigned> AGX::encodeFPImm8(double V) {
+ for (unsigned I = 0; I < 256; ++I) {
+ double D = decodeFPImm8(I);
+ if (D == V && std::signbit(D) == std::signbit(V))
+ return I;
+ }
+ return std::nullopt;
+}
+
+std::string AGX::formatFPImm8(unsigned Imm) {
+ // Every 8-bit float is a multiple of 1/64 below 32, so six decimals are
+ // exact. Print like Python's repr: no trailing zeros, at least one.
+ char Buf[32];
+ std::snprintf(Buf, sizeof(Buf), "%.6f", decodeFPImm8(Imm));
+ std::string S(Buf);
+ while (S.size() > 2 && S.back() == '0' && S[S.size() - 2] != '.')
+ S.pop_back();
+ return S;
+}
+
+StringRef AGX::getICondName(unsigned CC) {
+ switch (CC) {
+ case 0b0000: return "ueq";
+ case 0b0001: return "ult";
+ case 0b0010: return "ugt";
+ case 0b0100: return "seq";
+ case 0b0101: return "slt";
+ case 0b0110: return "sgt";
+ case 0b1000: return "nueq";
+ case 0b1001: return "ugte";
+ case 0b1010: return "ulte";
+ case 0b1100: return "nseq";
+ case 0b1101: return "sgte";
+ case 0b1110: return "slte";
+ default: return "";
+ }
+}
+
+StringRef AGX::getFCondName(unsigned CC) {
+ switch (CC) {
+ case 0b0000: return "eq";
+ case 0b0001: return "lt";
+ case 0b0010: return "gt";
+ case 0b0011: return "ltn";
+ case 0b0101: return "gte";
+ case 0b0110: return "lte";
+ case 0b0111: return "gtn";
+ case 0b1000: return "neq";
+ case 0b1001: return "nlt";
+ case 0b1010: return "ngt";
+ case 0b1011: return "nltn";
+ case 0b1101: return "ngte";
+ case 0b1110: return "nlte";
+ case 0b1111: return "ngtn";
+ default: return "";
+ }
+}
+
+std::optional<unsigned> AGX::parseICond(StringRef Name) {
+ for (unsigned CC = 0; CC < 16; ++CC)
+ if (!getICondName(CC).empty() && getICondName(CC) == Name)
+ return CC;
+ return std::nullopt;
+}
+
+std::optional<unsigned> AGX::parseFCond(StringRef Name) {
+ for (unsigned CC = 0; CC < 16; ++CC)
+ if (!getFCondName(CC).empty() && getFCondName(CC) == Name)
+ return CC;
+ return std::nullopt;
+}
+
+StringRef AGX::getMemFormatName(unsigned Fmt) {
+ switch (Fmt) {
+ case FMT_I8: return "i8";
+ case FMT_I16: return "i16";
+ case FMT_I32: return "i32";
+ case FMT_F16: return "f16";
+ case FMT_U8NORM: return "u8norm";
+ case FMT_S8NORM: return "s8norm";
+ case FMT_U16NORM: return "u16norm";
+ case FMT_S16NORM: return "s16norm";
+ case FMT_RGB10A2: return "rgb10a2";
+ case FMT_SRGBA8: return "srgba8";
+ case FMT_RG11B10F: return "rg11b10f";
+ case FMT_RGB9E5: return "rgb9e5";
+ case 14: return "rg11b10f.rtz";
+ case 15: return "rgb9e5.rtz";
+ default: return "";
+ }
+}
+
+std::optional<unsigned> AGX::parseMemFormat(StringRef Name) {
+ for (unsigned F = 0; F < 16; ++F)
+ if (!getMemFormatName(F).empty() && getMemFormatName(F) == Name)
+ return F;
+ return std::nullopt;
+}
+
+StringRef AGX::getAtomicOpName(unsigned Op) {
+ static const char *const Names[] = {"add", "sub", "xchg", "cmpxchg",
+ "umin", "imin", "umax", "imax",
+ "and", "or", "xor"};
+ return Op < std::size(Names) ? Names[Op] : "";
+}
+
+std::optional<unsigned> AGX::parseAtomicOp(StringRef Name) {
+ for (unsigned Op = 0; Op < 16; ++Op)
+ if (!getAtomicOpName(Op).empty() && getAtomicOpName(Op) == Name)
+ return Op;
+ return std::nullopt;
+}
+
+StringRef AGX::getConvertModeName(unsigned Mode) {
+ switch (Mode) {
+ case 0: return "u8_to_f";
+ case 1: return "s8_to_f";
+ case 4: return "f_to_u16";
+ case 5: return "f_to_s16";
+ case 6: return "u16_to_f";
+ case 7: return "s16_to_f";
+ case 8: return "f_to_u32";
+ case 9: return "f_to_s32";
+ case 10: return "u32_to_f";
+ case 11: return "s32_to_f";
+ default: return "";
+ }
+}
+
+std::optional<unsigned> AGX::parseConvertMode(StringRef Name) {
+ for (unsigned M = 0; M < 64; ++M)
+ if (!getConvertModeName(M).empty() && getConvertModeName(M) == Name)
+ return M;
+ return std::nullopt;
+}
+
+StringRef AGX::getRoundModeName(unsigned Mode) {
+ switch (Mode) {
+ case 0: return "rtz";
+ case 1: return "rte";
+ default: return "";
+ }
+}
+
+std::optional<unsigned> AGX::parseRoundMode(StringRef Name) {
+ return StringSwitch<std::optional<unsigned>>(Name)
+ .Case("rtz", 0)
+ .Case("rte", 1)
+ .Default(std::nullopt);
+}
+
+StringRef AGX::getSpecialRegName(unsigned SR) {
+ switch (SR) {
+ case 0: return "threadgroup_position_in_grid.x";
+ case 1: return "threadgroup_position_in_grid.y";
+ case 2: return "threadgroup_position_in_grid.z";
+ case 4: return "threads_per_threadgroup.x";
+ case 5: return "threads_per_threadgroup.y";
+ case 6: return "threads_per_threadgroup.z";
+ case 8: return "dispatch_threads_per_threadgroup.x";
+ case 9: return "dispatch_threads_per_threadgroup.y";
+ case 10: return "dispatch_threads_per_threadgroup.z";
+ case 14: return "samples_log2";
+ case 20: return "core_index";
+ case 21: return "vm_slot";
+ case 48: return "thread_position_in_threadgroup.x";
+ case 49: return "thread_position_in_threadgroup.y";
+ case 50: return "thread_position_in_threadgroup.z";
+ case 51: return "thread_index_in_threadgroup";
+ case 52: return "thread_index_in_simdgroup";
+ case 53: return "simdgroup_index_in_threadgroup";
+ case 56: return "active_thread_index_in_quadgroup";
+ case 57: return "total_active_threads_in_quadgroup";
+ case 58: return "active_thread_index_in_simdgroup";
+ case 59: return "total_active_threads_in_simdgroup";
+ case 60: return "internal_coverage_mask";
+ case 62: return "backfacing";
+ case 63: return "is_active_thread";
+ case 80: return "thread_position_in_grid.x";
+ case 81: return "thread_position_in_grid.y";
+ case 82: return "thread_position_in_grid.z";
+ case 124: return "input_sample_mask";
+ case 144: return "opfifo_cmd";
+ case 146: return "opfifo_data_l";
+ case 147: return "opfifo_data_h";
+ default: return "";
+ }
+}
+
+std::string AGX::getMemMaskName(unsigned Mask) {
+ std::string S;
+ if (Mask & 1)
+ S += 'x';
+ if (Mask & 2)
+ S += 'y';
+ if (Mask & 4)
+ S += 'z';
+ if (Mask & 8)
+ S += 'w';
+ return S;
+}
+
+std::optional<unsigned> AGX::parseMemMask(StringRef Name) {
+ for (unsigned M = 1; M < 16; ++M)
+ if (getMemMaskName(M) == Name)
+ return M;
+ return std::nullopt;
+}
+
+unsigned AGX::RegInfo::getSizeInBits() const {
+ switch (Kind) {
+ case RegKind::GPR16:
+ case RegKind::UGPR16:
+ case RegKind::Nest:
+ return 16;
+ case RegKind::GPR32:
+ case RegKind::UGPR32:
+ return 32;
+ case RegKind::GPR64:
+ case RegKind::UGPR64:
+ return 64;
+ case RegKind::GPR96:
+ return 96;
+ case RegKind::GPR128:
+ return 128;
+ case RegKind::Invalid:
+ break;
+ }
+ return 0;
+}
+
+namespace {
+struct KindClass {
+ AGX::RegKind Kind;
+ unsigned ClassID;
+};
+} // namespace
+
+static const KindClass KindClasses[] = {
+ {AGX::RegKind::GPR16, AGX::GPR16RegClassID},
+ {AGX::RegKind::GPR32, AGX::GPR32RegClassID},
+ {AGX::RegKind::GPR64, AGX::GPR32x2RegClassID},
+ {AGX::RegKind::GPR96, AGX::GPR96RegClassID},
+ {AGX::RegKind::GPR128, AGX::GPR128RegClassID},
+ {AGX::RegKind::UGPR16, AGX::UGPR16RegClassID},
+ {AGX::RegKind::UGPR32, AGX::UGPR32RegClassID},
+ {AGX::RegKind::UGPR64, AGX::UGPR64RegClassID},
+};
+
+AGX::RegInfo AGX::getRegInfo(MCRegister Reg, const MCRegisterInfo &MRI) {
+ RegInfo Info;
+ if (!Reg)
+ return Info;
+ for (const KindClass &KC : KindClasses) {
+ const MCRegisterClass &RC = MRI.getRegClass(KC.ClassID);
+ if (!RC.contains(Reg))
+ continue;
+ Info.Kind = KC.Kind;
+ MCRegister First = Reg;
+ if (KC.Kind == RegKind::GPR64 || KC.Kind == RegKind::GPR96 ||
+ KC.Kind == RegKind::GPR128 || KC.Kind == RegKind::UGPR64)
+ First = MRI.getSubReg(Reg, AGX::sub0);
+ Info.HalfIndex = MRI.getEncodingValue(First);
+ return Info;
+ }
+ return Info;
+}
+
+MCRegister AGX::getRegister(RegKind Kind, unsigned HalfIndex,
+ const MCRegisterInfo &MRI) {
+ unsigned ClassID = 0;
+ unsigned Index = HalfIndex;
+ switch (Kind) {
+ case RegKind::GPR16:
+ ClassID = AGX::GPR16RegClassID;
+ break;
+ case RegKind::UGPR16:
+ ClassID = AGX::UGPR16RegClassID;
+ break;
+ case RegKind::GPR32:
+ case RegKind::GPR64:
+ case RegKind::GPR96:
+ case RegKind::GPR128:
+ case RegKind::UGPR32:
+ case RegKind::UGPR64:
+ if (HalfIndex & 1)
+ return MCRegister();
+ Index = HalfIndex / 2;
+ ClassID = Kind == RegKind::GPR32 ? AGX::GPR32RegClassID
+ : Kind == RegKind::GPR64 ? AGX::GPR32x2RegClassID
+ : Kind == RegKind::GPR96 ? AGX::GPR96RegClassID
+ : Kind == RegKind::GPR128 ? AGX::GPR128RegClassID
+ : Kind == RegKind::UGPR32 ? AGX::UGPR32RegClassID
+ : AGX::UGPR64RegClassID;
+ break;
+ case RegKind::Nest:
+ case RegKind::Invalid:
+ return MCRegister();
+ }
+ const MCRegisterClass &RC = MRI.getRegClass(ClassID);
+ if (Index >= RC.getNumRegs())
+ return MCRegister();
+ MCRegister Reg = RC.getRegister(Index);
+ RegInfo Info = getRegInfo(Reg, MRI);
+ if (Info.Kind != Kind || Info.HalfIndex != HalfIndex)
+ return MCRegister();
+ return Reg;
+}
diff --git a/llvm/lib/Target/AGX/Utils/AGXBaseInfo.h b/llvm/lib/Target/AGX/Utils/AGXBaseInfo.h
new file mode 100644
index 0000000000000..8b199c94066da
--- /dev/null
+++ b/llvm/lib/Target/AGX/Utils/AGXBaseInfo.h
@@ -0,0 +1,185 @@
+//===-- AGXBaseInfo.h - Top level definitions for AGX -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Enumerations and helpers shared by the AGX MC layer, assembler,
+// disassembler and code generator.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AGX_UTILS_AGXBASEINFO_H
+#define LLVM_LIB_TARGET_AGX_UTILS_AGXBASEINFO_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/MC/MCInstrDesc.h"
+#include <cstdint>
+#include <optional>
+
+namespace llvm {
+
+class MCRegisterInfo;
+class MCRegister;
+
+namespace AGX {
+
+/// Operand types of AGX instructions.
+enum OperandType : unsigned {
+ /// ALU source: register, uniform register or 8-bit immediate.
+ OPERAND_SRC = MCOI::OPERAND_FIRST_TARGET,
+ /// ALU destination with cache hint.
+ OPERAND_DST,
+};
+
+/// Flags of the modifier operand that accompanies every ALU source.
+enum SrcMods : unsigned {
+ SRC_MOD_ABS = 1 << 0,
+ SRC_MOD_NEG = 1 << 1,
+ SRC_MOD_SX = 1 << 2,
+ /// Keep the value in the register cache.
+ SRC_MOD_CACHE = 1 << 3,
+ /// This is the last use of the register.
+ SRC_MOD_DISCARD = 1 << 4,
+};
+
+/// Fields of the TSFlags of AGX instructions, see AGXInstrFormats.td.
+namespace TSFlags {
+inline unsigned getShortSize(uint64_t F) { return F & 0xf; }
+inline unsigned getLongSize(uint64_t F) { return (F >> 4) & 0xf; }
+inline unsigned getLengthBit(uint64_t F) { return (F >> 8) & 0x7f; }
+inline bool isAsync(uint64_t F) { return F & (1ull << 15); }
+inline bool isExecMaskOp(uint64_t F) { return F & (1ull << 16); }
+inline bool isALU(uint64_t F) { return F & (1ull << 17); }
+inline bool isMemory(uint64_t F) { return F & (1ull << 18); }
+} // namespace TSFlags
+
+/// Memory formats of load/store instructions.
+enum MemFormat : unsigned {
+ FMT_I8 = 0,
+ FMT_I16 = 1,
+ FMT_I32 = 2,
+ FMT_F16 = 3,
+ FMT_U8NORM = 4,
+ FMT_S8NORM = 5,
+ FMT_U16NORM = 6,
+ FMT_S16NORM = 7,
+ FMT_RGB10A2 = 8,
+ FMT_SRGBA8 = 10,
+ FMT_RG11B10F = 12,
+ FMT_RGB9E5 = 13,
+};
+
+/// Operation numbers of SIMD-group reductions (llvm.agx.simd.reduce etc.).
+enum SIMDOp : unsigned {
+ SIMD_AND = 0,
+ SIMD_FADD = 1,
+ SIMD_OR = 2,
+ SIMD_FMUL = 3,
+ SIMD_XOR = 4,
+ SIMD_FMIN = 5,
+ SIMD_FMAX = 7,
+ SIMD_IADD = 16,
+ SIMD_SMIN = 20,
+ SIMD_SMAX = 22,
+ SIMD_UMIN = 28,
+ SIMD_UMAX = 30,
+};
+
+/// Address spaces.
+enum AddressSpace : unsigned {
+ GENERIC_ADDRESS = 0,
+ DEVICE_ADDRESS = 1,
+ THREADGROUP_ADDRESS = 3,
+ CONSTANT_ADDRESS = 4,
+ PRIVATE_ADDRESS = 5,
+};
+
+/// Special register numbers read by get_sr.
+enum SpecialReg : unsigned {
+ SR_THREADGROUP_POSITION_IN_GRID_X = 0,
+ SR_THREADS_PER_THREADGROUP_X = 4,
+ SR_CORE_INDEX = 20,
+ SR_THREAD_POSITION_IN_THREADGROUP_X = 48,
+ SR_THREAD_INDEX_IN_THREADGROUP = 51,
+ SR_THREAD_INDEX_IN_SIMDGROUP = 52,
+ SR_SIMDGROUP_INDEX_IN_THREADGROUP = 53,
+ SR_THREAD_POSITION_IN_GRID_X = 80,
+};
+
+/// Decode an 8-bit floating point ALU immediate: sign, 3-bit exponent and
+/// 4-bit fraction. Exponent zero gives fraction / 64.
+double decodeFPImm8(unsigned Imm);
+
+/// Encode \p V as an 8-bit floating point ALU immediate if it is exactly
+/// representable.
+std::optional<unsigned> encodeFPImm8(double V);
+
+/// Format an 8-bit floating point immediate the way the reference
+/// disassembler does (shortest round-trip decimal, always with a '.').
+std::string formatFPImm8(unsigned Imm);
+
+/// Names of enumerated operands. Each returns an empty string for values
+/// without a name.
+StringRef getICondName(unsigned CC);
+StringRef getFCondName(unsigned CC);
+StringRef getMemFormatName(unsigned Fmt);
+StringRef getAtomicOpName(unsigned Op);
+StringRef getConvertModeName(unsigned Mode);
+StringRef getRoundModeName(unsigned Mode);
+StringRef getSpecialRegName(unsigned SR);
+std::string getMemMaskName(unsigned Mask);
+
+/// Reverse lookups, used by the assembler.
+std::optional<unsigned> parseICond(StringRef Name);
+std::optional<unsigned> parseFCond(StringRef Name);
+std::optional<unsigned> parseMemFormat(StringRef Name);
+std::optional<unsigned> parseAtomicOp(StringRef Name);
+std::optional<unsigned> parseConvertMode(StringRef Name);
+std::optional<unsigned> parseRoundMode(StringRef Name);
+std::optional<unsigned> parseMemMask(StringRef Name);
+
+/// Register kinds of the AGX register file.
+enum class RegKind {
+ Invalid,
+ GPR16,
+ GPR32,
+ GPR64, ///< Register pair (any 32-bit tuple of two).
+ GPR96,
+ GPR128,
+ UGPR16,
+ UGPR32,
+ UGPR64,
+ Nest, ///< The nesting counter r0l used implicitly.
+};
+
+/// Kind and first half-register index of \p Reg.
+struct RegInfo {
+ RegKind Kind = RegKind::Invalid;
+ unsigned HalfIndex = 0;
+
+ bool isUniform() const {
+ return Kind == RegKind::UGPR16 || Kind == RegKind::UGPR32 ||
+ Kind == RegKind::UGPR64;
+ }
+ bool isGPR() const {
+ return Kind == RegKind::GPR16 || Kind == RegKind::GPR32 ||
+ Kind == RegKind::GPR64 || Kind == RegKind::GPR96 ||
+ Kind == RegKind::GPR128;
+ }
+ unsigned getSizeInBits() const;
+};
+
+RegInfo getRegInfo(MCRegister Reg, const MCRegisterInfo &MRI);
+
+/// The register of kind \p Kind starting at half register \p HalfIndex, or
+/// no register if there is none.
+MCRegister getRegister(RegKind Kind, unsigned HalfIndex,
+ const MCRegisterInfo &MRI);
+
+} // namespace AGX
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AGX_UTILS_AGXBASEINFO_H
diff --git a/llvm/lib/Target/AGX/Utils/CMakeLists.txt b/llvm/lib/Target/AGX/Utils/CMakeLists.txt
new file mode 100644
index 0000000000000..c9be6e3e172a7
--- /dev/null
+++ b/llvm/lib/Target/AGX/Utils/CMakeLists.txt
@@ -0,0 +1,12 @@
+add_llvm_component_library(LLVMAGXUtils
+ AGXBaseInfo.cpp
+
+ LINK_COMPONENTS
+ MC
+ Support
+
+ ADD_TO_COMPONENT
+ AGX
+ )
+
+add_dependencies(LLVMAGXUtils AGXCommonTableGen)
diff --git a/llvm/test/MC/AGX/g13-alu.s b/llvm/test/MC/AGX/g13-alu.s
new file mode 100644
index 0000000000000..4c8a451e51ddc
--- /dev/null
+++ b/llvm/test/MC/AGX/g13-alu.s
@@ -0,0 +1,178 @@
+# RUN: llvm-mc -triple=agx -mcpu=agx-g13g -show-encoding %s | FileCheck %s
+# RUN: llvm-mc -triple=agx -mcpu=agx-g14c -show-encoding %s | FileCheck %s
+
+# Encodings cross-checked against the applegpu reference assembler and
+# disassembler (dougallj/applegpu): each case is a random canonical encoding
+# that applegpu disassembles to the same text.
+
+asr r11h, r91l.cache, r108l
+# CHECK: asr r11h, r91l.cache, r108l{{ +}}// encoding: [0x2e,0xae,0xb6,0x84,0x05,0x00,0x00,0x0b]
+asr r108h.cache, u162l, r83h.discard
+# CHECK: asr r108h.cache, u162l, r83h.discard{{ +}}// encoding: [0xae,0xb2,0x44,0x75,0x0e,0x00,0x00,0x36]
+asr r90h, u212, u21
+# CHECK: asr r90h, u212, u21{{ +}}// encoding: [0x2e,0xea,0xe8,0xa5,0x1a,0x00,0x00,0x28]
+asr r58l.cache, 106, u84h
+# CHECK: asr r58l.cache, 106, u84h{{ +}}// encoding: [0xae,0xe8,0x2a,0x94,0x12,0x00,0x00,0x16]
+
+asrh r121h, r26, u7h
+# CHECK: asrh r121h, r26, u7h{{ +}}// encoding: [0x2e,0xe6,0x74,0xfe,0x10,0x00,0x00,0x30]
+asrh r101.cache, r117l.cache, r5l.discard
+# CHECK: asrh r101.cache, r117l.cache, r5l.discard{{ +}}// encoding: [0xae,0x95,0xaa,0xac,0x0c,0x00,0x00,0x3c]
+asrh r68.cache, r97h, u35h
+# CHECK: asrh r68.cache, r97h, u35h{{ +}}// encoding: [0xae,0x91,0x43,0x7c,0x10,0x00,0x00,0x2d]
+asrh r89h, u35, r38l.discard
+# CHECK: asrh r89h, u35, r38l.discard{{ +}}// encoding: [0x2e,0xe6,0x86,0xcd,0x0c,0x00,0x00,0x25]
+
+bfeil r61l, r69l, r15h.discard, u37l, mask 0x7FFFFFFF
+# CHECK: bfeil r61l, r69l, r15h.discard, u37l, mask 0x7FFFFFFF{{ +}}// encoding: [0x2e,0xf4,0x4a,0xf0,0xcd,0x0a,0x4d,0x98]
+bfeil r125h, r13l.discard, r81h.discard, 120, mask 0x3FFFF
+# CHECK: bfeil r125h, r13l.discard, r81h.discard, 120, mask 0x3FFFF{{ +}}// encoding: [0x2e,0xf6,0xda,0x30,0x8e,0x38,0x40,0xb2]
+bfeil r67h.cache, r21l, r53h.discard, r23.cache, mask 0x1FFFFFF
+# CHECK: bfeil r67h.cache, r21l, r53h.discard, r23.cache, mask 0x1FFFFFF{{ +}}// encoding: [0xae,0x8e,0x6a,0xb0,0x4e,0xae,0x0a,0xa1]
+bfeil r70, r117l.cache, u192l, r88l.cache, mask 0x1FFFF
+# CHECK: bfeil r70, r117l.cache, u192l, r88l.cache, mask 0x1FFFF{{ +}}// encoding: [0x2e,0x99,0xaa,0x00,0x54,0xb0,0x80,0xae]
+
+bfi r58h, r77h, u151l, u137h, mask 0x3FFFFFF
+# CHECK: bfi r58h, r77h, u151l, u137h, mask 0x3FFFFFF{{ +}}// encoding: [0x2e,0x6a,0x5b,0xe0,0x96,0x53,0x09,0x98]
+bfi r106h.cache, 20, r75.cache, u217h, mask 0x3FFFFFF
+# CHECK: bfi r106h.cache, 20, r75.cache, u217h, mask 0x3FFFFFF{{ +}}// encoding: [0xae,0x2a,0x14,0x60,0xa9,0x73,0x89,0xb2]
+bfi r58h.cache, u159, r21h, r112l.discard, mask 0x7FFF
+# CHECK: bfi r58h.cache, u159, r21h, r112l.discard, mask 0x7FFF{{ +}}// encoding: [0xae,0x6a,0xfe,0xb1,0xc6,0xe0,0xcc,0x10]
+bfi r21h, u214l, r100, r37.discard, mask 0xFFFFF
+# CHECK: bfi r21h, u214l, r100, r37.discard, mask 0xFFFFF{{ +}}// encoding: [0x2e,0x56,0x6c,0x81,0x24,0xca,0x46,0x8b]
+
+bitop 0010, r16l.cache, r92, r4l.discard
+# CHECK: bitop 0010, r16l.cache, r92, r4l.discard{{ +}}// encoding: [0xfe,0x40,0x78,0x82,0x4c,0x08]
+bitop 1101, r35h.cache, r63l.cache, r85l
+# CHECK: bitop 1101, r35h.cache, r63l.cache, r85l{{ +}}// encoding: [0xfe,0x0e,0xbe,0xac,0x86,0x16]
+bitop 1011, r89l, r1h, r22
+# CHECK: bitop 1011, r89l, r1h, r22{{ +}}// encoding: [0x7e,0x64,0x43,0xc4,0xe6,0x20]
+bitop 0010, r61l.cache, r48.discard, 71
+# CHECK: bitop 0010, r61l.cache, r48.discard, 71{{ +}}// encoding: [0xfe,0x74,0xe0,0x72,0x40,0x15]
+
+bitrev r52, r32.discard
+# CHECK: bitrev r52, r32.discard{{ +}}// encoding: [0x3e,0x51,0xc0,0x06,0x00,0x14]
+bitrev r0, u84h
+# CHECK: bitrev r0, u84h{{ +}}// encoding: [0x3e,0x01,0x29,0x05,0x00,0x08]
+bitrev r68l.cache, u33h
+# CHECK: bitrev r68l.cache, u33h{{ +}}// encoding: [0xbe,0x10,0x03,0x05,0x00,0x24]
+bitrev r62.cache, 182
+# CHECK: bitrev r62.cache, 182{{ +}}// encoding: [0xbe,0x79,0x36,0x04,0x00,0x18]
+
+convert 39, r117l.cache, u176, 3
+# CHECK: convert 39, r117l.cache, u176, 3{{ +}}// encoding: [0xbe,0xd4,0x27,0x0c,0x1e,0x31]
+convert s8_to_f, r13l, u121l, rtz
+# CHECK: convert s8_to_f, r13l, u121l, rtz{{ +}}// encoding: [0x3e,0xb4,0x01,0x20,0x13,0x03]
+convert 37, r10h.cache, r34l.discard, 3
+# CHECK: convert 37, r10h.cache, r34l.discard, 3{{ +}}// encoding: [0xbe,0xaa,0x25,0x4c,0x0c,0x01]
+convert 58, r11h, u241l, rtz
+# CHECK: convert 58, r11h, u241l, rtz{{ +}}// encoding: [0x3e,0xae,0x3a,0x20,0x16,0x03]
+
+extr r85h.cache, u149h, r122, r84h.cache, mask 0x7
+# CHECK: extr r85h.cache, u149h, r122, r84h.cache, mask 0x7{{ +}}// encoding: [0xae,0x56,0x6b,0x45,0xe7,0xa9,0x80,0x23]
+extr r3l, 215, 26, r37, mask 0xFFF
+# CHECK: extr r3l, 215, 26, r37, mask 0xFFF{{ +}}// encoding: [0x2e,0x0c,0x17,0xa4,0x01,0x4a,0x4e,0x0c]
+extr r105, r4l.discard, r50h.discard, u78l, mask 0x7FF
+# CHECK: extr r105, r4l.discard, r50h.discard, u78l, mask 0x7FF{{ +}}// encoding: [0x2e,0x25,0xc8,0x54,0xce,0x1c,0x89,0x31]
+extr r67l.cache, 161, u241, u205l, mask 0xFFFFFFF
+# CHECK: extr r67l.cache, 161, u241, u205l, mask 0xFFFFFFF{{ +}}// encoding: [0xae,0x0c,0x21,0x24,0x1e,0x5a,0x8d,0xab]
+
+fcmpsel lt, r107l, u87h.neg, r18l.cache, r119l.discard, r57h
+# CHECK: fcmpsel lt, r107l, u87h.neg, r18l.cache, r119l.discard, r57h{{ +}}// encoding: [0x02,0xac,0x2f,0x49,0x0a,0xee,0x30,0x27,0xd0,0x38]
+fcmpsel lt, r38h.cache, u201h.abs.neg, 2.375, r55l.discard, r65l.cache
+# CHECK: fcmpsel lt, r38h.cache, u201h.abs.neg, 2.375, r55l.discard, r65l.cache{{ +}}// encoding: [0x82,0x9a,0x53,0x3d,0x00,0xee,0x20,0x28,0x60,0x19]
+fcmpsel ltn, r97h, u154.neg, r28l.abs.neg, u110h, r64l.cache
+# CHECK: fcmpsel ltn, r97h, u154.neg, r28l.abs.neg, u110h, r64l.cache{{ +}}// encoding: [0x02,0x86,0xf4,0x89,0xc7,0x9d,0x01,0x68,0xe0,0x30]
+fcmpsel gte, r8l, r94h.cache.abs, u177l, r56h, r59h.discard
+# CHECK: fcmpsel gte, r8l, r94h.cache.abs, u177l, r56h, r59h.discard{{ +}}// encoding: [0x02,0xa0,0xbd,0x24,0x16,0x71,0x70,0xaf,0x50,0x09]
+
+ffs r16l, r48h
+# CHECK: ffs r16l, r48h{{ +}}// encoding: [0x3e,0x40,0x61,0x0c,0x00,0x04]
+ffs r105, r96h
+# CHECK: ffs r105, r96h{{ +}}// encoding: [0x3e,0x25,0x41,0x0c,0x00,0x3c]
+ffs r20.cache, r113.discard
+# CHECK: ffs r20.cache, r113.discard{{ +}}// encoding: [0xbe,0x51,0xe2,0x0e,0x00,0x0c]
+ffs r61h.cache, r69.cache
+# CHECK: ffs r61h.cache, r69.cache{{ +}}// encoding: [0xbe,0x76,0x8a,0x0e,0x00,0x18]
+
+get_sr r1h, sr20 (core_index)
+# CHECK: get_sr r1h, sr20 (core_index){{ +}}// encoding: [0x72,0x06,0x14,0x00]
+get_sr r56l, sr26
+# CHECK: get_sr r56l, sr26{{ +}}// encoding: [0x72,0x60,0x1a,0x10]
+get_sr r22l.cache, sr232
+# CHECK: get_sr r22l.cache, sr232{{ +}}// encoding: [0xf2,0x58,0x28,0x0c]
+get_sr r42l, sr106
+# CHECK: get_sr r42l, sr106{{ +}}// encoding: [0x72,0x28,0x2a,0x14]
+
+isub.sat r106l.cache, u181h, u183h, lsl 3
+# CHECK: isub.sat r106l.cache, u181h, u183h, lsl 3{{ +}}// encoding: [0xce,0x28,0x6b,0xf9,0x96,0x35,0x10,0x00]
+isub r97l.cache, r127.cache, 236, lsl 7
+# CHECK: isub r97l.cache, r127.cache, 236, lsl 7{{ +}}// encoding: [0x8e,0x04,0xbe,0xca,0x82,0x3f,0x30,0x00]
+iadd.sat r83l.cache, 58, u130, lsl 3
+# CHECK: iadd.sat r83l.cache, 58, u130, lsl 3{{ +}}// encoding: [0xce,0x4c,0x3a,0x40,0x9c,0x20,0x10,0x00]
+iadd.sat r72, r66h.cache.sx, u232.sx, lsl 7
+# CHECK: iadd.sat r72, r66h.cache.sx, u232.sx, lsl 7{{ +}}// encoding: [0x4e,0x21,0x85,0x04,0xdd,0x2b,0x30,0x00]
+
+icmpsel sgt, r19l.cache, r63h.discard, u95h, 78, 162
+# CHECK: icmpsel sgt, r19l.cache, r63h.discard, u95h, 78, 162{{ +}}// encoding: [0x92,0xcc,0xff,0xf0,0x13,0x0e,0x21,0xd2,0x60,0x06]
+icmpsel seq, r122, r69h, r96h.discard, u62, r88
+# CHECK: icmpsel seq, r122, r69h, r96h.discard, u62, r88{{ +}}// encoding: [0x12,0xe9,0x4b,0x10,0x0c,0xbc,0x01,0x87,0x60,0x3b]
+icmpsel ult, r18.cache, r118h.cache, u77, 212, r94.cache
+# CHECK: icmpsel ult, r18.cache, r118h.cache, u77, 212, r94.cache{{ +}}// encoding: [0x92,0xc9,0xad,0xa0,0x19,0x14,0xc1,0x2b,0xe0,0x0e]
+icmpsel ueq, r26l, u253l, r118h, r47l, r19h.cache
+# CHECK: icmpsel ueq, r26l, u253l, r118h, r47l, r19h.cache{{ +}}// encoding: [0x12,0xe8,0x7a,0xd1,0x06,0x5e,0x70,0x0a,0x40,0x0f]
+
+imsub r63.cache, r86h.discard, u50l.sx, r18l.cache.sx, lsl 7
+# CHECK: imsub r63.cache, r86h.discard, u50l.sx, r18l.cache.sx, lsl 7{{ +}}// encoding: [0x9e,0x7d,0xed,0x48,0xd2,0xa4,0x34,0x19]
+imadd.sat r20_r21, u151l, u193l, u62h.sx, lsl 7
+# CHECK: imadd.sat r20_r21, u151l, u193l, u62h.sx, lsl 7{{ +}}// encoding: [0x5e,0x53,0x6e,0x21,0x94,0x3d,0x75,0x02]
+imsub.sat r103h.cache, u164l, r24l.discard, r23h.discard.sx, lsl 5
+# CHECK: imsub.sat r103h.cache, u164l, r24l.discard, r23h.discard.sx, lsl 5{{ +}}// encoding: [0xde,0x1e,0x48,0x09,0x8f,0xef,0x24,0x34]
+imadd r91, u24h.sx, r10l.sx, r101.discard, lsl 3
+# CHECK: imadd r91, u24h.sx, r10l.sx, r101.discard, lsl 3{{ +}}// encoding: [0x1e,0x6d,0x31,0x45,0xc5,0xca,0xd2,0x20]
+
+intl r13h.cache, r104, u73l
+# CHECK: intl r13h.cache, r104, u73l{{ +}}// encoding: [0xbe,0x36,0x50,0x22,0x11,0x0e]
+intl r57l.cache, r30h.discard, r122h.cache
+# CHECK: intl r57l.cache, r30h.discard, r122h.cache{{ +}}// encoding: [0xbe,0x64,0xfd,0x50,0x0b,0x13]
+intl r27.cache, 243, r37h.discard
+# CHECK: intl r27.cache, 243, r37h.discard{{ +}}// encoding: [0xbe,0x6d,0x33,0xb0,0x0c,0x0d]
+intl r25.cache, 201, u108
+# CHECK: intl r25.cache, 201, u108{{ +}}// encoding: [0xbe,0x65,0x09,0x80,0x19,0x0f]
+
+mov_imm r54h, 38267
+# CHECK: mov_imm r54h, 38267{{ +}}// encoding: [0x62,0xda,0x7b,0x95,0x00,0x10]
+mov_imm r75h.cache, 41076
+# CHECK: mov_imm r75h.cache, 41076{{ +}}// encoding: [0xe2,0xae,0x74,0xa0,0x00,0x20]
+mov_imm r72h.cache, 39409
+# CHECK: mov_imm r72h.cache, 39409{{ +}}// encoding: [0xe2,0xa2,0xf1,0x99,0x00,0x20]
+mov_imm r119h, 28714
+# CHECK: mov_imm r119h, 28714{{ +}}// encoding: [0x62,0xde,0x2a,0x70,0x00,0x30]
+
+popcount r22h, r92l.discard
+# CHECK: popcount r22h, r92l.discard{{ +}}// encoding: [0x3e,0x5a,0xf8,0x08,0x00,0x08]
+popcount r85l, r95h
+# CHECK: popcount r85l, r95h{{ +}}// encoding: [0x3e,0x54,0x7f,0x08,0x00,0x28]
+popcount r57l, u191
+# CHECK: popcount r57l, u191{{ +}}// encoding: [0x3e,0x64,0xfe,0x09,0x00,0x14]
+popcount r2h.cache, 54
+# CHECK: popcount r2h.cache, 54{{ +}}// encoding: [0xbe,0x0a,0x36,0x08,0x00,0x00]
+
+shlhi r118l, r109.discard, r119l.cache, r9, mask 0xFF
+# CHECK: shlhi r118l, r109.discard, r119l.cache, r9, mask 0xFF{{ +}}// encoding: [0x2e,0x58,0xda,0xea,0x0a,0x52,0x0a,0x3f]
+shlhi r53l.cache, 251, 57, r33.discard, mask 0xFFFFF
+# CHECK: shlhi r53l.cache, 251, 57, r33.discard, mask 0xFFFFF{{ +}}// encoding: [0xae,0x54,0x3b,0x98,0x03,0xc2,0x46,0x9c]
+shlhi r87h.cache, r116l, r96.discard, 6
+# CHECK: shlhi r87h.cache, r116l, r96.discard, 6{{ +}}// encoding: [0xae,0x5e,0x68,0x08,0x2c,0x06,0x00,0x2f]
+shlhi r1l, u15h, 168, u57h, mask 0x7F
+# CHECK: shlhi r1l, u15h, 168, u57h, mask 0x7F{{ +}}// encoding: [0x2e,0x04,0x1f,0x89,0xc2,0x33,0x45,0x02]
+
+shrhi r92l.cache, u157h, u120, u64, mask 0xF
+# CHECK: shrhi r92l.cache, u157h, u120, u64, mask 0xF{{ +}}// encoding: [0xae,0xf0,0x7b,0x09,0x1b,0x80,0x85,0x23]
+shrhi r78h, u74l, r10l.cache, r30h.discard
+# CHECK: shrhi r78h, u74l, r10l.cache, r30h.discard{{ +}}// encoding: [0x2e,0xba,0x14,0x49,0x09,0xfd,0x00,0x28]
+shrhi r20, r9h.cache, r66.cache, u200h, mask 0x1F
+# CHECK: shrhi r20, r9h.cache, r66.cache, u200h, mask 0x1F{{ +}}// encoding: [0x2e,0xd1,0x93,0x48,0x68,0x51,0x85,0x02]
+shrhi r120.cache, r72l.cache, r5h, r120l.discard, mask 0x3FFFF
+# CHECK: shrhi r120.cache, r72l.cache, r5h, r120l.discard, mask 0x3FFFF{{ +}}// encoding: [0xae,0xe1,0x90,0xb8,0x84,0xf0,0xc0,0xb8]
+
diff --git a/llvm/test/MC/AGX/g13-control.s b/llvm/test/MC/AGX/g13-control.s
new file mode 100644
index 0000000000000..56e2869235d44
--- /dev/null
+++ b/llvm/test/MC/AGX/g13-control.s
@@ -0,0 +1,117 @@
+# RUN: llvm-mc -triple=agx -mcpu=agx-g13g -show-encoding %s | FileCheck %s
+# RUN: llvm-mc -triple=agx -mcpu=agx-g14c -show-encoding %s | FileCheck %s
+
+# Encodings cross-checked against the applegpu reference assembler and
+# disassembler (dougallj/applegpu): each case is a random canonical encoding
+# that applegpu disassembles to the same text.
+
+else_fcmp r0l, gte, u217.neg, r101l.cache, 1
+# CHECK: else_fcmp r0l, gte, u217.neg, r101l.cache, 1{{ +}}// encoding: [0x42,0xaa,0xf2,0xa9,0x08,0x0b]
+else_fcmp r0l, ngte, u196l.abs, r60.discard.abs, 1
+# CHECK: else_fcmp r0l, ngte, u196l.abs, r60.discard.abs, 1{{ +}}// encoding: [0x42,0xab,0x48,0x85,0x6f,0x09]
+else_fcmp r0l.cache, lte, r108.cache.abs, u221h, 2
+# CHECK: else_fcmp r0l.cache, lte, r108.cache.abs, u221h, 2{{ +}}// encoding: [0xc2,0xd2,0x98,0xb6,0x17,0x0e]
+else_fcmp r0l.cache, nltn, r10.abs, r110l.neg, 2
+# CHECK: else_fcmp r0l.cache, nltn, r10.abs, r110l.neg, 2{{ +}}// encoding: [0xc2,0x73,0x54,0xc6,0x85,0x03]
+
+else_icmp r0l, ult, u197l, r65l, 1
+# CHECK: else_icmp r0l, ult, u197l, r65l, 1{{ +}}// encoding: [0x52,0x2a,0x4a,0x21,0x04,0x0a]
+else_icmp r0l, ueq, u106h, 2, 0
+# CHECK: else_icmp r0l, ueq, u106h, 2, 0{{ +}}// encoding: [0x52,0x02,0x15,0x21,0x00,0x0c]
+else_icmp r0l, seq, u161h, r74h, 0
+# CHECK: else_icmp r0l, seq, u161h, r74h, 0{{ +}}// encoding: [0x52,0x82,0x43,0x51,0x05,0x06]
+else_icmp r0l, ult, r81l.discard, u17, 1
+# CHECK: else_icmp r0l, ult, r81l.discard, u17, 1{{ +}}// encoding: [0x52,0x2a,0xe2,0x20,0x1a,0x08]
+
+if_fcmp r0l, lt, u179h.abs, r104h, 0
+# CHECK: if_fcmp r0l, lt, u179h.abs, r104h, 0{{ +}}// encoding: [0x42,0x20,0x67,0x15,0x05,0x07]
+if_fcmp r0l.cache, ngtn, u200.neg, u15l.neg, 2
+# CHECK: if_fcmp r0l.cache, ngtn, u200.neg, u15l.neg, 2{{ +}}// encoding: [0xc2,0xf1,0xd0,0xe9,0x91,0x08]
+if_fcmp r0l.cache, lte, r122h.discard, r100l.neg, 0
+# CHECK: if_fcmp r0l.cache, lte, r122h.discard, r100l.neg, 0{{ +}}// encoding: [0xc2,0xc0,0xf5,0x80,0x84,0x0f]
+if_fcmp r0l, nlte, u145, r38.cache, 3
+# CHECK: if_fcmp r0l, nlte, u145, r38.cache, 3{{ +}}// encoding: [0x42,0xd9,0xe2,0xc1,0x28,0x01]
+
+if_icmp r0l, ugte, u24l, u47h, 0
+# CHECK: if_icmp r0l, ugte, u24l, u47h, 0{{ +}}// encoding: [0x52,0x21,0x30,0xf1,0x11,0x01]
+if_icmp r0l, slte, u29l, u200, 0
+# CHECK: if_icmp r0l, slte, u29l, u200, 0{{ +}}// encoding: [0x52,0xc1,0x3a,0x01,0x1d,0x02]
+if_icmp r0l, ugt, u92h, r108l, 2
+# CHECK: if_icmp r0l, ugt, u92h, r108l, 2{{ +}}// encoding: [0x52,0x50,0x39,0x81,0x05,0x0b]
+if_icmp r0l.cache, seq, u81l, r114h, 0
+# CHECK: if_icmp r0l.cache, seq, u81l, r114h, 0{{ +}}// encoding: [0xd2,0x80,0x22,0x51,0x06,0x0b]
+
+
+
+memory_barrier 1, 0, 15
+# CHECK: memory_barrier 1, 0, 15{{ +}}// encoding: [0xf5,0xf4]
+memory_barrier 3, 3, 13
+# CHECK: memory_barrier 3, 3, 13{{ +}}// encoding: [0xf5,0xdf]
+memory_barrier 3, 0, 9
+# CHECK: memory_barrier 3, 0, 9{{ +}}// encoding: [0xf5,0x9c]
+memory_barrier 2, 1, 12
+# CHECK: memory_barrier 2, 1, 12{{ +}}// encoding: [0xf5,0xc9]
+
+pop_exec r0l, 0
+# CHECK: pop_exec r0l, 0{{ +}}// encoding: [0x52,0x06,0x00,0x00,0x00,0x00]
+pop_exec r0l, 3
+# CHECK: pop_exec r0l, 3{{ +}}// encoding: [0x52,0x1e,0x00,0x00,0x00,0x00]
+pop_exec r0l.cache, 3
+# CHECK: pop_exec r0l.cache, 3{{ +}}// encoding: [0xd2,0x1e,0x00,0x00,0x00,0x00]
+pop_exec r0l.cache, 2
+# CHECK: pop_exec r0l.cache, 2{{ +}}// encoding: [0xd2,0x16,0x00,0x00,0x00,0x00]
+
+stop
+# CHECK: stop{{ +}}// encoding: [0x88,0x00]
+stop
+# CHECK: stop{{ +}}// encoding: [0x88,0x00]
+stop
+# CHECK: stop{{ +}}// encoding: [0x88,0x00]
+stop
+# CHECK: stop{{ +}}// encoding: [0x88,0x00]
+
+threadgroup_barrier
+# CHECK: threadgroup_barrier{{ +}}// encoding: [0x68,0x00]
+threadgroup_barrier
+# CHECK: threadgroup_barrier{{ +}}// encoding: [0x68,0x00]
+threadgroup_barrier
+# CHECK: threadgroup_barrier{{ +}}// encoding: [0x68,0x00]
+threadgroup_barrier
+# CHECK: threadgroup_barrier{{ +}}// encoding: [0x68,0x00]
+
+trap
+# CHECK: trap{{ +}}// encoding: [0x08,0x00]
+trap
+# CHECK: trap{{ +}}// encoding: [0x08,0x00]
+trap
+# CHECK: trap{{ +}}// encoding: [0x08,0x00]
+trap
+# CHECK: trap{{ +}}// encoding: [0x08,0x00]
+
+wait 1
+# CHECK: wait 1{{ +}}// encoding: [0x38,0x01]
+wait 0
+# CHECK: wait 0{{ +}}// encoding: [0x38,0x00]
+wait 0
+# CHECK: wait 0{{ +}}// encoding: [0x38,0x00]
+wait 1
+# CHECK: wait 1{{ +}}// encoding: [0x38,0x01]
+
+while_fcmp r0l, gtn, r77l.discard, u38.abs.neg, 2
+# CHECK: while_fcmp r0l, gtn, r77l.discard, u38.abs.neg, 2{{ +}}// encoding: [0x42,0xf4,0xda,0xc0,0xd8,0x09]
+while_fcmp r0l.cache, ltn, r35l.abs.neg, 3.125, 1
+# CHECK: while_fcmp r0l.cache, ltn, r35l.abs.neg, 3.125, 1{{ +}}// encoding: [0xc2,0x6c,0x46,0x9c,0x00,0x05]
+while_fcmp r0l, lte, r10.neg, u167l.abs.neg, 2
+# CHECK: while_fcmp r0l, lte, r10.neg, u167l.abs.neg, 2{{ +}}// encoding: [0x42,0xd4,0x54,0xea,0xd4,0x01]
+while_fcmp r0l, lt, r81h.cache.neg, u131, 3
+# CHECK: while_fcmp r0l, lt, r81h.cache.neg, u131, 3{{ +}}// encoding: [0x42,0x3c,0xa3,0x68,0x1c,0x08]
+
+while_icmp r0l, ueq, r83, u5, 2
+# CHECK: while_icmp r0l, ueq, r83, u5, 2{{ +}}// encoding: [0x52,0x14,0x66,0xa2,0x18,0x08]
+while_icmp r0l.cache, ulte, r48h.cache, u64, 1
+# CHECK: while_icmp r0l.cache, ulte, r48h.cache, u64, 1{{ +}}// encoding: [0xd2,0x4d,0xa1,0x00,0x18,0x06]
+while_icmp r0l.cache, seq, r20h.discard, u43l, 3
+# CHECK: while_icmp r0l.cache, seq, r20h.discard, u43l, 3{{ +}}// encoding: [0xd2,0x9c,0xe9,0x60,0x11,0x01]
+while_icmp r0l.cache, seq, r45l, r73l, 3
+# CHECK: while_icmp r0l.cache, seq, r45l, r73l, 3{{ +}}// encoding: [0xd2,0x9c,0x5a,0x20,0x05,0x06]
+
diff --git a/llvm/test/MC/AGX/g13-fp.s b/llvm/test/MC/AGX/g13-fp.s
new file mode 100644
index 0000000000000..134690b44da54
--- /dev/null
+++ b/llvm/test/MC/AGX/g13-fp.s
@@ -0,0 +1,162 @@
+# RUN: llvm-mc -triple=agx -mcpu=agx-g13g -show-encoding %s | FileCheck %s
+# RUN: llvm-mc -triple=agx -mcpu=agx-g14c -show-encoding %s | FileCheck %s
+
+# Encodings cross-checked against the applegpu reference assembler and
+# disassembler (dougallj/applegpu): each case is a random canonical encoding
+# that applegpu disassembles to the same text.
+
+ceil.sat r74h, u79h
+# CHECK: ceil.sat r74h, u79h{{ +}}// encoding: [0x4a,0xaa,0x1f,0x01,0x01,0x28]
+ceil r90l, r118h.neg
+# CHECK: ceil r90l, r118h.neg{{ +}}// encoding: [0x0a,0xe8,0x6d,0x08,0x01,0x2c]
+ceil.sat r10.cache, r59h.abs.neg
+# CHECK: ceil.sat r10.cache, r59h.abs.neg{{ +}}// encoding: [0xca,0xa9,0x77,0x0c,0x01,0x04]
+ceil r94l, u165.neg
+# CHECK: ceil r94l, u165.neg{{ +}}// encoding: [0x0a,0xf8,0xca,0x09,0x01,0x24]
+
+
+
+exp2.sat r54.cache, r0.cache.abs
+# CHECK: exp2.sat r54.cache, r0.cache.abs{{ +}}// encoding: [0xca,0xd9,0x80,0xd6,0x00,0x10]
+exp2.sat r103l.cache, u118
+# CHECK: exp2.sat r103l.cache, u118{{ +}}// encoding: [0xca,0x9c,0xac,0xd1,0x00,0x3c]
+exp2 r112h.cache, r95.abs
+# CHECK: exp2 r112h.cache, r95.abs{{ +}}// encoding: [0x8a,0xc2,0x7e,0xd6,0x00,0x38]
+exp2.sat r109.cache, u106h
+# CHECK: exp2.sat r109.cache, u106h{{ +}}// encoding: [0xca,0xb5,0x15,0xd1,0x00,0x3c]
+
+fadd16.sat r16h.cache, 7.5, u143h.neg
+# CHECK: fadd16.sat r16h.cache, 7.5, u143h.neg{{ +}}// encoding: [0xe6,0xc2,0x1e,0xf0,0x55,0x04]
+fadd16.sat r52l.cache, u24l.abs, r117h.discard
+# CHECK: fadd16.sat r52l.cache, u24l.abs, r117h.discard{{ +}}// encoding: [0xe6,0xd0,0x30,0xb3,0x0e,0x13]
+fadd16 r80h.cache, r12h, u105l.neg
+# CHECK: fadd16 r80h.cache, r12h, u105l.neg{{ +}}// encoding: [0xa6,0xc2,0x59,0x20,0x51,0x23]
+fadd16.sat r127l.cache, r120h.cache.abs.neg, u155l
+# CHECK: fadd16.sat r127l.cache, r120h.cache.abs.neg, u155l{{ +}}// encoding: [0xe6,0xfc,0xb1,0x66,0x17,0x3c]
+
+fadd32 r8l.cache, -7.5, r42.discard.abs.neg
+# CHECK: fadd32 r8l.cache, -7.5, r42.discard.abs.neg{{ +}}// encoding: [0xaa,0xa0,0x1e,0x40,0xed,0x0d]
+fadd32.sat r102h.cache, r71h.discard.neg, r10h.discard.abs
+# CHECK: fadd32.sat r102h.cache, r71h.discard.neg, r10h.discard.abs{{ +}}// encoding: [0xea,0x9a,0xcf,0x58,0x4d,0x38]
+fadd32 r124h, r54.discard.abs.neg, r54.discard
+# CHECK: fadd32 r124h, r54.discard.abs.neg, r54.discard{{ +}}// encoding: [0x2a,0xf2,0xec,0xce,0x2e,0x35]
+fadd32 r16h.cache, u89h.abs, u90h.abs
+# CHECK: fadd32 r16h.cache, u89h.abs, u90h.abs{{ +}}// encoding: [0xaa,0xc2,0x33,0x55,0x53,0x0a]
+
+floor.sat r98l, r36h.cache.abs
+# CHECK: floor.sat r98l, r36h.cache.abs{{ +}}// encoding: [0x4a,0x88,0x89,0x04,0x00,0x34]
+floor.sat r44h.cache, r31h.cache
+# CHECK: floor.sat r44h.cache, r31h.cache{{ +}}// encoding: [0xca,0xb2,0xbf,0x00,0x00,0x10]
+floor r121, r45l.cache.neg
+# CHECK: floor r121, r45l.cache.neg{{ +}}// encoding: [0x0a,0xe5,0x9a,0x08,0x00,0x34]
+floor.sat r18l, r118l.cache.neg
+# CHECK: floor.sat r18l, r118l.cache.neg{{ +}}// encoding: [0x4a,0xc8,0xac,0x08,0x00,0x0c]
+
+fmadd16 r56l.cache, r31l.neg, r86h.discard.neg, u253l.neg
+# CHECK: fmadd16 r56l.cache, r31l.neg, r86h.discard.neg, u253l.neg{{ +}}// encoding: [0xb6,0xe0,0x7e,0xd4,0x4e,0x7a,0xc5,0x12]
+fmadd16.sat r101l, r14l.abs, u154l, u150.abs
+# CHECK: fmadd16.sat r101l, r14l.abs, u154l, u150.abs{{ +}}// encoding: [0x76,0x94,0x5c,0x42,0x17,0xec,0x03,0x30]
+fmadd16 r5h, u163.neg, r27h.abs.neg, u217.abs.neg
+# CHECK: fmadd16 r5h, u163.neg, r27h.abs.neg, u217.abs.neg{{ +}}// encoding: [0x36,0x96,0xc6,0x75,0x67,0xf2,0x87,0x04]
+fmadd16.sat r50l.cache, u180l.neg, u99.neg, r94h.cache
+# CHECK: fmadd16.sat r50l.cache, u180l.neg, u99.neg, r94h.cache{{ +}}// encoding: [0xf6,0xc8,0x68,0x65,0x58,0xbd,0x80,0x17]
+
+fmadd32 r4.cache, u178l.abs.neg, u219l.abs.neg, r106h.abs.neg
+# CHECK: fmadd32 r4.cache, u178l.abs.neg, u219l.abs.neg, r106h.abs.neg{{ +}}// encoding: [0xba,0x91,0x64,0x6d,0xd7,0x55,0xcc,0x06]
+fmadd32 r34.cache, r45h.neg, r35h.cache.neg, u192l.abs.neg
+# CHECK: fmadd32 r34.cache, r45h.neg, r35h.cache.neg, u192l.abs.neg{{ +}}// encoding: [0xba,0x89,0x5b,0x78,0x88,0x40,0x8d,0x15]
+fmadd32.sat r101.cache, r44.cache.abs.neg, u16h.abs.neg, r104.discard
+# CHECK: fmadd32.sat r101.cache, r44.cache.abs.neg, u16h.abs.neg, r104.discard{{ +}}// encoding: [0xfa,0x95,0x98,0x1e,0xd2,0xd0,0xc2,0x34]
+fmadd32 r38h, u205l.abs.neg, r70l.cache.neg, r121.cache
+# CHECK: fmadd32 r38h, u205l.abs.neg, r70l.cache.neg, r121.cache{{ +}}// encoding: [0x3a,0x9a,0x5a,0xcd,0x88,0xb2,0xc2,0x1a]
+
+fmul16 r73l.cache, r77l, u193
+# CHECK: fmul16 r73l.cache, r77l, u193{{ +}}// encoding: [0x96,0xa4,0x5a,0x20,0x1c,0x2a]
+fmul16.sat r91h.cache, u46h.abs.neg, u167h.abs.neg
+# CHECK: fmul16.sat r91h.cache, u46h.abs.neg, u167h.abs.neg{{ +}}// encoding: [0xd6,0xee,0x1d,0xf7,0x74,0x25]
+fmul16 r16l, r17l.discard.neg, r42l.abs.neg
+# CHECK: fmul16 r16l, r17l.discard.neg, r42l.abs.neg{{ +}}// encoding: [0x16,0xc0,0xe2,0x44,0x65,0x01]
+fmul16.sat r107l.cache, u156l, r79l.abs.neg
+# CHECK: fmul16.sat r107l.cache, u156l, r79l.abs.neg{{ +}}// encoding: [0xd6,0xac,0x78,0xe1,0x65,0x32]
+
+fmul32 r28l, u6.abs, r7.neg
+# CHECK: fmul32 r28l, u6.abs, r7.neg{{ +}}// encoding: [0x1a,0xf0,0x8c,0xe5,0xa4,0x00]
+fmul32.sat r16h.cache, u208.neg, u196l.neg
+# CHECK: fmul32.sat r16h.cache, u208.neg, u196l.neg{{ +}}// encoding: [0xda,0xc2,0xe0,0x89,0x94,0x0a]
+fmul32 r65.cache, u42h.neg, u208h.neg
+# CHECK: fmul32 r65.cache, u42h.neg, u208h.neg{{ +}}// encoding: [0x9a,0x85,0x15,0x19,0x96,0x26]
+fmul32.sat r15l.cache, r38h.discard.neg, u124
+# CHECK: fmul32.sat r15l.cache, r38h.discard.neg, u124{{ +}}// encoding: [0xda,0xbc,0xcd,0x88,0x1b,0x07]
+
+log2 r121l.cache, r7.abs
+# CHECK: log2 r121l.cache, r7.abs{{ +}}// encoding: [0x8a,0xe4,0x4e,0xc6,0x00,0x30]
+log2.sat r83.cache, r91.abs
+# CHECK: log2.sat r83.cache, r91.abs{{ +}}// encoding: [0xca,0xcd,0x76,0xc6,0x00,0x28]
+log2.sat r45.cache, u69
+# CHECK: log2.sat r45.cache, u69{{ +}}// encoding: [0xca,0xb5,0x8a,0xc1,0x00,0x18]
+log2.sat r71l, r122h.neg
+# CHECK: log2.sat r71l, r122h.neg{{ +}}// encoding: [0x4a,0x9c,0x75,0xc8,0x00,0x2c]
+
+rcp.sat r91.cache, u245h
+# CHECK: rcp.sat r91.cache, u245h{{ +}}// encoding: [0xca,0xed,0x6b,0x81,0x00,0x2c]
+rcp.sat r39.cache, r107.cache.abs
+# CHECK: rcp.sat r39.cache, r107.cache.abs{{ +}}// encoding: [0xca,0x9d,0x96,0x86,0x00,0x1c]
+rcp r112h.cache, u107.abs.neg
+# CHECK: rcp r112h.cache, u107.abs.neg{{ +}}// encoding: [0x8a,0xc2,0x96,0x8d,0x00,0x3c]
+rcp.sat r80l, r55l.discard.abs.neg
+# CHECK: rcp.sat r80l, r55l.discard.abs.neg{{ +}}// encoding: [0x4a,0xc0,0xee,0x8c,0x00,0x24]
+
+rint r11l.cache, u97l.abs.neg
+# CHECK: rint r11l.cache, u97l.abs.neg{{ +}}// encoding: [0x8a,0xac,0x02,0x0d,0x03,0x0c]
+rint.sat r90.cache, u85l.abs
+# CHECK: rint.sat r90.cache, u85l.abs{{ +}}// encoding: [0xca,0xe9,0x2a,0x05,0x03,0x28]
+rint r104l.cache, r13l.abs.neg
+# CHECK: rint r104l.cache, r13l.abs.neg{{ +}}// encoding: [0x8a,0xa0,0x5a,0x0c,0x03,0x30]
+rint r85h.cache, u152
+# CHECK: rint r85h.cache, u152{{ +}}// encoding: [0x8a,0xd6,0xf0,0x01,0x03,0x20]
+
+rsqrt.sat r97, r102h.cache.neg
+# CHECK: rsqrt.sat r97, r102h.cache.neg{{ +}}// encoding: [0x4a,0x85,0x8d,0x98,0x00,0x3c]
+rsqrt.sat r114l.cache, u62h.abs
+# CHECK: rsqrt.sat r114l.cache, u62h.abs{{ +}}// encoding: [0xca,0xc8,0x3d,0x95,0x00,0x34]
+rsqrt.sat r84.cache, u91l
+# CHECK: rsqrt.sat r84.cache, u91l{{ +}}// encoding: [0xca,0xd1,0x36,0x91,0x00,0x28]
+rsqrt r73, r10l.neg
+# CHECK: rsqrt r73, r10l.neg{{ +}}// encoding: [0x0a,0xa5,0x54,0x98,0x00,0x20]
+
+rsqrt_special r46h.cache, r26l
+# CHECK: rsqrt_special r46h.cache, r26l{{ +}}// encoding: [0x8a,0xba,0x74,0x10,0x00,0x10]
+rsqrt_special r93, u120
+# CHECK: rsqrt_special r93, u120{{ +}}// encoding: [0x0a,0xf5,0xb0,0x11,0x00,0x2c]
+rsqrt_special.sat r1.cache, u148l.abs.neg
+# CHECK: rsqrt_special.sat r1.cache, u148l.abs.neg{{ +}}// encoding: [0xca,0x05,0x68,0x1d]
+rsqrt_special.sat r53h, u24h.neg
+# CHECK: rsqrt_special.sat r53h, u24h.neg{{ +}}// encoding: [0x4a,0xd6,0x31,0x19,0x00,0x10]
+
+sin_pt_1.sat r62l.cache, r94.cache
+# CHECK: sin_pt_1.sat r62l.cache, r94.cache{{ +}}// encoding: [0xca,0xf8,0xbc,0xa2,0x00,0x18]
+sin_pt_1.sat r14.cache, u170h.neg
+# CHECK: sin_pt_1.sat r14.cache, u170h.neg{{ +}}// encoding: [0xca,0xb9,0x55,0xa9,0x00,0x04]
+sin_pt_1.sat r79l, u6
+# CHECK: sin_pt_1.sat r79l, u6{{ +}}// encoding: [0x4a,0xbc,0x8c,0xa1,0x00,0x20]
+sin_pt_1 r115l.cache, u44
+# CHECK: sin_pt_1 r115l.cache, u44{{ +}}// encoding: [0x8a,0xcc,0x98,0xa1,0x00,0x34]
+
+sin_pt_2.sat r31h, r21h.discard.abs
+# CHECK: sin_pt_2.sat r31h, r21h.discard.abs{{ +}}// encoding: [0x4a,0x7e,0xeb,0xe4]
+sin_pt_2.sat r48l, r104h.discard.abs.neg
+# CHECK: sin_pt_2.sat r48l, r104h.discard.abs.neg{{ +}}// encoding: [0x4a,0xc0,0xd1,0xec,0x00,0x1c]
+sin_pt_2 r118, r68l.cache.abs.neg
+# CHECK: sin_pt_2 r118, r68l.cache.abs.neg{{ +}}// encoding: [0x0a,0xd9,0x88,0xec,0x00,0x38]
+sin_pt_2 r82l, r12h.cache.abs.neg
+# CHECK: sin_pt_2 r82l, r12h.cache.abs.neg{{ +}}// encoding: [0x0a,0xc8,0x99,0xec,0x00,0x20]
+
+trunc.sat r82, u57l.abs
+# CHECK: trunc.sat r82, u57l.abs{{ +}}// encoding: [0x4a,0xc9,0x32,0x05,0x02,0x24]
+trunc.sat r30, u117l.neg
+# CHECK: trunc.sat r30, u117l.neg{{ +}}// encoding: [0x4a,0xf9,0x2a,0x09,0x02,0x0c]
+trunc.sat r124l.cache, r55l.discard.abs
+# CHECK: trunc.sat r124l.cache, r55l.discard.abs{{ +}}// encoding: [0xca,0xf0,0xee,0x04,0x02,0x34]
+trunc r126h.cache, u57.abs
+# CHECK: trunc r126h.cache, u57.abs{{ +}}// encoding: [0x8a,0xfa,0xb2,0x05,0x02,0x34]
+
diff --git a/llvm/test/MC/AGX/g13-memory.s b/llvm/test/MC/AGX/g13-memory.s
new file mode 100644
index 0000000000000..4d1f6aedf3e67
--- /dev/null
+++ b/llvm/test/MC/AGX/g13-memory.s
@@ -0,0 +1,44 @@
+# RUN: llvm-mc -triple=agx -mcpu=agx-g13g -show-encoding %s | FileCheck %s
+# RUN: llvm-mc -triple=agx -mcpu=agx-g14c -show-encoding %s | FileCheck %s
+
+# Encodings cross-checked against the applegpu reference assembler and
+# disassembler (dougallj/applegpu): each case is a random canonical encoding
+# that applegpu disassembles to the same text.
+
+atomic umin, 0, r2, r37_r38, -28392, signed, r118, 0
+# CHECK: atomic umin, 0, r2, r37_r38, -28392, signed, r118, 0{{ +}}// encoding: [0x15,0x11,0x8a,0x85,0x41,0xd0,0xec,0x91]
+atomic and, 0, r39, r121_r122, -12555, unsigned, r21, 1
+# CHECK: atomic and, 0, r39, r121_r122, -12555, unsigned, r21, 1{{ +}}// encoding: [0x15,0x3a,0x52,0x87,0xff,0xf1,0x2a,0xce]
+atomic 11, 1, r74, r10_r11, -17279, signed, r76, 0
+# CHECK: atomic 11, 1, r74, r10_r11, -17279, signed, r76, 0{{ +}}// encoding: [0xd5,0x52,0x14,0xc5,0x18,0xd2,0x98,0xbc]
+atomic add, 0, r45, u43_u44, -1657, signed, r45, 1
+# CHECK: atomic add, 0, r45, u43_u44, -1657, signed, r45, 1{{ +}}// encoding: [0x15,0x68,0x76,0x8d,0x58,0xf1,0x5a,0xf9]
+
+
+device_store 1, i8, xy, r92l_r92h, r25_r26, 29863, signed, lsl 1
+# CHECK: device_store 1, i8, xy, r92l_r92h, r25_r26, 29863, signed, lsl 1{{ +}}// encoding: [0x45,0xe0,0x72,0x45,0x3a,0xc6,0x30,0x74]
+device_store 1, rgb10a2, xz, r97h_r98l, r107_r108, 14332, signed, lsl 1
+# CHECK: device_store 1, rgb10a2, xz, r97h_r98l, r107_r108, 14332, signed, lsl 1{{ +}}// encoding: [0x45,0x0c,0xc6,0x45,0xdf,0xc7,0x51,0x37]
+device_store 0, rg11b10f.rtz, xyw, r88h_r89l_r89h, r90_r91, 24244, signed
+# CHECK: device_store 0, rg11b10f.rtz, xyw, r88h_r89l_r89h, r90_r91, 24244, signed{{ +}}// encoding: [0x45,0xc7,0x44,0x05,0xbb,0xc2,0xb1,0x5e]
+device_store 0, srgba8, xyzw, r47h_r48l_r48h_r49l, u73_u74, -13603, signed, lsl 1
+# CHECK: device_store 0, srgba8, xyzw, r47h_r48l_r48h_r49l, u73_u74, -13603, signed, lsl 1{{ +}}// encoding: [0x45,0x7d,0xd2,0x0d,0x9d,0xc5,0xf1,0xca]
+
+threadgroup_load u8norm, xy, r92_r93, u217h, 17515
+# CHECK: threadgroup_load u8norm, xy, r92_r93, u217h, 17515{{ +}}// encoding: [0x69,0xf1,0xf3,0xb4,0x36,0x00,0x11,0x29]
+threadgroup_load i16, xyw, r105h_r106l_r106h, u110l, -27664
+# CHECK: threadgroup_load i16, xyw, r105h_r106l_r106h, u110l, -27664{{ +}}// encoding: [0x69,0xa6,0x5c,0x01,0xb7,0x00,0x4f,0x3e]
+threadgroup_load srgba8, x, r60h, r32l, -2730
+# CHECK: threadgroup_load srgba8, x, r60h, r32l, -2730{{ +}}// encoding: [0x69,0xf2,0x00,0x6a,0x15,0x00,0xd5,0x17]
+threadgroup_load i16, z, r10l, r103h, -28133
+# CHECK: threadgroup_load i16, z, r10l, r103h, -28133{{ +}}// encoding: [0x69,0xa8,0x0f,0xb1,0x45,0x00,0x48,0x0e]
+
+threadgroup_store rg11b10f.rtz, xy, r0h_r1l, r40l, -10643
+# CHECK: threadgroup_store rg11b10f.rtz, xy, r0h_r1l, r40l, -10643{{ +}}// encoding: [0x29,0x82,0x10,0xde,0x36,0x00,0x59,0x07]
+threadgroup_store u16norm, xy, r89h_r90l, r50l, -14279
+# CHECK: threadgroup_store u16norm, xy, r89h_r90l, r50l, -14279{{ +}}// encoding: [0x29,0xe6,0x24,0x96,0x37,0x00,0x20,0x27]
+threadgroup_store i32, xz, r110l_r110h, u65h, -17937
+# CHECK: threadgroup_store i32, xz, r110l_r110h, u65h, -17937{{ +}}// encoding: [0x29,0xb8,0x43,0xf2,0x56,0x00,0xe7,0x3a]
+threadgroup_store rg11b10f.rtz, xzw, r100l_r100h_r101l, u65h, 30145
+# CHECK: threadgroup_store rg11b10f.rtz, xzw, r100l_r100h_r101l, u65h, 30145{{ +}}// encoding: [0x29,0x90,0x43,0x1e,0xd4,0x00,0xd7,0x39]
+
diff --git a/llvm/test/MC/AGX/g13-simd.s b/llvm/test/MC/AGX/g13-simd.s
new file mode 100644
index 0000000000000..6d991e8d3d2a6
--- /dev/null
+++ b/llvm/test/MC/AGX/g13-simd.s
@@ -0,0 +1,466 @@
+# RUN: llvm-mc -triple=agx -mcpu=agx-g13g -show-encoding %s | FileCheck %s
+# RUN: llvm-mc -triple=agx -mcpu=agx-g14c -show-encoding %s | FileCheck %s
+
+# Encodings cross-checked against the applegpu reference assembler and
+# disassembler (dougallj/applegpu): each case is a random canonical encoding
+# that applegpu disassembles to the same text.
+
+fcmp_ballot r85l.cache, nlte, r13l.neg, r81l
+# CHECK: fcmp_ballot r85l.cache, nlte, r13l.neg, r81l{{ +}}// encoding: [0xa2,0x54,0x5a,0x28,0x06,0xa2,0x01,0xc0]
+fcmp_ballot r87, gtn, u43.abs.neg, r63l.cache.abs
+# CHECK: fcmp_ballot r87, gtn, u43.abs.neg, r63l.cache.abs{{ +}}// encoding: [0x22,0x5d,0x96,0xed,0x4b,0x25,0x01,0xe0]
+fcmp_ballot r100l, ngt, u99.neg, 0.21875
+# CHECK: fcmp_ballot r100l, ngt, u99.neg, 0.21875{{ +}}// encoding: [0x22,0x10,0x86,0xe9,0x00,0xbc,0x01,0x40]
+fcmp_ballot r3, eq, r57l.abs.neg, r9h.discard
+# CHECK: fcmp_ballot r3, eq, r57l.abs.neg, r9h.discard{{ +}}// encoding: [0x22,0x0d,0x72,0x3c,0x0d,0x04,0x01,0x00]
+
+fcmp_quad_ballot r44l, ngt, r121h.cache.abs.neg, r125h.abs
+# CHECK: fcmp_quad_ballot r44l, ngt, r121h.cache.abs.neg, r125h.abs{{ +}}// encoding: [0x22,0x30,0xb3,0xbc,0x47,0x9f,0x00,0x40]
+fcmp_quad_ballot r106l, lt, r112h.cache.neg, r81.discard.abs
+# CHECK: fcmp_quad_ballot r106l, lt, r112h.cache.neg, r81.discard.abs{{ +}}// encoding: [0x22,0x28,0xa1,0x28,0x6e,0x3e,0x00,0x20]
+fcmp_quad_ballot r85l.cache, neq, r21l, r88l.discard
+# CHECK: fcmp_quad_ballot r85l.cache, neq, r21l, r88l.discard{{ +}}// encoding: [0xa2,0x54,0x6a,0x00,0x0f,0xa2,0x00,0x00]
+fcmp_quad_ballot r98l.cache, nlt, u184.neg, u202.abs.neg
+# CHECK: fcmp_quad_ballot r98l.cache, nlt, u184.neg, u202.abs.neg{{ +}}// encoding: [0xa2,0x08,0xf0,0x49,0xdd,0xb6,0x00,0x20]
+
+icmp_ballot r37, sgt, u99, u74l
+# CHECK: icmp_ballot r37, sgt, u99, u74l{{ +}}// encoding: [0x32,0x15,0x86,0x41,0x11,0x1e,0x01,0xc0]
+icmp_ballot r34l, seq, r39.discard, u225
+# CHECK: icmp_ballot r34l, seq, r39.discard, u225{{ +}}// encoding: [0x32,0x08,0xce,0x22,0x1c,0x17,0x01,0x80]
+icmp_ballot r24, nseq, 220, r104h.discard
+# CHECK: icmp_ballot r24, nseq, 220, r104h.discard{{ +}}// encoding: [0x32,0x61,0x1c,0x10,0x0d,0x8f,0x01,0x80]
+icmp_ballot r37h.cache, slte, u26l, 207
+# CHECK: icmp_ballot r37h.cache, slte, u26l, 207{{ +}}// encoding: [0xb2,0x16,0x34,0xf1,0x00,0x93,0x01,0xc0]
+
+icmp_quad_ballot r124h, nueq, 147, r87.cache
+# CHECK: icmp_quad_ballot r124h, nueq, 147, r87.cache{{ +}}// encoding: [0x32,0x72,0x13,0xe0,0x2a,0xba,0x00,0x00]
+icmp_quad_ballot r59h, nueq, u148, r53.discard
+# CHECK: icmp_quad_ballot r59h, nueq, u148, r53.discard{{ +}}// encoding: [0x32,0x6e,0xe8,0xa1,0x2e,0x91,0x00,0x00]
+icmp_quad_ballot r9h.cache, ugte, r78h.discard, u221h
+# CHECK: icmp_quad_ballot r9h.cache, ugte, r78h.discard, u221h{{ +}}// encoding: [0xb2,0x26,0xdd,0xb0,0x17,0x8a,0x00,0x20]
+icmp_quad_ballot r112h, slte, r101h.discard, u204h
+# CHECK: icmp_quad_ballot r112h, slte, r101h.discard, u204h{{ +}}// encoding: [0x32,0x42,0xcb,0x90,0x15,0xbe,0x00,0xc0]
+
+quad_and r101l.cache, r79h.cache
+# CHECK: quad_and r101l.cache, r79h.cache{{ +}}// encoding: [0xef,0x94,0x9f,0x00,0x00,0x38]
+quad_and r79.cache, r48_r49.discard
+# CHECK: quad_and r79.cache, r48_r49.discard{{ +}}// encoding: [0xef,0xbd,0xe0,0x03,0x00,0x24]
+quad_and r91l.cache, r54.cache
+# CHECK: quad_and r91l.cache, r54.cache{{ +}}// encoding: [0xef,0xec,0xac,0x02,0x00,0x24]
+quad_and r111, 55
+# CHECK: quad_and r111, 55{{ +}}// encoding: [0x6f,0xbd,0x37,0x00,0x00,0x30]
+
+quad_fadd r107h.cache, r126l.discard
+# CHECK: quad_fadd r107h.cache, r126l.discard{{ +}}// encoding: [0xef,0xae,0xfc,0x10,0x00,0x3c]
+quad_fadd r87, r21_r22
+# CHECK: quad_fadd r87, r21_r22{{ +}}// encoding: [0x6f,0xdd,0x6a,0x13,0x00,0x20]
+quad_fadd r71h, r48l
+# CHECK: quad_fadd r71h, r48l{{ +}}// encoding: [0x6f,0x9e,0x60,0x10,0x00,0x24]
+quad_fadd r63l.cache, r25l.cache
+# CHECK: quad_fadd r63l.cache, r25l.cache{{ +}}// encoding: [0xef,0xfc,0xb2,0x10,0x00,0x10]
+
+quad_fmul r57.cache, r42h.discard
+# CHECK: quad_fmul r57.cache, r42h.discard{{ +}}// encoding: [0xef,0xe5,0xd5,0x10,0x40,0x14]
+quad_fmul r113l.cache, r113l
+# CHECK: quad_fmul r113l.cache, r113l{{ +}}// encoding: [0xef,0xc4,0x62,0x10,0x40,0x3c]
+quad_fmul r40l, 188
+# CHECK: quad_fmul r40l, 188{{ +}}// encoding: [0x6f,0xa0,0x3c,0x10,0x40,0x18]
+quad_fmul r80.cache, u132
+# CHECK: quad_fmul r80.cache, u132{{ +}}// encoding: [0xef,0xc1,0xc8,0x11,0x40,0x20]
+
+quad_iadd r21l.cache, u0h
+# CHECK: quad_iadd r21l.cache, u0h{{ +}}// encoding: [0xef,0xd4,0x01,0x01,0x00,0x80]
+quad_iadd r42l.cache, r40h
+# CHECK: quad_iadd r42l.cache, r40h{{ +}}// encoding: [0xef,0xa8,0x51,0x00,0x00,0x94]
+quad_iadd r22h, r125l
+# CHECK: quad_iadd r22h, r125l{{ +}}// encoding: [0x6f,0xda,0x7a,0x00,0x00,0x8c]
+quad_iadd r46l, r17h.discard
+# CHECK: quad_iadd r46l, r17h.discard{{ +}}// encoding: [0x6f,0xb8,0xe3,0x00,0x00,0x90]
+
+quad_max.f r9, r98h.cache
+# CHECK: quad_max.f r9, r98h.cache{{ +}}// encoding: [0x6f,0xa5,0x85,0x10,0xc0,0x0c]
+quad_max.f r9l.cache, r1
+# CHECK: quad_max.f r9l.cache, r1{{ +}}// encoding: [0xef,0xa4,0x42,0x12,0xc0,0x00]
+quad_max.f r101h, u35h
+# CHECK: quad_max.f r101h, u35h{{ +}}// encoding: [0x6f,0x96,0x07,0x11,0xc0,0x34]
+quad_max.f r93, u97h
+# CHECK: quad_max.f r93, u97h{{ +}}// encoding: [0x6f,0xf5,0x03,0x11,0xc0,0x2c]
+
+quad_max.s r119l.cache, 56
+# CHECK: quad_max.s r119l.cache, 56{{ +}}// encoding: [0xef,0xdc,0x38,0x00,0xc0,0xb0]
+quad_max.s r82h, 145
+# CHECK: quad_max.s r82h, 145{{ +}}// encoding: [0x6f,0xca,0x11,0x00,0xc0,0xa8]
+quad_max.s r112h, r68_r69
+# CHECK: quad_max.s r112h, r68_r69{{ +}}// encoding: [0x6f,0xc2,0x48,0x03,0xc0,0xb8]
+quad_max.s r104, r62.cache
+# CHECK: quad_max.s r104, r62.cache{{ +}}// encoding: [0x6f,0xa1,0xbc,0x02,0xc0,0xb4]
+
+quad_max.u r27h, 139
+# CHECK: quad_max.u r27h, 139{{ +}}// encoding: [0x6f,0xee,0x0b,0x00,0xc0,0x89]
+quad_max.u r53, r71_r72
+# CHECK: quad_max.u r53, r71_r72{{ +}}// encoding: [0x6f,0xd5,0x4e,0x03,0xc0,0x99]
+quad_max.u r93h.cache, r121_r122.discard
+# CHECK: quad_max.u r93h.cache, r121_r122.discard{{ +}}// encoding: [0xef,0xf6,0xf2,0x03,0xc0,0xad]
+quad_max.u r81, r117.cache
+# CHECK: quad_max.u r81, r117.cache{{ +}}// encoding: [0x6f,0xc5,0xaa,0x02,0xc0,0xad]
+
+quad_min.f r53l.cache, u63
+# CHECK: quad_min.f r53l.cache, u63{{ +}}// encoding: [0xef,0xd4,0xbe,0x11,0x80,0x14]
+quad_min.f r97.cache, r105l.cache
+# CHECK: quad_min.f r97.cache, r105l.cache{{ +}}// encoding: [0xef,0x85,0x92,0x10,0x80,0x3c]
+quad_min.f r27h, 228
+# CHECK: quad_min.f r27h, 228{{ +}}// encoding: [0x6f,0xee,0x24,0x10,0x80,0x0c]
+quad_min.f r32.cache, u205h
+# CHECK: quad_min.f r32.cache, u205h{{ +}}// encoding: [0xef,0x81,0x5b,0x11,0x80,0x18]
+
+quad_min.s r98l.cache, r119h
+# CHECK: quad_min.s r98l.cache, r119h{{ +}}// encoding: [0xef,0x88,0x6f,0x00,0x80,0xbc]
+quad_min.s r30h.cache, r65l.cache
+# CHECK: quad_min.s r30h.cache, r65l.cache{{ +}}// encoding: [0xef,0xfa,0x82,0x00,0x80,0x88]
+quad_min.s r127.cache, 80
+# CHECK: quad_min.s r127.cache, 80{{ +}}// encoding: [0xef,0xfd,0x10,0x00,0x80,0xb4]
+quad_min.s r14.cache, 42
+# CHECK: quad_min.s r14.cache, 42{{ +}}// encoding: [0xef,0xb9,0x2a,0x00,0x80,0x80]
+
+quad_min.u r1h.cache, r105_r106.discard
+# CHECK: quad_min.u r1h.cache, r105_r106.discard{{ +}}// encoding: [0xef,0x86,0xd2,0x03,0x80,0x8d]
+quad_min.u r80h.cache, r108l.discard
+# CHECK: quad_min.u r80h.cache, r108l.discard{{ +}}// encoding: [0xef,0xc2,0xd8,0x00,0x80,0xad]
+quad_min.u r116h.cache, u13l
+# CHECK: quad_min.u r116h.cache, u13l{{ +}}// encoding: [0xef,0xd2,0x1a,0x01,0x80,0xb1]
+quad_min.u r13h.cache, r110_r111
+# CHECK: quad_min.u r13h.cache, r110_r111{{ +}}// encoding: [0xef,0xb6,0x5c,0x03,0x80,0x8d]
+
+quad_or r74h.cache, 68
+# CHECK: quad_or r74h.cache, 68{{ +}}// encoding: [0xef,0xaa,0x04,0x00,0x40,0x24]
+quad_or r70h.cache, r34l.cache
+# CHECK: quad_or r70h.cache, r34l.cache{{ +}}// encoding: [0xef,0x9a,0x84,0x00,0x40,0x24]
+quad_or r67.cache, r18
+# CHECK: quad_or r67.cache, r18{{ +}}// encoding: [0xef,0x8d,0x64,0x02,0x40,0x20]
+quad_or r81l, r75_r76.discard
+# CHECK: quad_or r81l, r75_r76.discard{{ +}}// encoding: [0x6f,0xc4,0xd6,0x03,0x40,0x28]
+
+quad_prefix_fadd r106.cache, r123l.cache
+# CHECK: quad_prefix_fadd r106.cache, r123l.cache{{ +}}// encoding: [0xef,0xa9,0xb6,0x50,0x00,0x3c]
+quad_prefix_fadd r116.cache, 80
+# CHECK: quad_prefix_fadd r116.cache, 80{{ +}}// encoding: [0xef,0xd1,0x10,0x50,0x00,0x34]
+quad_prefix_fadd r64l.cache, r75.cache
+# CHECK: quad_prefix_fadd r64l.cache, r75.cache{{ +}}// encoding: [0xef,0x80,0x96,0x52,0x00,0x28]
+quad_prefix_fadd r32l, u25
+# CHECK: quad_prefix_fadd r32l, u25{{ +}}// encoding: [0x6f,0x80,0xb2,0x51,0x00,0x10]
+
+quad_prefix_fmul r1l.cache, 3
+# CHECK: quad_prefix_fmul r1l.cache, 3{{ +}}// encoding: [0xef,0x84,0x03,0x50,0x40,0x00]
+quad_prefix_fmul r73h, u95l
+# CHECK: quad_prefix_fmul r73h, u95l{{ +}}// encoding: [0x6f,0xa6,0x3e,0x51,0x40,0x28]
+quad_prefix_fmul r12.cache, r30l.discard
+# CHECK: quad_prefix_fmul r12.cache, r30l.discard{{ +}}// encoding: [0xef,0xb1,0xfc,0x50,0x40,0x00]
+quad_prefix_fmul r26.cache, u238l
+# CHECK: quad_prefix_fmul r26.cache, u238l{{ +}}// encoding: [0xef,0xe9,0x5c,0x51,0x40,0x0c]
+
+quad_prefix_iadd r30l.cache, 14
+# CHECK: quad_prefix_iadd r30l.cache, 14{{ +}}// encoding: [0xef,0xf8,0x0e,0x40,0x00,0x80]
+quad_prefix_iadd r64.cache, r27l
+# CHECK: quad_prefix_iadd r64.cache, r27l{{ +}}// encoding: [0xef,0x81,0x76,0x40,0x00,0xa0]
+quad_prefix_iadd r71.cache, u212l
+# CHECK: quad_prefix_iadd r71.cache, u212l{{ +}}// encoding: [0xef,0x9d,0x68,0x41,0x00,0xa8]
+quad_prefix_iadd r65l.cache, r19l.cache
+# CHECK: quad_prefix_iadd r65l.cache, r19l.cache{{ +}}// encoding: [0xef,0x84,0xa6,0x40,0x00,0xa0]
+
+quad_shuffle r121.cache, r79_r80.cache, u22l
+# CHECK: quad_shuffle r121.cache, r79_r80.cache, u22l{{ +}}// encoding: [0xef,0x65,0x9e,0xc3,0x12,0x38]
+quad_shuffle r80l.cache, r15_r16.discard, 184
+# CHECK: quad_shuffle r80l.cache, r15_r16.discard, 184{{ +}}// encoding: [0xef,0x40,0xde,0x83,0x03,0x22]
+quad_shuffle r35, r54l, r82.discard
+# CHECK: quad_shuffle r35, r54l, r82.discard{{ +}}// encoding: [0x6f,0x0d,0x6c,0x40,0x2e,0x16]
+quad_shuffle r47l, r102, 86
+# CHECK: quad_shuffle r47l, r102, 86{{ +}}// encoding: [0x6f,0x3c,0x4c,0x62,0x01,0x1d]
+
+quad_shuffle_down r100h.cache, u8, r44
+# CHECK: quad_shuffle_down r100h.cache, u8, r44{{ +}}// encoding: [0xef,0x12,0x90,0x81,0xe5,0x31]
+quad_shuffle_down r86.cache, r101_r102.cache, r84h.cache
+# CHECK: quad_shuffle_down r86.cache, r101_r102.cache, r84h.cache{{ +}}// encoding: [0xef,0x59,0x8a,0x93,0xca,0x2e]
+quad_shuffle_down r51, r71_r72.discard, u117h
+# CHECK: quad_shuffle_down r51, r71_r72.discard, u117h{{ +}}// encoding: [0x6f,0x4d,0xce,0xb3,0xd2,0x1b]
+quad_shuffle_down r23l, r80_r81.cache, u34l
+# CHECK: quad_shuffle_down r23l, r80_r81.cache, u34l{{ +}}// encoding: [0x6f,0x5c,0xa0,0x43,0xd0,0x09]
+
+quad_shuffle_rotate_up r11l, r50l.discard, 252
+# CHECK: quad_shuffle_rotate_up r11l, r50l.discard, 252{{ +}}// encoding: [0x6f,0x2c,0xe4,0xc0,0x83,0x87]
+quad_shuffle_rotate_up r87.cache, u48, u2h
+# CHECK: quad_shuffle_rotate_up r87.cache, u48, u2h{{ +}}// encoding: [0xef,0x5d,0xa0,0x51,0x90,0xa4]
+quad_shuffle_rotate_up r6.cache, 94, u114h
+# CHECK: quad_shuffle_rotate_up r6.cache, 94, u114h{{ +}}// encoding: [0xef,0x19,0x1e,0x50,0x92,0x87]
+quad_shuffle_rotate_up r81, r17h, u224
+# CHECK: quad_shuffle_rotate_up r81, r17h, u224{{ +}}// encoding: [0x6f,0x45,0x63,0x00,0x9c,0xa3]
+
+quad_shuffle_up r65l, r95l.discard, r96_r97.discard
+# CHECK: quad_shuffle_up r65l, r95l.discard, r96_r97.discard{{ +}}// encoding: [0x6f,0x04,0xfe,0x00,0xbc,0x2b]
+quad_shuffle_up r17.cache, r110l.cache, r5h.discard
+# CHECK: quad_shuffle_up r17.cache, r110l.cache, r5h.discard{{ +}}// encoding: [0xef,0x45,0x9c,0xb0,0x8c,0x0c]
+quad_shuffle_up r86h, r55l.discard, r112
+# CHECK: quad_shuffle_up r86h, r55l.discard, r112{{ +}}// encoding: [0x6f,0x5a,0xee,0x00,0xa6,0x27]
+quad_shuffle_up r88h, u177h, 62
+# CHECK: quad_shuffle_up r88h, u177h, 62{{ +}}// encoding: [0x6f,0x62,0x63,0xe1,0x83,0x24]
+
+quad_shuffle_xor r45, r21_r22, r92_r93
+# CHECK: quad_shuffle_xor r45, r21_r22, r92_r93{{ +}}// encoding: [0x6f,0x35,0x6a,0x83,0x77,0x12]
+quad_shuffle_xor r50h.cache, 216, r50_r51.cache
+# CHECK: quad_shuffle_xor r50h.cache, 216, r50_r51.cache{{ +}}// encoding: [0xef,0x4a,0x18,0x40,0x7a,0x1d]
+quad_shuffle_xor r102.cache, r95l, r23h.cache
+# CHECK: quad_shuffle_xor r102.cache, r95l, r23h.cache{{ +}}// encoding: [0xef,0x19,0x7e,0xf0,0x4a,0x38]
+quad_shuffle_xor r52h, r29_r30.cache, r107_r108
+# CHECK: quad_shuffle_xor r52h, r29_r30.cache, r107_r108{{ +}}// encoding: [0x6f,0x52,0xba,0x63,0x75,0x13]
+
+quad_xor r13l, u202l
+# CHECK: quad_xor r13l, u202l{{ +}}// encoding: [0x6f,0xb4,0x54,0x01,0x80,0x08]
+quad_xor r104h.cache, r106.discard
+# CHECK: quad_xor r104h.cache, r106.discard{{ +}}// encoding: [0xef,0xa2,0xd4,0x02,0x80,0x3c]
+quad_xor r18l, r25l.cache
+# CHECK: quad_xor r18l, r25l.cache{{ +}}// encoding: [0x6f,0xc8,0xb2,0x00,0x80,0x00]
+quad_xor r103.cache, 27
+# CHECK: quad_xor r103.cache, 27{{ +}}// encoding: [0xef,0x9d,0x1b,0x00,0x80,0x30]
+
+simd_and r31.cache, u120h
+# CHECK: simd_and r31.cache, u120h{{ +}}// encoding: [0xef,0xfd,0x31,0x21,0x00,0x0c]
+simd_and r125h.cache, u236
+# CHECK: simd_and r125h.cache, u236{{ +}}// encoding: [0xef,0xf6,0xd8,0x21,0x00,0x3c]
+simd_and r61, u83l
+# CHECK: simd_and r61, u83l{{ +}}// encoding: [0x6f,0xf5,0x26,0x21,0x00,0x18]
+simd_and r114, r41h.cache
+# CHECK: simd_and r114, r41h.cache{{ +}}// encoding: [0x6f,0xc9,0x93,0x20,0x00,0x34]
+
+simd_fadd r112l.cache, u77h
+# CHECK: simd_fadd r112l.cache, u77h{{ +}}// encoding: [0xef,0xc0,0x1b,0x31,0x00,0x38]
+simd_fadd r21.cache, r72_r73.cache
+# CHECK: simd_fadd r21.cache, r72_r73.cache{{ +}}// encoding: [0xef,0xd5,0x90,0x33,0x00,0x08]
+simd_fadd r50h.cache, r90.cache
+# CHECK: simd_fadd r50h.cache, r90.cache{{ +}}// encoding: [0xef,0xca,0xb4,0x32,0x00,0x18]
+simd_fadd r3l, r30l.cache
+# CHECK: simd_fadd r3l, r30l.cache{{ +}}// encoding: [0x6f,0x8c,0xbc,0x30,0x00,0x00]
+
+simd_fmul r82h, u244h
+# CHECK: simd_fmul r82h, u244h{{ +}}// encoding: [0x6f,0xca,0x69,0x31,0x40,0x2c]
+simd_fmul r62l.cache, 121
+# CHECK: simd_fmul r62l.cache, 121{{ +}}// encoding: [0xef,0xf8,0x39,0x30,0x40,0x14]
+simd_fmul r76l, u155l
+# CHECK: simd_fmul r76l, u155l{{ +}}// encoding: [0x6f,0xb0,0x76,0x31,0x40,0x20]
+simd_fmul r109.cache, 244
+# CHECK: simd_fmul r109.cache, 244{{ +}}// encoding: [0xef,0xb5,0x34,0x30,0x40,0x3c]
+
+simd_iadd r46l, r33.discard
+# CHECK: simd_iadd r46l, r33.discard{{ +}}// encoding: [0x6f,0xb8,0xc2,0x22,0x00,0x94]
+simd_iadd r45, u129l
+# CHECK: simd_iadd r45, u129l{{ +}}// encoding: [0x6f,0xb5,0x42,0x21,0x00,0x90]
+simd_iadd r123l, u241l
+# CHECK: simd_iadd r123l, u241l{{ +}}// encoding: [0x6f,0xec,0x62,0x21,0x00,0xbc]
+simd_iadd r86l, u153l
+# CHECK: simd_iadd r86l, u153l{{ +}}// encoding: [0x6f,0xd8,0x72,0x21,0x00,0xa0]
+
+simd_max.f r55h, u176h
+# CHECK: simd_max.f r55h, u176h{{ +}}// encoding: [0x6f,0xde,0x61,0x31,0xc0,0x14]
+simd_max.f r38l.cache, u201l
+# CHECK: simd_max.f r38l.cache, u201l{{ +}}// encoding: [0xef,0x98,0x52,0x31,0xc0,0x18]
+simd_max.f r23l.cache, 23
+# CHECK: simd_max.f r23l.cache, 23{{ +}}// encoding: [0xef,0xdc,0x17,0x30,0xc0,0x00]
+simd_max.f r40l, r24_r25.cache
+# CHECK: simd_max.f r40l, r24_r25.cache{{ +}}// encoding: [0x6f,0xa0,0xb0,0x33,0xc0,0x10]
+
+simd_max.s r100h, r51h.cache
+# CHECK: simd_max.s r100h, r51h.cache{{ +}}// encoding: [0x6f,0x92,0xa7,0x20,0xc0,0xb4]
+simd_max.s r74h.cache, r37.cache
+# CHECK: simd_max.s r74h.cache, r37.cache{{ +}}// encoding: [0xef,0xaa,0x8a,0x22,0xc0,0xa4]
+simd_max.s r65h, u110
+# CHECK: simd_max.s r65h, u110{{ +}}// encoding: [0x6f,0x86,0x9c,0x21,0xc0,0xac]
+simd_max.s r15h.cache, u103l
+# CHECK: simd_max.s r15h.cache, u103l{{ +}}// encoding: [0xef,0xbe,0x0e,0x21,0xc0,0x8c]
+
+simd_max.u r74, r96h.cache
+# CHECK: simd_max.u r74, r96h.cache{{ +}}// encoding: [0x6f,0xa9,0x81,0x20,0xc0,0xad]
+simd_max.u r22l.cache, 183
+# CHECK: simd_max.u r22l.cache, 183{{ +}}// encoding: [0xef,0xd8,0x37,0x20,0xc0,0x89]
+simd_max.u r102.cache, r107h.cache
+# CHECK: simd_max.u r102.cache, r107h.cache{{ +}}// encoding: [0xef,0x99,0x97,0x20,0xc0,0xbd]
+simd_max.u r71, r55
+# CHECK: simd_max.u r71, r55{{ +}}// encoding: [0x6f,0x9d,0x6e,0x22,0xc0,0xa5]
+
+simd_min.f r5h, u12h
+# CHECK: simd_min.f r5h, u12h{{ +}}// encoding: [0x6f,0x96,0x19,0x31,0x80,0x00]
+simd_min.f r100l, u253h
+# CHECK: simd_min.f r100l, u253h{{ +}}// encoding: [0x6f,0x90,0x7b,0x31,0x80,0x3c]
+simd_min.f r73l.cache, r113_r114.cache
+# CHECK: simd_min.f r73l.cache, r113_r114.cache{{ +}}// encoding: [0xef,0xa4,0xa2,0x33,0x80,0x2c]
+simd_min.f r71.cache, r55_r56.cache
+# CHECK: simd_min.f r71.cache, r55_r56.cache{{ +}}// encoding: [0xef,0x9d,0xae,0x33,0x80,0x24]
+
+simd_min.s r87h, r67_r68.discard
+# CHECK: simd_min.s r87h, r67_r68.discard{{ +}}// encoding: [0x6f,0xde,0xc6,0x23,0x80,0xa8]
+simd_min.s r66l.cache, u140h
+# CHECK: simd_min.s r66l.cache, u140h{{ +}}// encoding: [0xef,0x88,0x59,0x21,0x80,0xa0]
+simd_min.s r32h, u53l
+# CHECK: simd_min.s r32h, u53l{{ +}}// encoding: [0x6f,0x82,0x2a,0x21,0x80,0x94]
+simd_min.s r24h.cache, r99h
+# CHECK: simd_min.s r24h.cache, r99h{{ +}}// encoding: [0xef,0xe2,0x47,0x20,0x80,0x8c]
+
+simd_min.u r6.cache, r100l
+# CHECK: simd_min.u r6.cache, r100l{{ +}}// encoding: [0xef,0x99,0x48,0x20,0x80,0x8d]
+simd_min.u r9h, r13l
+# CHECK: simd_min.u r9h, r13l{{ +}}// encoding: [0x6f,0xa6,0x5a,0x20,0x80,0x81]
+simd_min.u r85l, r80l.discard
+# CHECK: simd_min.u r85l, r80l.discard{{ +}}// encoding: [0x6f,0xd4,0xe0,0x20,0x80,0xa9]
+simd_min.u r106, u53l
+# CHECK: simd_min.u r106, u53l{{ +}}// encoding: [0x6f,0xa9,0x2a,0x21,0x80,0xb5]
+
+simd_or r49.cache, r81
+# CHECK: simd_or r49.cache, r81{{ +}}// encoding: [0xef,0xc5,0x62,0x22,0x40,0x18]
+simd_or r56, u251l
+# CHECK: simd_or r56, u251l{{ +}}// encoding: [0x6f,0xe1,0x76,0x21,0x40,0x1c]
+simd_or r62l, r114h.cache
+# CHECK: simd_or r62l, r114h.cache{{ +}}// encoding: [0x6f,0xf8,0xa5,0x20,0x40,0x1c]
+simd_or r127l, u26
+# CHECK: simd_or r127l, u26{{ +}}// encoding: [0x6f,0xfc,0xb4,0x21,0x40,0x30]
+
+simd_prefix_and r30h, r65l.discard
+# CHECK: simd_prefix_and r30h, r65l.discard{{ +}}// encoding: [0x6f,0xfa,0xc2,0x60,0x00,0x08]
+simd_prefix_and r124h, r40h.cache
+# CHECK: simd_prefix_and r124h, r40h.cache{{ +}}// encoding: [0x6f,0xf2,0x91,0x60,0x00,0x34]
+simd_prefix_and r12l, r82_r83.cache
+# CHECK: simd_prefix_and r12l, r82_r83.cache{{ +}}// encoding: [0x6f,0xb0,0xa4,0x63,0x00,0x08]
+simd_prefix_and r40, r38l.discard
+# CHECK: simd_prefix_and r40, r38l.discard{{ +}}// encoding: [0x6f,0xa1,0xcc,0x60,0x00,0x14]
+
+simd_prefix_fadd r123l, r119l.discard
+# CHECK: simd_prefix_fadd r123l, r119l.discard{{ +}}// encoding: [0x6f,0xec,0xee,0x70,0x00,0x3c]
+simd_prefix_fadd r126.cache, r45_r46
+# CHECK: simd_prefix_fadd r126.cache, r45_r46{{ +}}// encoding: [0xef,0xf9,0x5a,0x73,0x00,0x34]
+simd_prefix_fadd r83, r51l.discard
+# CHECK: simd_prefix_fadd r83, r51l.discard{{ +}}// encoding: [0x6f,0xcd,0xe6,0x70,0x00,0x24]
+simd_prefix_fadd r23.cache, u68l
+# CHECK: simd_prefix_fadd r23.cache, u68l{{ +}}// encoding: [0xef,0xdd,0x08,0x71,0x00,0x08]
+
+simd_prefix_fmul r49, r47h.discard
+# CHECK: simd_prefix_fmul r49, r47h.discard{{ +}}// encoding: [0x6f,0xc5,0xdf,0x70,0x40,0x14]
+simd_prefix_fmul r61h.cache, 221
+# CHECK: simd_prefix_fmul r61h.cache, 221{{ +}}// encoding: [0xef,0xf6,0x1d,0x70,0x40,0x1c]
+simd_prefix_fmul r66, r62_r63.cache
+# CHECK: simd_prefix_fmul r66, r62_r63.cache{{ +}}// encoding: [0x6f,0x89,0xbc,0x73,0x40,0x24]
+simd_prefix_fmul r33, r1
+# CHECK: simd_prefix_fmul r33, r1{{ +}}// encoding: [0x6f,0x85,0x42,0x72,0x40,0x10]
+
+simd_prefix_iadd r95l.cache, r85_r86.cache
+# CHECK: simd_prefix_iadd r95l.cache, r85_r86.cache{{ +}}// encoding: [0xef,0xfc,0xaa,0x63,0x00,0xa8]
+simd_prefix_iadd r39l, r47l.cache
+# CHECK: simd_prefix_iadd r39l, r47l.cache{{ +}}// encoding: [0x6f,0x9c,0x9e,0x60,0x00,0x94]
+simd_prefix_iadd r7h, u155
+# CHECK: simd_prefix_iadd r7h, u155{{ +}}// encoding: [0x6f,0x9e,0xf6,0x61,0x00,0x80]
+simd_prefix_iadd r107.cache, u63l
+# CHECK: simd_prefix_iadd r107.cache, u63l{{ +}}// encoding: [0xef,0xad,0x3e,0x61,0x00,0xb4]
+
+simd_prefix_max.s r87h.cache, r82l.cache
+# CHECK: simd_prefix_max.s r87h.cache, r82l.cache{{ +}}// encoding: [0xef,0xde,0xa4,0x60,0xc0,0xa8]
+simd_prefix_max.s r38l, r107
+# CHECK: simd_prefix_max.s r38l, r107{{ +}}// encoding: [0x6f,0x98,0x56,0x62,0xc0,0x9c]
+simd_prefix_max.s r121l, u109
+# CHECK: simd_prefix_max.s r121l, u109{{ +}}// encoding: [0x6f,0xe4,0x9a,0x61,0xc0,0xbc]
+simd_prefix_max.s r63h, r28l.cache
+# CHECK: simd_prefix_max.s r63h, r28l.cache{{ +}}// encoding: [0x6f,0xfe,0xb8,0x60,0xc0,0x90]
+
+simd_prefix_max.u r114.cache, u220
+# CHECK: simd_prefix_max.u r114.cache, u220{{ +}}// encoding: [0xef,0xc9,0xf8,0x61,0xc0,0xb9]
+simd_prefix_max.u r32.cache, u154h
+# CHECK: simd_prefix_max.u r32.cache, u154h{{ +}}// encoding: [0xef,0x81,0x75,0x61,0xc0,0x91]
+simd_prefix_max.u r123h, r115
+# CHECK: simd_prefix_max.u r123h, r115{{ +}}// encoding: [0x6f,0xee,0x66,0x62,0xc0,0xbd]
+simd_prefix_max.u r52h.cache, r95l
+# CHECK: simd_prefix_max.u r52h.cache, r95l{{ +}}// encoding: [0xef,0xd2,0x7e,0x60,0xc0,0x99]
+
+simd_prefix_min.s r1l.cache, r90h.cache
+# CHECK: simd_prefix_min.s r1l.cache, r90h.cache{{ +}}// encoding: [0xef,0x84,0xb5,0x60,0x80,0x88]
+simd_prefix_min.s r67l, r12h.cache
+# CHECK: simd_prefix_min.s r67l, r12h.cache{{ +}}// encoding: [0x6f,0x8c,0x99,0x60,0x80,0xa0]
+simd_prefix_min.s r15, r30h.discard
+# CHECK: simd_prefix_min.s r15, r30h.discard{{ +}}// encoding: [0x6f,0xbd,0xfd,0x60,0x80,0x80]
+simd_prefix_min.s r1, r120l.discard
+# CHECK: simd_prefix_min.s r1, r120l.discard{{ +}}// encoding: [0x6f,0x85,0xf0,0x60,0x80,0x8c]
+
+simd_prefix_min.u r97h, u242
+# CHECK: simd_prefix_min.u r97h, u242{{ +}}// encoding: [0x6f,0x86,0xe4,0x61,0x80,0xbd]
+simd_prefix_min.u r67.cache, 144
+# CHECK: simd_prefix_min.u r67.cache, 144{{ +}}// encoding: [0xef,0x8d,0x10,0x60,0x80,0xa9]
+simd_prefix_min.u r74l.cache, u140h
+# CHECK: simd_prefix_min.u r74l.cache, u140h{{ +}}// encoding: [0xef,0xa8,0x59,0x61,0x80,0xa1]
+simd_prefix_min.u r23h.cache, 232
+# CHECK: simd_prefix_min.u r23h.cache, 232{{ +}}// encoding: [0xef,0xde,0x28,0x60,0x80,0x8d]
+
+simd_prefix_or r77l.cache, u92
+# CHECK: simd_prefix_or r77l.cache, u92{{ +}}// encoding: [0xef,0xb4,0xb8,0x61,0x40,0x28]
+simd_prefix_or r0l, u218l
+# CHECK: simd_prefix_or r0l, u218l{{ +}}// encoding: [0x6f,0x80,0x74,0x61,0x40,0x08]
+simd_prefix_or r49h, r26h
+# CHECK: simd_prefix_or r49h, r26h{{ +}}// encoding: [0x6f,0xc6,0x75,0x60,0x40,0x10]
+simd_prefix_or r13h.cache, 206
+# CHECK: simd_prefix_or r13h.cache, 206{{ +}}// encoding: [0xef,0xb6,0x0e,0x60,0x40,0x0c]
+
+simd_prefix_xor r40h.cache, u94l
+# CHECK: simd_prefix_xor r40h.cache, u94l{{ +}}// encoding: [0xef,0xa2,0x3c,0x61,0x80,0x18]
+simd_prefix_xor r81, u144h
+# CHECK: simd_prefix_xor r81, u144h{{ +}}// encoding: [0x6f,0xc5,0x61,0x61,0x80,0x20]
+simd_prefix_xor r51l.cache, 88
+# CHECK: simd_prefix_xor r51l.cache, 88{{ +}}// encoding: [0xef,0xcc,0x18,0x60,0x80,0x14]
+simd_prefix_xor r61h, u188h
+# CHECK: simd_prefix_xor r61h, u188h{{ +}}// encoding: [0x6f,0xf6,0x79,0x61,0x80,0x14]
+
+simd_shuffle r59l.cache, r101h, r12l.cache
+# CHECK: simd_shuffle r59l.cache, r101h, r12l.cache{{ +}}// encoding: [0xef,0x6c,0x4b,0x84,0x09,0x1c]
+simd_shuffle r3l, u66h, r97l.discard
+# CHECK: simd_shuffle r3l, u66h, r97l.discard{{ +}}// encoding: [0x6f,0x0c,0x05,0x25,0x0c,0x0b]
+simd_shuffle r29, r62l, r73l.discard
+# CHECK: simd_shuffle r29, r62l, r73l.discard{{ +}}// encoding: [0x6f,0x75,0x7c,0x24,0x0d,0x06]
+simd_shuffle r100l, 206, r90h.cache
+# CHECK: simd_shuffle r100l, 206, r90h.cache{{ +}}// encoding: [0x6f,0x10,0x0e,0x54,0x0b,0x3e]
+
+simd_shuffle_down r103, r118l, u27h
+# CHECK: simd_shuffle_down r103, r118l, u27h{{ +}}// encoding: [0x6f,0x1d,0x6c,0x74,0xd3,0x3c]
+simd_shuffle_down r123h.cache, 215, u148h
+# CHECK: simd_shuffle_down r123h.cache, 215, u148h{{ +}}// encoding: [0xef,0x6e,0x17,0x94,0xd6,0x3c]
+simd_shuffle_down r16.cache, r96, 76
+# CHECK: simd_shuffle_down r16.cache, r96, 76{{ +}}// encoding: [0xef,0x41,0x40,0xc6,0xc0,0x0d]
+simd_shuffle_down r59.cache, r41h.discard, u214h
+# CHECK: simd_shuffle_down r59.cache, r41h.discard, u214h{{ +}}// encoding: [0xef,0x6d,0xd3,0xd4,0xd6,0x16]
+
+simd_shuffle_rotate_up r106h, r83.discard, u179l
+# CHECK: simd_shuffle_rotate_up r106h, r83.discard, u179l{{ +}}// encoding: [0x6f,0x2a,0xe6,0x66,0x96,0xb9]
+simd_shuffle_rotate_up r10l, r30.discard, r33h.discard
+# CHECK: simd_shuffle_rotate_up r10l, r30.discard, r33h.discard{{ +}}// encoding: [0x6f,0x28,0xfc,0x36,0x8c,0x81]
+simd_shuffle_rotate_up r43h.cache, r63h.cache, r65l.discard
+# CHECK: simd_shuffle_rotate_up r43h.cache, r63h.cache, r65l.discard{{ +}}// encoding: [0xef,0x2e,0xbf,0x24,0x8c,0x96]
+simd_shuffle_rotate_up r31h, u229h, u200
+# CHECK: simd_shuffle_rotate_up r31h, u229h, u200{{ +}}// encoding: [0x6f,0x7e,0x4b,0x05,0x9d,0x8e]
+
+simd_shuffle_up r102, 149, r123h.cache
+# CHECK: simd_shuffle_up r102, 149, r123h.cache{{ +}}// encoding: [0x6f,0x19,0x15,0x74,0x8b,0x3b]
+simd_shuffle_up r103h.cache, u116l, u236l
+# CHECK: simd_shuffle_up r103h.cache, u116l, u236l{{ +}}// encoding: [0xef,0x1e,0x28,0x85,0x95,0x3f]
+simd_shuffle_up r95h, u249h, r71l.discard
+# CHECK: simd_shuffle_up r95h, u249h, r71l.discard{{ +}}// encoding: [0x6f,0x7e,0x73,0xe5,0x8c,0x2e]
+simd_shuffle_up r5h.cache, r117h, r88l.discard
+# CHECK: simd_shuffle_up r5h.cache, r117h, r88l.discard{{ +}}// encoding: [0xef,0x16,0x6b,0x04,0x8f,0x0e]
+
+simd_shuffle_xor r101.cache, u248l, r121l.discard
+# CHECK: simd_shuffle_xor r101.cache, u248l, r121l.discard{{ +}}// encoding: [0xef,0x15,0x70,0x25,0x4f,0x3f]
+simd_shuffle_xor r112h.cache, r80.discard, u7l
+# CHECK: simd_shuffle_xor r112h.cache, r80.discard, u7l{{ +}}// encoding: [0xef,0x42,0xe0,0xe6,0x50,0x38]
+simd_shuffle_xor r39h, u93l, u22
+# CHECK: simd_shuffle_xor r39h, u93l, u22{{ +}}// encoding: [0x6f,0x1e,0x3a,0xc5,0x5a,0x18]
+simd_shuffle_xor r38l, r86l, u40l
+# CHECK: simd_shuffle_xor r38l, r86l, u40l{{ +}}// encoding: [0x6f,0x18,0x6c,0x04,0x51,0x19]
+
+simd_xor r85l.cache, u161
+# CHECK: simd_xor r85l.cache, u161{{ +}}// encoding: [0xef,0xd4,0xc2,0x21,0x80,0x24]
+simd_xor r67l.cache, r117.discard
+# CHECK: simd_xor r67l.cache, r117.discard{{ +}}// encoding: [0xef,0x8c,0xea,0x22,0x80,0x2c]
+simd_xor r81.cache, r87.discard
+# CHECK: simd_xor r81.cache, r87.discard{{ +}}// encoding: [0xef,0xc5,0xee,0x22,0x80,0x28]
+simd_xor r41.cache, r99h
+# CHECK: simd_xor r41.cache, r99h{{ +}}// encoding: [0xef,0xa5,0x47,0x20,0x80,0x1c]
+
diff --git a/llvm/test/MC/AGX/lit.local.cfg b/llvm/test/MC/AGX/lit.local.cfg
new file mode 100644
index 0000000000000..e7054032f0deb
--- /dev/null
+++ b/llvm/test/MC/AGX/lit.local.cfg
@@ -0,0 +1,2 @@
+if not "AGX" in config.root.targets:
+ config.unsupported = True
diff --git a/llvm/test/MC/Disassembler/AGX/g13-alu.txt b/llvm/test/MC/Disassembler/AGX/g13-alu.txt
new file mode 100644
index 0000000000000..ae8b64ac7c5df
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AGX/g13-alu.txt
@@ -0,0 +1,176 @@
+# RUN: llvm-mc -triple=agx -mcpu=agx-g13g -disassemble -show-encoding %s | FileCheck %s
+
+# Encodings cross-checked against the applegpu reference disassembler
+# (dougallj/applegpu).
+
+0x2e 0xae 0xb6 0x84 0x05 0x00 0x00 0x0b
+# CHECK: asr r11h, r91l.cache, r108l{{ +}}// encoding: [0x2e,0xae,0xb6,0x84,0x05,0x00,0x00,0x0b]
+0xae 0xb2 0x44 0x75 0x0e 0x00 0x00 0x36
+# CHECK: asr r108h.cache, u162l, r83h.discard{{ +}}// encoding: [0xae,0xb2,0x44,0x75,0x0e,0x00,0x00,0x36]
+0x2e 0xea 0xe8 0xa5 0x1a 0x00 0x00 0x28
+# CHECK: asr r90h, u212, u21{{ +}}// encoding: [0x2e,0xea,0xe8,0xa5,0x1a,0x00,0x00,0x28]
+0xae 0xe8 0x2a 0x94 0x12 0x00 0x00 0x16
+# CHECK: asr r58l.cache, 106, u84h{{ +}}// encoding: [0xae,0xe8,0x2a,0x94,0x12,0x00,0x00,0x16]
+
+0x2e 0xe6 0x74 0xfe 0x10 0x00 0x00 0x30
+# CHECK: asrh r121h, r26, u7h{{ +}}// encoding: [0x2e,0xe6,0x74,0xfe,0x10,0x00,0x00,0x30]
+0xae 0x95 0xaa 0xac 0x0c 0x00 0x00 0x3c
+# CHECK: asrh r101.cache, r117l.cache, r5l.discard{{ +}}// encoding: [0xae,0x95,0xaa,0xac,0x0c,0x00,0x00,0x3c]
+0xae 0x91 0x43 0x7c 0x10 0x00 0x00 0x2d
+# CHECK: asrh r68.cache, r97h, u35h{{ +}}// encoding: [0xae,0x91,0x43,0x7c,0x10,0x00,0x00,0x2d]
+0x2e 0xe6 0x86 0xcd 0x0c 0x00 0x00 0x25
+# CHECK: asrh r89h, u35, r38l.discard{{ +}}// encoding: [0x2e,0xe6,0x86,0xcd,0x0c,0x00,0x00,0x25]
+
+0x2e 0xf4 0x4a 0xf0 0xcd 0x0a 0x4d 0x98
+# CHECK: bfeil r61l, r69l, r15h.discard, u37l, mask 0x7FFFFFFF{{ +}}// encoding: [0x2e,0xf4,0x4a,0xf0,0xcd,0x0a,0x4d,0x98]
+0x2e 0xf6 0xda 0x30 0x8e 0x38 0x40 0xb2
+# CHECK: bfeil r125h, r13l.discard, r81h.discard, 120, mask 0x3FFFF{{ +}}// encoding: [0x2e,0xf6,0xda,0x30,0x8e,0x38,0x40,0xb2]
+0xae 0x8e 0x6a 0xb0 0x4e 0xae 0x0a 0xa1
+# CHECK: bfeil r67h.cache, r21l, r53h.discard, r23.cache, mask 0x1FFFFFF{{ +}}// encoding: [0xae,0x8e,0x6a,0xb0,0x4e,0xae,0x0a,0xa1]
+0x2e 0x99 0xaa 0x00 0x54 0xb0 0x80 0xae
+# CHECK: bfeil r70, r117l.cache, u192l, r88l.cache, mask 0x1FFFF{{ +}}// encoding: [0x2e,0x99,0xaa,0x00,0x54,0xb0,0x80,0xae]
+
+0x2e 0x6a 0x5b 0xe0 0x96 0x53 0x09 0x98
+# CHECK: bfi r58h, r77h, u151l, u137h, mask 0x3FFFFFF{{ +}}// encoding: [0x2e,0x6a,0x5b,0xe0,0x96,0x53,0x09,0x98]
+0xae 0x2a 0x14 0x60 0xa9 0x73 0x89 0xb2
+# CHECK: bfi r106h.cache, 20, r75.cache, u217h, mask 0x3FFFFFF{{ +}}// encoding: [0xae,0x2a,0x14,0x60,0xa9,0x73,0x89,0xb2]
+0xae 0x6a 0xfe 0xb1 0xc6 0xe0 0xcc 0x10
+# CHECK: bfi r58h.cache, u159, r21h, r112l.discard, mask 0x7FFF{{ +}}// encoding: [0xae,0x6a,0xfe,0xb1,0xc6,0xe0,0xcc,0x10]
+0x2e 0x56 0x6c 0x81 0x24 0xca 0x46 0x8b
+# CHECK: bfi r21h, u214l, r100, r37.discard, mask 0xFFFFF{{ +}}// encoding: [0x2e,0x56,0x6c,0x81,0x24,0xca,0x46,0x8b]
+
+0xfe 0x40 0x78 0x82 0x4c 0x08
+# CHECK: bitop 0010, r16l.cache, r92, r4l.discard{{ +}}// encoding: [0xfe,0x40,0x78,0x82,0x4c,0x08]
+0xfe 0x0e 0xbe 0xac 0x86 0x16
+# CHECK: bitop 1101, r35h.cache, r63l.cache, r85l{{ +}}// encoding: [0xfe,0x0e,0xbe,0xac,0x86,0x16]
+0x7e 0x64 0x43 0xc4 0xe6 0x20
+# CHECK: bitop 1011, r89l, r1h, r22{{ +}}// encoding: [0x7e,0x64,0x43,0xc4,0xe6,0x20]
+0xfe 0x74 0xe0 0x72 0x40 0x15
+# CHECK: bitop 0010, r61l.cache, r48.discard, 71{{ +}}// encoding: [0xfe,0x74,0xe0,0x72,0x40,0x15]
+
+0x3e 0x51 0xc0 0x06 0x00 0x14
+# CHECK: bitrev r52, r32.discard{{ +}}// encoding: [0x3e,0x51,0xc0,0x06,0x00,0x14]
+0x3e 0x01 0x29 0x05 0x00 0x08
+# CHECK: bitrev r0, u84h{{ +}}// encoding: [0x3e,0x01,0x29,0x05,0x00,0x08]
+0xbe 0x10 0x03 0x05 0x00 0x24
+# CHECK: bitrev r68l.cache, u33h{{ +}}// encoding: [0xbe,0x10,0x03,0x05,0x00,0x24]
+0xbe 0x79 0x36 0x04 0x00 0x18
+# CHECK: bitrev r62.cache, 182{{ +}}// encoding: [0xbe,0x79,0x36,0x04,0x00,0x18]
+
+0xbe 0xd4 0x27 0x0c 0x1e 0x31
+# CHECK: convert 39, r117l.cache, u176, 3{{ +}}// encoding: [0xbe,0xd4,0x27,0x0c,0x1e,0x31]
+0x3e 0xb4 0x01 0x20 0x13 0x03
+# CHECK: convert s8_to_f, r13l, u121l, rtz{{ +}}// encoding: [0x3e,0xb4,0x01,0x20,0x13,0x03]
+0xbe 0xaa 0x25 0x4c 0x0c 0x01
+# CHECK: convert 37, r10h.cache, r34l.discard, 3{{ +}}// encoding: [0xbe,0xaa,0x25,0x4c,0x0c,0x01]
+0x3e 0xae 0x3a 0x20 0x16 0x03
+# CHECK: convert 58, r11h, u241l, rtz{{ +}}// encoding: [0x3e,0xae,0x3a,0x20,0x16,0x03]
+
+0xae 0x56 0x6b 0x45 0xe7 0xa9 0x80 0x23
+# CHECK: extr r85h.cache, u149h, r122, r84h.cache, mask 0x7{{ +}}// encoding: [0xae,0x56,0x6b,0x45,0xe7,0xa9,0x80,0x23]
+0x2e 0x0c 0x17 0xa4 0x01 0x4a 0x4e 0x0c
+# CHECK: extr r3l, 215, 26, r37, mask 0xFFF{{ +}}// encoding: [0x2e,0x0c,0x17,0xa4,0x01,0x4a,0x4e,0x0c]
+0x2e 0x25 0xc8 0x54 0xce 0x1c 0x89 0x31
+# CHECK: extr r105, r4l.discard, r50h.discard, u78l, mask 0x7FF{{ +}}// encoding: [0x2e,0x25,0xc8,0x54,0xce,0x1c,0x89,0x31]
+0xae 0x0c 0x21 0x24 0x1e 0x5a 0x8d 0xab
+# CHECK: extr r67l.cache, 161, u241, u205l, mask 0xFFFFFFF{{ +}}// encoding: [0xae,0x0c,0x21,0x24,0x1e,0x5a,0x8d,0xab]
+
+0x02 0xac 0x2f 0x49 0x0a 0xee 0x30 0x27 0xd0 0x38
+# CHECK: fcmpsel lt, r107l, u87h.neg, r18l.cache, r119l.discard, r57h{{ +}}// encoding: [0x02,0xac,0x2f,0x49,0x0a,0xee,0x30,0x27,0xd0,0x38]
+0x82 0x9a 0x53 0x3d 0x00 0xee 0x20 0x28 0x60 0x19
+# CHECK: fcmpsel lt, r38h.cache, u201h.abs.neg, 2.375, r55l.discard, r65l.cache{{ +}}// encoding: [0x82,0x9a,0x53,0x3d,0x00,0xee,0x20,0x28,0x60,0x19]
+0x02 0x86 0xf4 0x89 0xc7 0x9d 0x01 0x68 0xe0 0x30
+# CHECK: fcmpsel ltn, r97h, u154.neg, r28l.abs.neg, u110h, r64l.cache{{ +}}// encoding: [0x02,0x86,0xf4,0x89,0xc7,0x9d,0x01,0x68,0xe0,0x30]
+0x02 0xa0 0xbd 0x24 0x16 0x71 0x70 0xaf 0x50 0x09
+# CHECK: fcmpsel gte, r8l, r94h.cache.abs, u177l, r56h, r59h.discard{{ +}}// encoding: [0x02,0xa0,0xbd,0x24,0x16,0x71,0x70,0xaf,0x50,0x09]
+
+0x3e 0x40 0x61 0x0c 0x00 0x04
+# CHECK: ffs r16l, r48h{{ +}}// encoding: [0x3e,0x40,0x61,0x0c,0x00,0x04]
+0x3e 0x25 0x41 0x0c 0x00 0x3c
+# CHECK: ffs r105, r96h{{ +}}// encoding: [0x3e,0x25,0x41,0x0c,0x00,0x3c]
+0xbe 0x51 0xe2 0x0e 0x00 0x0c
+# CHECK: ffs r20.cache, r113.discard{{ +}}// encoding: [0xbe,0x51,0xe2,0x0e,0x00,0x0c]
+0xbe 0x76 0x8a 0x0e 0x00 0x18
+# CHECK: ffs r61h.cache, r69.cache{{ +}}// encoding: [0xbe,0x76,0x8a,0x0e,0x00,0x18]
+
+0x72 0x06 0x14 0x00
+# CHECK: get_sr r1h, sr20 (core_index){{ +}}// encoding: [0x72,0x06,0x14,0x00]
+0x72 0x60 0x1a 0x10
+# CHECK: get_sr r56l, sr26{{ +}}// encoding: [0x72,0x60,0x1a,0x10]
+0xf2 0x58 0x28 0x0c
+# CHECK: get_sr r22l.cache, sr232{{ +}}// encoding: [0xf2,0x58,0x28,0x0c]
+0x72 0x28 0x2a 0x14
+# CHECK: get_sr r42l, sr106{{ +}}// encoding: [0x72,0x28,0x2a,0x14]
+
+0xce 0x28 0x6b 0xf9 0x96 0x35 0x10 0x00
+# CHECK: isub.sat r106l.cache, u181h, u183h, lsl 3{{ +}}// encoding: [0xce,0x28,0x6b,0xf9,0x96,0x35,0x10,0x00]
+0x8e 0x04 0xbe 0xca 0x82 0x3f 0x30 0x00
+# CHECK: isub r97l.cache, r127.cache, 236, lsl 7{{ +}}// encoding: [0x8e,0x04,0xbe,0xca,0x82,0x3f,0x30,0x00]
+0xce 0x4c 0x3a 0x40 0x9c 0x20 0x10 0x00
+# CHECK: iadd.sat r83l.cache, 58, u130, lsl 3{{ +}}// encoding: [0xce,0x4c,0x3a,0x40,0x9c,0x20,0x10,0x00]
+0x4e 0x21 0x85 0x04 0xdd 0x2b 0x30 0x00
+# CHECK: iadd.sat r72, r66h.cache.sx, u232.sx, lsl 7{{ +}}// encoding: [0x4e,0x21,0x85,0x04,0xdd,0x2b,0x30,0x00]
+
+0x92 0xcc 0xff 0xf0 0x13 0x0e 0x21 0xd2 0x60 0x06
+# CHECK: icmpsel sgt, r19l.cache, r63h.discard, u95h, 78, 162{{ +}}// encoding: [0x92,0xcc,0xff,0xf0,0x13,0x0e,0x21,0xd2,0x60,0x06]
+0x12 0xe9 0x4b 0x10 0x0c 0xbc 0x01 0x87 0x60 0x3b
+# CHECK: icmpsel seq, r122, r69h, r96h.discard, u62, r88{{ +}}// encoding: [0x12,0xe9,0x4b,0x10,0x0c,0xbc,0x01,0x87,0x60,0x3b]
+0x92 0xc9 0xad 0xa0 0x19 0x14 0xc1 0x2b 0xe0 0x0e
+# CHECK: icmpsel ult, r18.cache, r118h.cache, u77, 212, r94.cache{{ +}}// encoding: [0x92,0xc9,0xad,0xa0,0x19,0x14,0xc1,0x2b,0xe0,0x0e]
+0x12 0xe8 0x7a 0xd1 0x06 0x5e 0x70 0x0a 0x40 0x0f
+# CHECK: icmpsel ueq, r26l, u253l, r118h, r47l, r19h.cache{{ +}}// encoding: [0x12,0xe8,0x7a,0xd1,0x06,0x5e,0x70,0x0a,0x40,0x0f]
+
+0x9e 0x7d 0xed 0x48 0xd2 0xa4 0x34 0x19
+# CHECK: imsub r63.cache, r86h.discard, u50l.sx, r18l.cache.sx, lsl 7{{ +}}// encoding: [0x9e,0x7d,0xed,0x48,0xd2,0xa4,0x34,0x19]
+0x5e 0x53 0x6e 0x21 0x94 0x3d 0x75 0x02
+# CHECK: imadd.sat r20_r21, u151l, u193l, u62h.sx, lsl 7{{ +}}// encoding: [0x5e,0x53,0x6e,0x21,0x94,0x3d,0x75,0x02]
+0xde 0x1e 0x48 0x09 0x8f 0xef 0x24 0x34
+# CHECK: imsub.sat r103h.cache, u164l, r24l.discard, r23h.discard.sx, lsl 5{{ +}}// encoding: [0xde,0x1e,0x48,0x09,0x8f,0xef,0x24,0x34]
+0x1e 0x6d 0x31 0x45 0xc5 0xca 0xd2 0x20
+# CHECK: imadd r91, u24h.sx, r10l.sx, r101.discard, lsl 3{{ +}}// encoding: [0x1e,0x6d,0x31,0x45,0xc5,0xca,0xd2,0x20]
+
+0xbe 0x36 0x50 0x22 0x11 0x0e
+# CHECK: intl r13h.cache, r104, u73l{{ +}}// encoding: [0xbe,0x36,0x50,0x22,0x11,0x0e]
+0xbe 0x64 0xfd 0x50 0x0b 0x13
+# CHECK: intl r57l.cache, r30h.discard, r122h.cache{{ +}}// encoding: [0xbe,0x64,0xfd,0x50,0x0b,0x13]
+0xbe 0x6d 0x33 0xb0 0x0c 0x0d
+# CHECK: intl r27.cache, 243, r37h.discard{{ +}}// encoding: [0xbe,0x6d,0x33,0xb0,0x0c,0x0d]
+0xbe 0x65 0x09 0x80 0x19 0x0f
+# CHECK: intl r25.cache, 201, u108{{ +}}// encoding: [0xbe,0x65,0x09,0x80,0x19,0x0f]
+
+0x62 0xda 0x7b 0x95 0x00 0x10
+# CHECK: mov_imm r54h, 38267{{ +}}// encoding: [0x62,0xda,0x7b,0x95,0x00,0x10]
+0xe2 0xae 0x74 0xa0 0x00 0x20
+# CHECK: mov_imm r75h.cache, 41076{{ +}}// encoding: [0xe2,0xae,0x74,0xa0,0x00,0x20]
+0xe2 0xa2 0xf1 0x99 0x00 0x20
+# CHECK: mov_imm r72h.cache, 39409{{ +}}// encoding: [0xe2,0xa2,0xf1,0x99,0x00,0x20]
+0x62 0xde 0x2a 0x70 0x00 0x30
+# CHECK: mov_imm r119h, 28714{{ +}}// encoding: [0x62,0xde,0x2a,0x70,0x00,0x30]
+
+0x3e 0x5a 0xf8 0x08 0x00 0x08
+# CHECK: popcount r22h, r92l.discard{{ +}}// encoding: [0x3e,0x5a,0xf8,0x08,0x00,0x08]
+0x3e 0x54 0x7f 0x08 0x00 0x28
+# CHECK: popcount r85l, r95h{{ +}}// encoding: [0x3e,0x54,0x7f,0x08,0x00,0x28]
+0x3e 0x64 0xfe 0x09 0x00 0x14
+# CHECK: popcount r57l, u191{{ +}}// encoding: [0x3e,0x64,0xfe,0x09,0x00,0x14]
+0xbe 0x0a 0x36 0x08 0x00 0x00
+# CHECK: popcount r2h.cache, 54{{ +}}// encoding: [0xbe,0x0a,0x36,0x08,0x00,0x00]
+
+0x2e 0x58 0xda 0xea 0x0a 0x52 0x0a 0x3f
+# CHECK: shlhi r118l, r109.discard, r119l.cache, r9, mask 0xFF{{ +}}// encoding: [0x2e,0x58,0xda,0xea,0x0a,0x52,0x0a,0x3f]
+0xae 0x54 0x3b 0x98 0x03 0xc2 0x46 0x9c
+# CHECK: shlhi r53l.cache, 251, 57, r33.discard, mask 0xFFFFF{{ +}}// encoding: [0xae,0x54,0x3b,0x98,0x03,0xc2,0x46,0x9c]
+0xae 0x5e 0x68 0x08 0x2c 0x06 0x00 0x2f
+# CHECK: shlhi r87h.cache, r116l, r96.discard, 6{{ +}}// encoding: [0xae,0x5e,0x68,0x08,0x2c,0x06,0x00,0x2f]
+0x2e 0x04 0x1f 0x89 0xc2 0x33 0x45 0x02
+# CHECK: shlhi r1l, u15h, 168, u57h, mask 0x7F{{ +}}// encoding: [0x2e,0x04,0x1f,0x89,0xc2,0x33,0x45,0x02]
+
+0xae 0xf0 0x7b 0x09 0x1b 0x80 0x85 0x23
+# CHECK: shrhi r92l.cache, u157h, u120, u64, mask 0xF{{ +}}// encoding: [0xae,0xf0,0x7b,0x09,0x1b,0x80,0x85,0x23]
+0x2e 0xba 0x14 0x49 0x09 0xfd 0x00 0x28
+# CHECK: shrhi r78h, u74l, r10l.cache, r30h.discard{{ +}}// encoding: [0x2e,0xba,0x14,0x49,0x09,0xfd,0x00,0x28]
+0x2e 0xd1 0x93 0x48 0x68 0x51 0x85 0x02
+# CHECK: shrhi r20, r9h.cache, r66.cache, u200h, mask 0x1F{{ +}}// encoding: [0x2e,0xd1,0x93,0x48,0x68,0x51,0x85,0x02]
+0xae 0xe1 0x90 0xb8 0x84 0xf0 0xc0 0xb8
+# CHECK: shrhi r120.cache, r72l.cache, r5h, r120l.discard, mask 0x3FFFF{{ +}}// encoding: [0xae,0xe1,0x90,0xb8,0x84,0xf0,0xc0,0xb8]
+
diff --git a/llvm/test/MC/Disassembler/AGX/g13-control.txt b/llvm/test/MC/Disassembler/AGX/g13-control.txt
new file mode 100644
index 0000000000000..c0b51bfe240cc
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AGX/g13-control.txt
@@ -0,0 +1,115 @@
+# RUN: llvm-mc -triple=agx -mcpu=agx-g13g -disassemble -show-encoding %s | FileCheck %s
+
+# Encodings cross-checked against the applegpu reference disassembler
+# (dougallj/applegpu).
+
+0x42 0xaa 0xf2 0xa9 0x08 0x0b
+# CHECK: else_fcmp r0l, gte, u217.neg, r101l.cache, 1{{ +}}// encoding: [0x42,0xaa,0xf2,0xa9,0x08,0x0b]
+0x42 0xab 0x48 0x85 0x6f 0x09
+# CHECK: else_fcmp r0l, ngte, u196l.abs, r60.discard.abs, 1{{ +}}// encoding: [0x42,0xab,0x48,0x85,0x6f,0x09]
+0xc2 0xd2 0x98 0xb6 0x17 0x0e
+# CHECK: else_fcmp r0l.cache, lte, r108.cache.abs, u221h, 2{{ +}}// encoding: [0xc2,0xd2,0x98,0xb6,0x17,0x0e]
+0xc2 0x73 0x54 0xc6 0x85 0x03
+# CHECK: else_fcmp r0l.cache, nltn, r10.abs, r110l.neg, 2{{ +}}// encoding: [0xc2,0x73,0x54,0xc6,0x85,0x03]
+
+0x52 0x2a 0x4a 0x21 0x04 0x0a
+# CHECK: else_icmp r0l, ult, u197l, r65l, 1{{ +}}// encoding: [0x52,0x2a,0x4a,0x21,0x04,0x0a]
+0x52 0x02 0x15 0x21 0x00 0x0c
+# CHECK: else_icmp r0l, ueq, u106h, 2, 0{{ +}}// encoding: [0x52,0x02,0x15,0x21,0x00,0x0c]
+0x52 0x82 0x43 0x51 0x05 0x06
+# CHECK: else_icmp r0l, seq, u161h, r74h, 0{{ +}}// encoding: [0x52,0x82,0x43,0x51,0x05,0x06]
+0x52 0x2a 0xe2 0x20 0x1a 0x08
+# CHECK: else_icmp r0l, ult, r81l.discard, u17, 1{{ +}}// encoding: [0x52,0x2a,0xe2,0x20,0x1a,0x08]
+
+0x42 0x20 0x67 0x15 0x05 0x07
+# CHECK: if_fcmp r0l, lt, u179h.abs, r104h, 0{{ +}}// encoding: [0x42,0x20,0x67,0x15,0x05,0x07]
+0xc2 0xf1 0xd0 0xe9 0x91 0x08
+# CHECK: if_fcmp r0l.cache, ngtn, u200.neg, u15l.neg, 2{{ +}}// encoding: [0xc2,0xf1,0xd0,0xe9,0x91,0x08]
+0xc2 0xc0 0xf5 0x80 0x84 0x0f
+# CHECK: if_fcmp r0l.cache, lte, r122h.discard, r100l.neg, 0{{ +}}// encoding: [0xc2,0xc0,0xf5,0x80,0x84,0x0f]
+0x42 0xd9 0xe2 0xc1 0x28 0x01
+# CHECK: if_fcmp r0l, nlte, u145, r38.cache, 3{{ +}}// encoding: [0x42,0xd9,0xe2,0xc1,0x28,0x01]
+
+0x52 0x21 0x30 0xf1 0x11 0x01
+# CHECK: if_icmp r0l, ugte, u24l, u47h, 0{{ +}}// encoding: [0x52,0x21,0x30,0xf1,0x11,0x01]
+0x52 0xc1 0x3a 0x01 0x1d 0x02
+# CHECK: if_icmp r0l, slte, u29l, u200, 0{{ +}}// encoding: [0x52,0xc1,0x3a,0x01,0x1d,0x02]
+0x52 0x50 0x39 0x81 0x05 0x0b
+# CHECK: if_icmp r0l, ugt, u92h, r108l, 2{{ +}}// encoding: [0x52,0x50,0x39,0x81,0x05,0x0b]
+0xd2 0x80 0x22 0x51 0x06 0x0b
+# CHECK: if_icmp r0l.cache, seq, u81l, r114h, 0{{ +}}// encoding: [0xd2,0x80,0x22,0x51,0x06,0x0b]
+
+
+
+0xf5 0xf4
+# CHECK: memory_barrier 1, 0, 15{{ +}}// encoding: [0xf5,0xf4]
+0xf5 0xdf
+# CHECK: memory_barrier 3, 3, 13{{ +}}// encoding: [0xf5,0xdf]
+0xf5 0x9c
+# CHECK: memory_barrier 3, 0, 9{{ +}}// encoding: [0xf5,0x9c]
+0xf5 0xc9
+# CHECK: memory_barrier 2, 1, 12{{ +}}// encoding: [0xf5,0xc9]
+
+0x52 0x06 0x00 0x00 0x00 0x00
+# CHECK: pop_exec r0l, 0{{ +}}// encoding: [0x52,0x06,0x00,0x00,0x00,0x00]
+0x52 0x1e 0x00 0x00 0x00 0x00
+# CHECK: pop_exec r0l, 3{{ +}}// encoding: [0x52,0x1e,0x00,0x00,0x00,0x00]
+0xd2 0x1e 0x00 0x00 0x00 0x00
+# CHECK: pop_exec r0l.cache, 3{{ +}}// encoding: [0xd2,0x1e,0x00,0x00,0x00,0x00]
+0xd2 0x16 0x00 0x00 0x00 0x00
+# CHECK: pop_exec r0l.cache, 2{{ +}}// encoding: [0xd2,0x16,0x00,0x00,0x00,0x00]
+
+0x88 0x00
+# CHECK: stop{{ +}}// encoding: [0x88,0x00]
+0x88 0x00
+# CHECK: stop{{ +}}// encoding: [0x88,0x00]
+0x88 0x00
+# CHECK: stop{{ +}}// encoding: [0x88,0x00]
+0x88 0x00
+# CHECK: stop{{ +}}// encoding: [0x88,0x00]
+
+0x68 0x00
+# CHECK: threadgroup_barrier{{ +}}// encoding: [0x68,0x00]
+0x68 0x00
+# CHECK: threadgroup_barrier{{ +}}// encoding: [0x68,0x00]
+0x68 0x00
+# CHECK: threadgroup_barrier{{ +}}// encoding: [0x68,0x00]
+0x68 0x00
+# CHECK: threadgroup_barrier{{ +}}// encoding: [0x68,0x00]
+
+0x08 0x00
+# CHECK: trap{{ +}}// encoding: [0x08,0x00]
+0x08 0x00
+# CHECK: trap{{ +}}// encoding: [0x08,0x00]
+0x08 0x00
+# CHECK: trap{{ +}}// encoding: [0x08,0x00]
+0x08 0x00
+# CHECK: trap{{ +}}// encoding: [0x08,0x00]
+
+0x38 0x01
+# CHECK: wait 1{{ +}}// encoding: [0x38,0x01]
+0x38 0x00
+# CHECK: wait 0{{ +}}// encoding: [0x38,0x00]
+0x38 0x00
+# CHECK: wait 0{{ +}}// encoding: [0x38,0x00]
+0x38 0x01
+# CHECK: wait 1{{ +}}// encoding: [0x38,0x01]
+
+0x42 0xf4 0xda 0xc0 0xd8 0x09
+# CHECK: while_fcmp r0l, gtn, r77l.discard, u38.abs.neg, 2{{ +}}// encoding: [0x42,0xf4,0xda,0xc0,0xd8,0x09]
+0xc2 0x6c 0x46 0x9c 0x00 0x05
+# CHECK: while_fcmp r0l.cache, ltn, r35l.abs.neg, 3.125, 1{{ +}}// encoding: [0xc2,0x6c,0x46,0x9c,0x00,0x05]
+0x42 0xd4 0x54 0xea 0xd4 0x01
+# CHECK: while_fcmp r0l, lte, r10.neg, u167l.abs.neg, 2{{ +}}// encoding: [0x42,0xd4,0x54,0xea,0xd4,0x01]
+0x42 0x3c 0xa3 0x68 0x1c 0x08
+# CHECK: while_fcmp r0l, lt, r81h.cache.neg, u131, 3{{ +}}// encoding: [0x42,0x3c,0xa3,0x68,0x1c,0x08]
+
+0x52 0x14 0x66 0xa2 0x18 0x08
+# CHECK: while_icmp r0l, ueq, r83, u5, 2{{ +}}// encoding: [0x52,0x14,0x66,0xa2,0x18,0x08]
+0xd2 0x4d 0xa1 0x00 0x18 0x06
+# CHECK: while_icmp r0l.cache, ulte, r48h.cache, u64, 1{{ +}}// encoding: [0xd2,0x4d,0xa1,0x00,0x18,0x06]
+0xd2 0x9c 0xe9 0x60 0x11 0x01
+# CHECK: while_icmp r0l.cache, seq, r20h.discard, u43l, 3{{ +}}// encoding: [0xd2,0x9c,0xe9,0x60,0x11,0x01]
+0xd2 0x9c 0x5a 0x20 0x05 0x06
+# CHECK: while_icmp r0l.cache, seq, r45l, r73l, 3{{ +}}// encoding: [0xd2,0x9c,0x5a,0x20,0x05,0x06]
+
diff --git a/llvm/test/MC/Disassembler/AGX/g13-fp.txt b/llvm/test/MC/Disassembler/AGX/g13-fp.txt
new file mode 100644
index 0000000000000..09ab06ed469de
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AGX/g13-fp.txt
@@ -0,0 +1,160 @@
+# RUN: llvm-mc -triple=agx -mcpu=agx-g13g -disassemble -show-encoding %s | FileCheck %s
+
+# Encodings cross-checked against the applegpu reference disassembler
+# (dougallj/applegpu).
+
+0x4a 0xaa 0x1f 0x01 0x01 0x28
+# CHECK: ceil.sat r74h, u79h{{ +}}// encoding: [0x4a,0xaa,0x1f,0x01,0x01,0x28]
+0x0a 0xe8 0x6d 0x08 0x01 0x2c
+# CHECK: ceil r90l, r118h.neg{{ +}}// encoding: [0x0a,0xe8,0x6d,0x08,0x01,0x2c]
+0xca 0xa9 0x77 0x0c 0x01 0x04
+# CHECK: ceil.sat r10.cache, r59h.abs.neg{{ +}}// encoding: [0xca,0xa9,0x77,0x0c,0x01,0x04]
+0x0a 0xf8 0xca 0x09 0x01 0x24
+# CHECK: ceil r94l, u165.neg{{ +}}// encoding: [0x0a,0xf8,0xca,0x09,0x01,0x24]
+
+
+
+0xca 0xd9 0x80 0xd6 0x00 0x10
+# CHECK: exp2.sat r54.cache, r0.cache.abs{{ +}}// encoding: [0xca,0xd9,0x80,0xd6,0x00,0x10]
+0xca 0x9c 0xac 0xd1 0x00 0x3c
+# CHECK: exp2.sat r103l.cache, u118{{ +}}// encoding: [0xca,0x9c,0xac,0xd1,0x00,0x3c]
+0x8a 0xc2 0x7e 0xd6 0x00 0x38
+# CHECK: exp2 r112h.cache, r95.abs{{ +}}// encoding: [0x8a,0xc2,0x7e,0xd6,0x00,0x38]
+0xca 0xb5 0x15 0xd1 0x00 0x3c
+# CHECK: exp2.sat r109.cache, u106h{{ +}}// encoding: [0xca,0xb5,0x15,0xd1,0x00,0x3c]
+
+0xe6 0xc2 0x1e 0xf0 0x55 0x04
+# CHECK: fadd16.sat r16h.cache, 7.5, u143h.neg{{ +}}// encoding: [0xe6,0xc2,0x1e,0xf0,0x55,0x04]
+0xe6 0xd0 0x30 0xb3 0x0e 0x13
+# CHECK: fadd16.sat r52l.cache, u24l.abs, r117h.discard{{ +}}// encoding: [0xe6,0xd0,0x30,0xb3,0x0e,0x13]
+0xa6 0xc2 0x59 0x20 0x51 0x23
+# CHECK: fadd16 r80h.cache, r12h, u105l.neg{{ +}}// encoding: [0xa6,0xc2,0x59,0x20,0x51,0x23]
+0xe6 0xfc 0xb1 0x66 0x17 0x3c
+# CHECK: fadd16.sat r127l.cache, r120h.cache.abs.neg, u155l{{ +}}// encoding: [0xe6,0xfc,0xb1,0x66,0x17,0x3c]
+
+0xaa 0xa0 0x1e 0x40 0xed 0x0d
+# CHECK: fadd32 r8l.cache, -7.5, r42.discard.abs.neg{{ +}}// encoding: [0xaa,0xa0,0x1e,0x40,0xed,0x0d]
+0xea 0x9a 0xcf 0x58 0x4d 0x38
+# CHECK: fadd32.sat r102h.cache, r71h.discard.neg, r10h.discard.abs{{ +}}// encoding: [0xea,0x9a,0xcf,0x58,0x4d,0x38]
+0x2a 0xf2 0xec 0xce 0x2e 0x35
+# CHECK: fadd32 r124h, r54.discard.abs.neg, r54.discard{{ +}}// encoding: [0x2a,0xf2,0xec,0xce,0x2e,0x35]
+0xaa 0xc2 0x33 0x55 0x53 0x0a
+# CHECK: fadd32 r16h.cache, u89h.abs, u90h.abs{{ +}}// encoding: [0xaa,0xc2,0x33,0x55,0x53,0x0a]
+
+0x4a 0x88 0x89 0x04 0x00 0x34
+# CHECK: floor.sat r98l, r36h.cache.abs{{ +}}// encoding: [0x4a,0x88,0x89,0x04,0x00,0x34]
+0xca 0xb2 0xbf 0x00 0x00 0x10
+# CHECK: floor.sat r44h.cache, r31h.cache{{ +}}// encoding: [0xca,0xb2,0xbf,0x00,0x00,0x10]
+0x0a 0xe5 0x9a 0x08 0x00 0x34
+# CHECK: floor r121, r45l.cache.neg{{ +}}// encoding: [0x0a,0xe5,0x9a,0x08,0x00,0x34]
+0x4a 0xc8 0xac 0x08 0x00 0x0c
+# CHECK: floor.sat r18l, r118l.cache.neg{{ +}}// encoding: [0x4a,0xc8,0xac,0x08,0x00,0x0c]
+
+0xb6 0xe0 0x7e 0xd4 0x4e 0x7a 0xc5 0x12
+# CHECK: fmadd16 r56l.cache, r31l.neg, r86h.discard.neg, u253l.neg{{ +}}// encoding: [0xb6,0xe0,0x7e,0xd4,0x4e,0x7a,0xc5,0x12]
+0x76 0x94 0x5c 0x42 0x17 0xec 0x03 0x30
+# CHECK: fmadd16.sat r101l, r14l.abs, u154l, u150.abs{{ +}}// encoding: [0x76,0x94,0x5c,0x42,0x17,0xec,0x03,0x30]
+0x36 0x96 0xc6 0x75 0x67 0xf2 0x87 0x04
+# CHECK: fmadd16 r5h, u163.neg, r27h.abs.neg, u217.abs.neg{{ +}}// encoding: [0x36,0x96,0xc6,0x75,0x67,0xf2,0x87,0x04]
+0xf6 0xc8 0x68 0x65 0x58 0xbd 0x80 0x17
+# CHECK: fmadd16.sat r50l.cache, u180l.neg, u99.neg, r94h.cache{{ +}}// encoding: [0xf6,0xc8,0x68,0x65,0x58,0xbd,0x80,0x17]
+
+0xba 0x91 0x64 0x6d 0xd7 0x55 0xcc 0x06
+# CHECK: fmadd32 r4.cache, u178l.abs.neg, u219l.abs.neg, r106h.abs.neg{{ +}}// encoding: [0xba,0x91,0x64,0x6d,0xd7,0x55,0xcc,0x06]
+0xba 0x89 0x5b 0x78 0x88 0x40 0x8d 0x15
+# CHECK: fmadd32 r34.cache, r45h.neg, r35h.cache.neg, u192l.abs.neg{{ +}}// encoding: [0xba,0x89,0x5b,0x78,0x88,0x40,0x8d,0x15]
+0xfa 0x95 0x98 0x1e 0xd2 0xd0 0xc2 0x34
+# CHECK: fmadd32.sat r101.cache, r44.cache.abs.neg, u16h.abs.neg, r104.discard{{ +}}// encoding: [0xfa,0x95,0x98,0x1e,0xd2,0xd0,0xc2,0x34]
+0x3a 0x9a 0x5a 0xcd 0x88 0xb2 0xc2 0x1a
+# CHECK: fmadd32 r38h, u205l.abs.neg, r70l.cache.neg, r121.cache{{ +}}// encoding: [0x3a,0x9a,0x5a,0xcd,0x88,0xb2,0xc2,0x1a]
+
+0x96 0xa4 0x5a 0x20 0x1c 0x2a
+# CHECK: fmul16 r73l.cache, r77l, u193{{ +}}// encoding: [0x96,0xa4,0x5a,0x20,0x1c,0x2a]
+0xd6 0xee 0x1d 0xf7 0x74 0x25
+# CHECK: fmul16.sat r91h.cache, u46h.abs.neg, u167h.abs.neg{{ +}}// encoding: [0xd6,0xee,0x1d,0xf7,0x74,0x25]
+0x16 0xc0 0xe2 0x44 0x65 0x01
+# CHECK: fmul16 r16l, r17l.discard.neg, r42l.abs.neg{{ +}}// encoding: [0x16,0xc0,0xe2,0x44,0x65,0x01]
+0xd6 0xac 0x78 0xe1 0x65 0x32
+# CHECK: fmul16.sat r107l.cache, u156l, r79l.abs.neg{{ +}}// encoding: [0xd6,0xac,0x78,0xe1,0x65,0x32]
+
+0x1a 0xf0 0x8c 0xe5 0xa4 0x00
+# CHECK: fmul32 r28l, u6.abs, r7.neg{{ +}}// encoding: [0x1a,0xf0,0x8c,0xe5,0xa4,0x00]
+0xda 0xc2 0xe0 0x89 0x94 0x0a
+# CHECK: fmul32.sat r16h.cache, u208.neg, u196l.neg{{ +}}// encoding: [0xda,0xc2,0xe0,0x89,0x94,0x0a]
+0x9a 0x85 0x15 0x19 0x96 0x26
+# CHECK: fmul32 r65.cache, u42h.neg, u208h.neg{{ +}}// encoding: [0x9a,0x85,0x15,0x19,0x96,0x26]
+0xda 0xbc 0xcd 0x88 0x1b 0x07
+# CHECK: fmul32.sat r15l.cache, r38h.discard.neg, u124{{ +}}// encoding: [0xda,0xbc,0xcd,0x88,0x1b,0x07]
+
+0x8a 0xe4 0x4e 0xc6 0x00 0x30
+# CHECK: log2 r121l.cache, r7.abs{{ +}}// encoding: [0x8a,0xe4,0x4e,0xc6,0x00,0x30]
+0xca 0xcd 0x76 0xc6 0x00 0x28
+# CHECK: log2.sat r83.cache, r91.abs{{ +}}// encoding: [0xca,0xcd,0x76,0xc6,0x00,0x28]
+0xca 0xb5 0x8a 0xc1 0x00 0x18
+# CHECK: log2.sat r45.cache, u69{{ +}}// encoding: [0xca,0xb5,0x8a,0xc1,0x00,0x18]
+0x4a 0x9c 0x75 0xc8 0x00 0x2c
+# CHECK: log2.sat r71l, r122h.neg{{ +}}// encoding: [0x4a,0x9c,0x75,0xc8,0x00,0x2c]
+
+0xca 0xed 0x6b 0x81 0x00 0x2c
+# CHECK: rcp.sat r91.cache, u245h{{ +}}// encoding: [0xca,0xed,0x6b,0x81,0x00,0x2c]
+0xca 0x9d 0x96 0x86 0x00 0x1c
+# CHECK: rcp.sat r39.cache, r107.cache.abs{{ +}}// encoding: [0xca,0x9d,0x96,0x86,0x00,0x1c]
+0x8a 0xc2 0x96 0x8d 0x00 0x3c
+# CHECK: rcp r112h.cache, u107.abs.neg{{ +}}// encoding: [0x8a,0xc2,0x96,0x8d,0x00,0x3c]
+0x4a 0xc0 0xee 0x8c 0x00 0x24
+# CHECK: rcp.sat r80l, r55l.discard.abs.neg{{ +}}// encoding: [0x4a,0xc0,0xee,0x8c,0x00,0x24]
+
+0x8a 0xac 0x02 0x0d 0x03 0x0c
+# CHECK: rint r11l.cache, u97l.abs.neg{{ +}}// encoding: [0x8a,0xac,0x02,0x0d,0x03,0x0c]
+0xca 0xe9 0x2a 0x05 0x03 0x28
+# CHECK: rint.sat r90.cache, u85l.abs{{ +}}// encoding: [0xca,0xe9,0x2a,0x05,0x03,0x28]
+0x8a 0xa0 0x5a 0x0c 0x03 0x30
+# CHECK: rint r104l.cache, r13l.abs.neg{{ +}}// encoding: [0x8a,0xa0,0x5a,0x0c,0x03,0x30]
+0x8a 0xd6 0xf0 0x01 0x03 0x20
+# CHECK: rint r85h.cache, u152{{ +}}// encoding: [0x8a,0xd6,0xf0,0x01,0x03,0x20]
+
+0x4a 0x85 0x8d 0x98 0x00 0x3c
+# CHECK: rsqrt.sat r97, r102h.cache.neg{{ +}}// encoding: [0x4a,0x85,0x8d,0x98,0x00,0x3c]
+0xca 0xc8 0x3d 0x95 0x00 0x34
+# CHECK: rsqrt.sat r114l.cache, u62h.abs{{ +}}// encoding: [0xca,0xc8,0x3d,0x95,0x00,0x34]
+0xca 0xd1 0x36 0x91 0x00 0x28
+# CHECK: rsqrt.sat r84.cache, u91l{{ +}}// encoding: [0xca,0xd1,0x36,0x91,0x00,0x28]
+0x0a 0xa5 0x54 0x98 0x00 0x20
+# CHECK: rsqrt r73, r10l.neg{{ +}}// encoding: [0x0a,0xa5,0x54,0x98,0x00,0x20]
+
+0x8a 0xba 0x74 0x10 0x00 0x10
+# CHECK: rsqrt_special r46h.cache, r26l{{ +}}// encoding: [0x8a,0xba,0x74,0x10,0x00,0x10]
+0x0a 0xf5 0xb0 0x11 0x00 0x2c
+# CHECK: rsqrt_special r93, u120{{ +}}// encoding: [0x0a,0xf5,0xb0,0x11,0x00,0x2c]
+0xca 0x05 0x68 0x1d
+# CHECK: rsqrt_special.sat r1.cache, u148l.abs.neg{{ +}}// encoding: [0xca,0x05,0x68,0x1d]
+0x4a 0xd6 0x31 0x19 0x00 0x10
+# CHECK: rsqrt_special.sat r53h, u24h.neg{{ +}}// encoding: [0x4a,0xd6,0x31,0x19,0x00,0x10]
+
+0xca 0xf8 0xbc 0xa2 0x00 0x18
+# CHECK: sin_pt_1.sat r62l.cache, r94.cache{{ +}}// encoding: [0xca,0xf8,0xbc,0xa2,0x00,0x18]
+0xca 0xb9 0x55 0xa9 0x00 0x04
+# CHECK: sin_pt_1.sat r14.cache, u170h.neg{{ +}}// encoding: [0xca,0xb9,0x55,0xa9,0x00,0x04]
+0x4a 0xbc 0x8c 0xa1 0x00 0x20
+# CHECK: sin_pt_1.sat r79l, u6{{ +}}// encoding: [0x4a,0xbc,0x8c,0xa1,0x00,0x20]
+0x8a 0xcc 0x98 0xa1 0x00 0x34
+# CHECK: sin_pt_1 r115l.cache, u44{{ +}}// encoding: [0x8a,0xcc,0x98,0xa1,0x00,0x34]
+
+0x4a 0x7e 0xeb 0xe4
+# CHECK: sin_pt_2.sat r31h, r21h.discard.abs{{ +}}// encoding: [0x4a,0x7e,0xeb,0xe4]
+0x4a 0xc0 0xd1 0xec 0x00 0x1c
+# CHECK: sin_pt_2.sat r48l, r104h.discard.abs.neg{{ +}}// encoding: [0x4a,0xc0,0xd1,0xec,0x00,0x1c]
+0x0a 0xd9 0x88 0xec 0x00 0x38
+# CHECK: sin_pt_2 r118, r68l.cache.abs.neg{{ +}}// encoding: [0x0a,0xd9,0x88,0xec,0x00,0x38]
+0x0a 0xc8 0x99 0xec 0x00 0x20
+# CHECK: sin_pt_2 r82l, r12h.cache.abs.neg{{ +}}// encoding: [0x0a,0xc8,0x99,0xec,0x00,0x20]
+
+0x4a 0xc9 0x32 0x05 0x02 0x24
+# CHECK: trunc.sat r82, u57l.abs{{ +}}// encoding: [0x4a,0xc9,0x32,0x05,0x02,0x24]
+0x4a 0xf9 0x2a 0x09 0x02 0x0c
+# CHECK: trunc.sat r30, u117l.neg{{ +}}// encoding: [0x4a,0xf9,0x2a,0x09,0x02,0x0c]
+0xca 0xf0 0xee 0x04 0x02 0x34
+# CHECK: trunc.sat r124l.cache, r55l.discard.abs{{ +}}// encoding: [0xca,0xf0,0xee,0x04,0x02,0x34]
+0x8a 0xfa 0xb2 0x05 0x02 0x34
+# CHECK: trunc r126h.cache, u57.abs{{ +}}// encoding: [0x8a,0xfa,0xb2,0x05,0x02,0x34]
+
diff --git a/llvm/test/MC/Disassembler/AGX/g13-memory.txt b/llvm/test/MC/Disassembler/AGX/g13-memory.txt
new file mode 100644
index 0000000000000..da5aeab4235e3
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AGX/g13-memory.txt
@@ -0,0 +1,42 @@
+# RUN: llvm-mc -triple=agx -mcpu=agx-g13g -disassemble -show-encoding %s | FileCheck %s
+
+# Encodings cross-checked against the applegpu reference disassembler
+# (dougallj/applegpu).
+
+0x15 0x11 0x8a 0x85 0x41 0xd0 0xec 0x91
+# CHECK: atomic umin, 0, r2, r37_r38, -28392, signed, r118, 0{{ +}}// encoding: [0x15,0x11,0x8a,0x85,0x41,0xd0,0xec,0x91]
+0x15 0x3a 0x52 0x87 0xff 0xf1 0x2a 0xce
+# CHECK: atomic and, 0, r39, r121_r122, -12555, unsigned, r21, 1{{ +}}// encoding: [0x15,0x3a,0x52,0x87,0xff,0xf1,0x2a,0xce]
+0xd5 0x52 0x14 0xc5 0x18 0xd2 0x98 0xbc
+# CHECK: atomic 11, 1, r74, r10_r11, -17279, signed, r76, 0{{ +}}// encoding: [0xd5,0x52,0x14,0xc5,0x18,0xd2,0x98,0xbc]
+0x15 0x68 0x76 0x8d 0x58 0xf1 0x5a 0xf9
+# CHECK: atomic add, 0, r45, u43_u44, -1657, signed, r45, 1{{ +}}// encoding: [0x15,0x68,0x76,0x8d,0x58,0xf1,0x5a,0xf9]
+
+
+0x45 0xe0 0x72 0x45 0x3a 0xc6 0x30 0x74
+# CHECK: device_store 1, i8, xy, r92l_r92h, r25_r26, 29863, signed, lsl 1{{ +}}// encoding: [0x45,0xe0,0x72,0x45,0x3a,0xc6,0x30,0x74]
+0x45 0x0c 0xc6 0x45 0xdf 0xc7 0x51 0x37
+# CHECK: device_store 1, rgb10a2, xz, r97h_r98l, r107_r108, 14332, signed, lsl 1{{ +}}// encoding: [0x45,0x0c,0xc6,0x45,0xdf,0xc7,0x51,0x37]
+0x45 0xc7 0x44 0x05 0xbb 0xc2 0xb1 0x5e
+# CHECK: device_store 0, rg11b10f.rtz, xyw, r88h_r89l_r89h, r90_r91, 24244, signed{{ +}}// encoding: [0x45,0xc7,0x44,0x05,0xbb,0xc2,0xb1,0x5e]
+0x45 0x7d 0xd2 0x0d 0x9d 0xc5 0xf1 0xca
+# CHECK: device_store 0, srgba8, xyzw, r47h_r48l_r48h_r49l, u73_u74, -13603, signed, lsl 1{{ +}}// encoding: [0x45,0x7d,0xd2,0x0d,0x9d,0xc5,0xf1,0xca]
+
+0x69 0xf1 0xf3 0xb4 0x36 0x00 0x11 0x29
+# CHECK: threadgroup_load u8norm, xy, r92_r93, u217h, 17515{{ +}}// encoding: [0x69,0xf1,0xf3,0xb4,0x36,0x00,0x11,0x29]
+0x69 0xa6 0x5c 0x01 0xb7 0x00 0x4f 0x3e
+# CHECK: threadgroup_load i16, xyw, r105h_r106l_r106h, u110l, -27664{{ +}}// encoding: [0x69,0xa6,0x5c,0x01,0xb7,0x00,0x4f,0x3e]
+0x69 0xf2 0x00 0x6a 0x15 0x00 0xd5 0x17
+# CHECK: threadgroup_load srgba8, x, r60h, r32l, -2730{{ +}}// encoding: [0x69,0xf2,0x00,0x6a,0x15,0x00,0xd5,0x17]
+0x69 0xa8 0x0f 0xb1 0x45 0x00 0x48 0x0e
+# CHECK: threadgroup_load i16, z, r10l, r103h, -28133{{ +}}// encoding: [0x69,0xa8,0x0f,0xb1,0x45,0x00,0x48,0x0e]
+
+0x29 0x82 0x10 0xde 0x36 0x00 0x59 0x07
+# CHECK: threadgroup_store rg11b10f.rtz, xy, r0h_r1l, r40l, -10643{{ +}}// encoding: [0x29,0x82,0x10,0xde,0x36,0x00,0x59,0x07]
+0x29 0xe6 0x24 0x96 0x37 0x00 0x20 0x27
+# CHECK: threadgroup_store u16norm, xy, r89h_r90l, r50l, -14279{{ +}}// encoding: [0x29,0xe6,0x24,0x96,0x37,0x00,0x20,0x27]
+0x29 0xb8 0x43 0xf2 0x56 0x00 0xe7 0x3a
+# CHECK: threadgroup_store i32, xz, r110l_r110h, u65h, -17937{{ +}}// encoding: [0x29,0xb8,0x43,0xf2,0x56,0x00,0xe7,0x3a]
+0x29 0x90 0x43 0x1e 0xd4 0x00 0xd7 0x39
+# CHECK: threadgroup_store rg11b10f.rtz, xzw, r100l_r100h_r101l, u65h, 30145{{ +}}// encoding: [0x29,0x90,0x43,0x1e,0xd4,0x00,0xd7,0x39]
+
diff --git a/llvm/test/MC/Disassembler/AGX/g13-simd.txt b/llvm/test/MC/Disassembler/AGX/g13-simd.txt
new file mode 100644
index 0000000000000..8c3871c06ba04
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AGX/g13-simd.txt
@@ -0,0 +1,464 @@
+# RUN: llvm-mc -triple=agx -mcpu=agx-g13g -disassemble -show-encoding %s | FileCheck %s
+
+# Encodings cross-checked against the applegpu reference disassembler
+# (dougallj/applegpu).
+
+0xa2 0x54 0x5a 0x28 0x06 0xa2 0x01 0xc0
+# CHECK: fcmp_ballot r85l.cache, nlte, r13l.neg, r81l{{ +}}// encoding: [0xa2,0x54,0x5a,0x28,0x06,0xa2,0x01,0xc0]
+0x22 0x5d 0x96 0xed 0x4b 0x25 0x01 0xe0
+# CHECK: fcmp_ballot r87, gtn, u43.abs.neg, r63l.cache.abs{{ +}}// encoding: [0x22,0x5d,0x96,0xed,0x4b,0x25,0x01,0xe0]
+0x22 0x10 0x86 0xe9 0x00 0xbc 0x01 0x40
+# CHECK: fcmp_ballot r100l, ngt, u99.neg, 0.21875{{ +}}// encoding: [0x22,0x10,0x86,0xe9,0x00,0xbc,0x01,0x40]
+0x22 0x0d 0x72 0x3c 0x0d 0x04 0x01 0x00
+# CHECK: fcmp_ballot r3, eq, r57l.abs.neg, r9h.discard{{ +}}// encoding: [0x22,0x0d,0x72,0x3c,0x0d,0x04,0x01,0x00]
+
+0x22 0x30 0xb3 0xbc 0x47 0x9f 0x00 0x40
+# CHECK: fcmp_quad_ballot r44l, ngt, r121h.cache.abs.neg, r125h.abs{{ +}}// encoding: [0x22,0x30,0xb3,0xbc,0x47,0x9f,0x00,0x40]
+0x22 0x28 0xa1 0x28 0x6e 0x3e 0x00 0x20
+# CHECK: fcmp_quad_ballot r106l, lt, r112h.cache.neg, r81.discard.abs{{ +}}// encoding: [0x22,0x28,0xa1,0x28,0x6e,0x3e,0x00,0x20]
+0xa2 0x54 0x6a 0x00 0x0f 0xa2 0x00 0x00
+# CHECK: fcmp_quad_ballot r85l.cache, neq, r21l, r88l.discard{{ +}}// encoding: [0xa2,0x54,0x6a,0x00,0x0f,0xa2,0x00,0x00]
+0xa2 0x08 0xf0 0x49 0xdd 0xb6 0x00 0x20
+# CHECK: fcmp_quad_ballot r98l.cache, nlt, u184.neg, u202.abs.neg{{ +}}// encoding: [0xa2,0x08,0xf0,0x49,0xdd,0xb6,0x00,0x20]
+
+0x32 0x15 0x86 0x41 0x11 0x1e 0x01 0xc0
+# CHECK: icmp_ballot r37, sgt, u99, u74l{{ +}}// encoding: [0x32,0x15,0x86,0x41,0x11,0x1e,0x01,0xc0]
+0x32 0x08 0xce 0x22 0x1c 0x17 0x01 0x80
+# CHECK: icmp_ballot r34l, seq, r39.discard, u225{{ +}}// encoding: [0x32,0x08,0xce,0x22,0x1c,0x17,0x01,0x80]
+0x32 0x61 0x1c 0x10 0x0d 0x8f 0x01 0x80
+# CHECK: icmp_ballot r24, nseq, 220, r104h.discard{{ +}}// encoding: [0x32,0x61,0x1c,0x10,0x0d,0x8f,0x01,0x80]
+0xb2 0x16 0x34 0xf1 0x00 0x93 0x01 0xc0
+# CHECK: icmp_ballot r37h.cache, slte, u26l, 207{{ +}}// encoding: [0xb2,0x16,0x34,0xf1,0x00,0x93,0x01,0xc0]
+
+0x32 0x72 0x13 0xe0 0x2a 0xba 0x00 0x00
+# CHECK: icmp_quad_ballot r124h, nueq, 147, r87.cache{{ +}}// encoding: [0x32,0x72,0x13,0xe0,0x2a,0xba,0x00,0x00]
+0x32 0x6e 0xe8 0xa1 0x2e 0x91 0x00 0x00
+# CHECK: icmp_quad_ballot r59h, nueq, u148, r53.discard{{ +}}// encoding: [0x32,0x6e,0xe8,0xa1,0x2e,0x91,0x00,0x00]
+0xb2 0x26 0xdd 0xb0 0x17 0x8a 0x00 0x20
+# CHECK: icmp_quad_ballot r9h.cache, ugte, r78h.discard, u221h{{ +}}// encoding: [0xb2,0x26,0xdd,0xb0,0x17,0x8a,0x00,0x20]
+0x32 0x42 0xcb 0x90 0x15 0xbe 0x00 0xc0
+# CHECK: icmp_quad_ballot r112h, slte, r101h.discard, u204h{{ +}}// encoding: [0x32,0x42,0xcb,0x90,0x15,0xbe,0x00,0xc0]
+
+0xef 0x94 0x9f 0x00 0x00 0x38
+# CHECK: quad_and r101l.cache, r79h.cache{{ +}}// encoding: [0xef,0x94,0x9f,0x00,0x00,0x38]
+0xef 0xbd 0xe0 0x03 0x00 0x24
+# CHECK: quad_and r79.cache, r48_r49.discard{{ +}}// encoding: [0xef,0xbd,0xe0,0x03,0x00,0x24]
+0xef 0xec 0xac 0x02 0x00 0x24
+# CHECK: quad_and r91l.cache, r54.cache{{ +}}// encoding: [0xef,0xec,0xac,0x02,0x00,0x24]
+0x6f 0xbd 0x37 0x00 0x00 0x30
+# CHECK: quad_and r111, 55{{ +}}// encoding: [0x6f,0xbd,0x37,0x00,0x00,0x30]
+
+0xef 0xae 0xfc 0x10 0x00 0x3c
+# CHECK: quad_fadd r107h.cache, r126l.discard{{ +}}// encoding: [0xef,0xae,0xfc,0x10,0x00,0x3c]
+0x6f 0xdd 0x6a 0x13 0x00 0x20
+# CHECK: quad_fadd r87, r21_r22{{ +}}// encoding: [0x6f,0xdd,0x6a,0x13,0x00,0x20]
+0x6f 0x9e 0x60 0x10 0x00 0x24
+# CHECK: quad_fadd r71h, r48l{{ +}}// encoding: [0x6f,0x9e,0x60,0x10,0x00,0x24]
+0xef 0xfc 0xb2 0x10 0x00 0x10
+# CHECK: quad_fadd r63l.cache, r25l.cache{{ +}}// encoding: [0xef,0xfc,0xb2,0x10,0x00,0x10]
+
+0xef 0xe5 0xd5 0x10 0x40 0x14
+# CHECK: quad_fmul r57.cache, r42h.discard{{ +}}// encoding: [0xef,0xe5,0xd5,0x10,0x40,0x14]
+0xef 0xc4 0x62 0x10 0x40 0x3c
+# CHECK: quad_fmul r113l.cache, r113l{{ +}}// encoding: [0xef,0xc4,0x62,0x10,0x40,0x3c]
+0x6f 0xa0 0x3c 0x10 0x40 0x18
+# CHECK: quad_fmul r40l, 188{{ +}}// encoding: [0x6f,0xa0,0x3c,0x10,0x40,0x18]
+0xef 0xc1 0xc8 0x11 0x40 0x20
+# CHECK: quad_fmul r80.cache, u132{{ +}}// encoding: [0xef,0xc1,0xc8,0x11,0x40,0x20]
+
+0xef 0xd4 0x01 0x01 0x00 0x80
+# CHECK: quad_iadd r21l.cache, u0h{{ +}}// encoding: [0xef,0xd4,0x01,0x01,0x00,0x80]
+0xef 0xa8 0x51 0x00 0x00 0x94
+# CHECK: quad_iadd r42l.cache, r40h{{ +}}// encoding: [0xef,0xa8,0x51,0x00,0x00,0x94]
+0x6f 0xda 0x7a 0x00 0x00 0x8c
+# CHECK: quad_iadd r22h, r125l{{ +}}// encoding: [0x6f,0xda,0x7a,0x00,0x00,0x8c]
+0x6f 0xb8 0xe3 0x00 0x00 0x90
+# CHECK: quad_iadd r46l, r17h.discard{{ +}}// encoding: [0x6f,0xb8,0xe3,0x00,0x00,0x90]
+
+0x6f 0xa5 0x85 0x10 0xc0 0x0c
+# CHECK: quad_max.f r9, r98h.cache{{ +}}// encoding: [0x6f,0xa5,0x85,0x10,0xc0,0x0c]
+0xef 0xa4 0x42 0x12 0xc0 0x00
+# CHECK: quad_max.f r9l.cache, r1{{ +}}// encoding: [0xef,0xa4,0x42,0x12,0xc0,0x00]
+0x6f 0x96 0x07 0x11 0xc0 0x34
+# CHECK: quad_max.f r101h, u35h{{ +}}// encoding: [0x6f,0x96,0x07,0x11,0xc0,0x34]
+0x6f 0xf5 0x03 0x11 0xc0 0x2c
+# CHECK: quad_max.f r93, u97h{{ +}}// encoding: [0x6f,0xf5,0x03,0x11,0xc0,0x2c]
+
+0xef 0xdc 0x38 0x00 0xc0 0xb0
+# CHECK: quad_max.s r119l.cache, 56{{ +}}// encoding: [0xef,0xdc,0x38,0x00,0xc0,0xb0]
+0x6f 0xca 0x11 0x00 0xc0 0xa8
+# CHECK: quad_max.s r82h, 145{{ +}}// encoding: [0x6f,0xca,0x11,0x00,0xc0,0xa8]
+0x6f 0xc2 0x48 0x03 0xc0 0xb8
+# CHECK: quad_max.s r112h, r68_r69{{ +}}// encoding: [0x6f,0xc2,0x48,0x03,0xc0,0xb8]
+0x6f 0xa1 0xbc 0x02 0xc0 0xb4
+# CHECK: quad_max.s r104, r62.cache{{ +}}// encoding: [0x6f,0xa1,0xbc,0x02,0xc0,0xb4]
+
+0x6f 0xee 0x0b 0x00 0xc0 0x89
+# CHECK: quad_max.u r27h, 139{{ +}}// encoding: [0x6f,0xee,0x0b,0x00,0xc0,0x89]
+0x6f 0xd5 0x4e 0x03 0xc0 0x99
+# CHECK: quad_max.u r53, r71_r72{{ +}}// encoding: [0x6f,0xd5,0x4e,0x03,0xc0,0x99]
+0xef 0xf6 0xf2 0x03 0xc0 0xad
+# CHECK: quad_max.u r93h.cache, r121_r122.discard{{ +}}// encoding: [0xef,0xf6,0xf2,0x03,0xc0,0xad]
+0x6f 0xc5 0xaa 0x02 0xc0 0xad
+# CHECK: quad_max.u r81, r117.cache{{ +}}// encoding: [0x6f,0xc5,0xaa,0x02,0xc0,0xad]
+
+0xef 0xd4 0xbe 0x11 0x80 0x14
+# CHECK: quad_min.f r53l.cache, u63{{ +}}// encoding: [0xef,0xd4,0xbe,0x11,0x80,0x14]
+0xef 0x85 0x92 0x10 0x80 0x3c
+# CHECK: quad_min.f r97.cache, r105l.cache{{ +}}// encoding: [0xef,0x85,0x92,0x10,0x80,0x3c]
+0x6f 0xee 0x24 0x10 0x80 0x0c
+# CHECK: quad_min.f r27h, 228{{ +}}// encoding: [0x6f,0xee,0x24,0x10,0x80,0x0c]
+0xef 0x81 0x5b 0x11 0x80 0x18
+# CHECK: quad_min.f r32.cache, u205h{{ +}}// encoding: [0xef,0x81,0x5b,0x11,0x80,0x18]
+
+0xef 0x88 0x6f 0x00 0x80 0xbc
+# CHECK: quad_min.s r98l.cache, r119h{{ +}}// encoding: [0xef,0x88,0x6f,0x00,0x80,0xbc]
+0xef 0xfa 0x82 0x00 0x80 0x88
+# CHECK: quad_min.s r30h.cache, r65l.cache{{ +}}// encoding: [0xef,0xfa,0x82,0x00,0x80,0x88]
+0xef 0xfd 0x10 0x00 0x80 0xb4
+# CHECK: quad_min.s r127.cache, 80{{ +}}// encoding: [0xef,0xfd,0x10,0x00,0x80,0xb4]
+0xef 0xb9 0x2a 0x00 0x80 0x80
+# CHECK: quad_min.s r14.cache, 42{{ +}}// encoding: [0xef,0xb9,0x2a,0x00,0x80,0x80]
+
+0xef 0x86 0xd2 0x03 0x80 0x8d
+# CHECK: quad_min.u r1h.cache, r105_r106.discard{{ +}}// encoding: [0xef,0x86,0xd2,0x03,0x80,0x8d]
+0xef 0xc2 0xd8 0x00 0x80 0xad
+# CHECK: quad_min.u r80h.cache, r108l.discard{{ +}}// encoding: [0xef,0xc2,0xd8,0x00,0x80,0xad]
+0xef 0xd2 0x1a 0x01 0x80 0xb1
+# CHECK: quad_min.u r116h.cache, u13l{{ +}}// encoding: [0xef,0xd2,0x1a,0x01,0x80,0xb1]
+0xef 0xb6 0x5c 0x03 0x80 0x8d
+# CHECK: quad_min.u r13h.cache, r110_r111{{ +}}// encoding: [0xef,0xb6,0x5c,0x03,0x80,0x8d]
+
+0xef 0xaa 0x04 0x00 0x40 0x24
+# CHECK: quad_or r74h.cache, 68{{ +}}// encoding: [0xef,0xaa,0x04,0x00,0x40,0x24]
+0xef 0x9a 0x84 0x00 0x40 0x24
+# CHECK: quad_or r70h.cache, r34l.cache{{ +}}// encoding: [0xef,0x9a,0x84,0x00,0x40,0x24]
+0xef 0x8d 0x64 0x02 0x40 0x20
+# CHECK: quad_or r67.cache, r18{{ +}}// encoding: [0xef,0x8d,0x64,0x02,0x40,0x20]
+0x6f 0xc4 0xd6 0x03 0x40 0x28
+# CHECK: quad_or r81l, r75_r76.discard{{ +}}// encoding: [0x6f,0xc4,0xd6,0x03,0x40,0x28]
+
+0xef 0xa9 0xb6 0x50 0x00 0x3c
+# CHECK: quad_prefix_fadd r106.cache, r123l.cache{{ +}}// encoding: [0xef,0xa9,0xb6,0x50,0x00,0x3c]
+0xef 0xd1 0x10 0x50 0x00 0x34
+# CHECK: quad_prefix_fadd r116.cache, 80{{ +}}// encoding: [0xef,0xd1,0x10,0x50,0x00,0x34]
+0xef 0x80 0x96 0x52 0x00 0x28
+# CHECK: quad_prefix_fadd r64l.cache, r75.cache{{ +}}// encoding: [0xef,0x80,0x96,0x52,0x00,0x28]
+0x6f 0x80 0xb2 0x51 0x00 0x10
+# CHECK: quad_prefix_fadd r32l, u25{{ +}}// encoding: [0x6f,0x80,0xb2,0x51,0x00,0x10]
+
+0xef 0x84 0x03 0x50 0x40 0x00
+# CHECK: quad_prefix_fmul r1l.cache, 3{{ +}}// encoding: [0xef,0x84,0x03,0x50,0x40,0x00]
+0x6f 0xa6 0x3e 0x51 0x40 0x28
+# CHECK: quad_prefix_fmul r73h, u95l{{ +}}// encoding: [0x6f,0xa6,0x3e,0x51,0x40,0x28]
+0xef 0xb1 0xfc 0x50 0x40 0x00
+# CHECK: quad_prefix_fmul r12.cache, r30l.discard{{ +}}// encoding: [0xef,0xb1,0xfc,0x50,0x40,0x00]
+0xef 0xe9 0x5c 0x51 0x40 0x0c
+# CHECK: quad_prefix_fmul r26.cache, u238l{{ +}}// encoding: [0xef,0xe9,0x5c,0x51,0x40,0x0c]
+
+0xef 0xf8 0x0e 0x40 0x00 0x80
+# CHECK: quad_prefix_iadd r30l.cache, 14{{ +}}// encoding: [0xef,0xf8,0x0e,0x40,0x00,0x80]
+0xef 0x81 0x76 0x40 0x00 0xa0
+# CHECK: quad_prefix_iadd r64.cache, r27l{{ +}}// encoding: [0xef,0x81,0x76,0x40,0x00,0xa0]
+0xef 0x9d 0x68 0x41 0x00 0xa8
+# CHECK: quad_prefix_iadd r71.cache, u212l{{ +}}// encoding: [0xef,0x9d,0x68,0x41,0x00,0xa8]
+0xef 0x84 0xa6 0x40 0x00 0xa0
+# CHECK: quad_prefix_iadd r65l.cache, r19l.cache{{ +}}// encoding: [0xef,0x84,0xa6,0x40,0x00,0xa0]
+
+0xef 0x65 0x9e 0xc3 0x12 0x38
+# CHECK: quad_shuffle r121.cache, r79_r80.cache, u22l{{ +}}// encoding: [0xef,0x65,0x9e,0xc3,0x12,0x38]
+0xef 0x40 0xde 0x83 0x03 0x22
+# CHECK: quad_shuffle r80l.cache, r15_r16.discard, 184{{ +}}// encoding: [0xef,0x40,0xde,0x83,0x03,0x22]
+0x6f 0x0d 0x6c 0x40 0x2e 0x16
+# CHECK: quad_shuffle r35, r54l, r82.discard{{ +}}// encoding: [0x6f,0x0d,0x6c,0x40,0x2e,0x16]
+0x6f 0x3c 0x4c 0x62 0x01 0x1d
+# CHECK: quad_shuffle r47l, r102, 86{{ +}}// encoding: [0x6f,0x3c,0x4c,0x62,0x01,0x1d]
+
+0xef 0x12 0x90 0x81 0xe5 0x31
+# CHECK: quad_shuffle_down r100h.cache, u8, r44{{ +}}// encoding: [0xef,0x12,0x90,0x81,0xe5,0x31]
+0xef 0x59 0x8a 0x93 0xca 0x2e
+# CHECK: quad_shuffle_down r86.cache, r101_r102.cache, r84h.cache{{ +}}// encoding: [0xef,0x59,0x8a,0x93,0xca,0x2e]
+0x6f 0x4d 0xce 0xb3 0xd2 0x1b
+# CHECK: quad_shuffle_down r51, r71_r72.discard, u117h{{ +}}// encoding: [0x6f,0x4d,0xce,0xb3,0xd2,0x1b]
+0x6f 0x5c 0xa0 0x43 0xd0 0x09
+# CHECK: quad_shuffle_down r23l, r80_r81.cache, u34l{{ +}}// encoding: [0x6f,0x5c,0xa0,0x43,0xd0,0x09]
+
+0x6f 0x2c 0xe4 0xc0 0x83 0x87
+# CHECK: quad_shuffle_rotate_up r11l, r50l.discard, 252{{ +}}// encoding: [0x6f,0x2c,0xe4,0xc0,0x83,0x87]
+0xef 0x5d 0xa0 0x51 0x90 0xa4
+# CHECK: quad_shuffle_rotate_up r87.cache, u48, u2h{{ +}}// encoding: [0xef,0x5d,0xa0,0x51,0x90,0xa4]
+0xef 0x19 0x1e 0x50 0x92 0x87
+# CHECK: quad_shuffle_rotate_up r6.cache, 94, u114h{{ +}}// encoding: [0xef,0x19,0x1e,0x50,0x92,0x87]
+0x6f 0x45 0x63 0x00 0x9c 0xa3
+# CHECK: quad_shuffle_rotate_up r81, r17h, u224{{ +}}// encoding: [0x6f,0x45,0x63,0x00,0x9c,0xa3]
+
+0x6f 0x04 0xfe 0x00 0xbc 0x2b
+# CHECK: quad_shuffle_up r65l, r95l.discard, r96_r97.discard{{ +}}// encoding: [0x6f,0x04,0xfe,0x00,0xbc,0x2b]
+0xef 0x45 0x9c 0xb0 0x8c 0x0c
+# CHECK: quad_shuffle_up r17.cache, r110l.cache, r5h.discard{{ +}}// encoding: [0xef,0x45,0x9c,0xb0,0x8c,0x0c]
+0x6f 0x5a 0xee 0x00 0xa6 0x27
+# CHECK: quad_shuffle_up r86h, r55l.discard, r112{{ +}}// encoding: [0x6f,0x5a,0xee,0x00,0xa6,0x27]
+0x6f 0x62 0x63 0xe1 0x83 0x24
+# CHECK: quad_shuffle_up r88h, u177h, 62{{ +}}// encoding: [0x6f,0x62,0x63,0xe1,0x83,0x24]
+
+0x6f 0x35 0x6a 0x83 0x77 0x12
+# CHECK: quad_shuffle_xor r45, r21_r22, r92_r93{{ +}}// encoding: [0x6f,0x35,0x6a,0x83,0x77,0x12]
+0xef 0x4a 0x18 0x40 0x7a 0x1d
+# CHECK: quad_shuffle_xor r50h.cache, 216, r50_r51.cache{{ +}}// encoding: [0xef,0x4a,0x18,0x40,0x7a,0x1d]
+0xef 0x19 0x7e 0xf0 0x4a 0x38
+# CHECK: quad_shuffle_xor r102.cache, r95l, r23h.cache{{ +}}// encoding: [0xef,0x19,0x7e,0xf0,0x4a,0x38]
+0x6f 0x52 0xba 0x63 0x75 0x13
+# CHECK: quad_shuffle_xor r52h, r29_r30.cache, r107_r108{{ +}}// encoding: [0x6f,0x52,0xba,0x63,0x75,0x13]
+
+0x6f 0xb4 0x54 0x01 0x80 0x08
+# CHECK: quad_xor r13l, u202l{{ +}}// encoding: [0x6f,0xb4,0x54,0x01,0x80,0x08]
+0xef 0xa2 0xd4 0x02 0x80 0x3c
+# CHECK: quad_xor r104h.cache, r106.discard{{ +}}// encoding: [0xef,0xa2,0xd4,0x02,0x80,0x3c]
+0x6f 0xc8 0xb2 0x00 0x80 0x00
+# CHECK: quad_xor r18l, r25l.cache{{ +}}// encoding: [0x6f,0xc8,0xb2,0x00,0x80,0x00]
+0xef 0x9d 0x1b 0x00 0x80 0x30
+# CHECK: quad_xor r103.cache, 27{{ +}}// encoding: [0xef,0x9d,0x1b,0x00,0x80,0x30]
+
+0xef 0xfd 0x31 0x21 0x00 0x0c
+# CHECK: simd_and r31.cache, u120h{{ +}}// encoding: [0xef,0xfd,0x31,0x21,0x00,0x0c]
+0xef 0xf6 0xd8 0x21 0x00 0x3c
+# CHECK: simd_and r125h.cache, u236{{ +}}// encoding: [0xef,0xf6,0xd8,0x21,0x00,0x3c]
+0x6f 0xf5 0x26 0x21 0x00 0x18
+# CHECK: simd_and r61, u83l{{ +}}// encoding: [0x6f,0xf5,0x26,0x21,0x00,0x18]
+0x6f 0xc9 0x93 0x20 0x00 0x34
+# CHECK: simd_and r114, r41h.cache{{ +}}// encoding: [0x6f,0xc9,0x93,0x20,0x00,0x34]
+
+0xef 0xc0 0x1b 0x31 0x00 0x38
+# CHECK: simd_fadd r112l.cache, u77h{{ +}}// encoding: [0xef,0xc0,0x1b,0x31,0x00,0x38]
+0xef 0xd5 0x90 0x33 0x00 0x08
+# CHECK: simd_fadd r21.cache, r72_r73.cache{{ +}}// encoding: [0xef,0xd5,0x90,0x33,0x00,0x08]
+0xef 0xca 0xb4 0x32 0x00 0x18
+# CHECK: simd_fadd r50h.cache, r90.cache{{ +}}// encoding: [0xef,0xca,0xb4,0x32,0x00,0x18]
+0x6f 0x8c 0xbc 0x30 0x00 0x00
+# CHECK: simd_fadd r3l, r30l.cache{{ +}}// encoding: [0x6f,0x8c,0xbc,0x30,0x00,0x00]
+
+0x6f 0xca 0x69 0x31 0x40 0x2c
+# CHECK: simd_fmul r82h, u244h{{ +}}// encoding: [0x6f,0xca,0x69,0x31,0x40,0x2c]
+0xef 0xf8 0x39 0x30 0x40 0x14
+# CHECK: simd_fmul r62l.cache, 121{{ +}}// encoding: [0xef,0xf8,0x39,0x30,0x40,0x14]
+0x6f 0xb0 0x76 0x31 0x40 0x20
+# CHECK: simd_fmul r76l, u155l{{ +}}// encoding: [0x6f,0xb0,0x76,0x31,0x40,0x20]
+0xef 0xb5 0x34 0x30 0x40 0x3c
+# CHECK: simd_fmul r109.cache, 244{{ +}}// encoding: [0xef,0xb5,0x34,0x30,0x40,0x3c]
+
+0x6f 0xb8 0xc2 0x22 0x00 0x94
+# CHECK: simd_iadd r46l, r33.discard{{ +}}// encoding: [0x6f,0xb8,0xc2,0x22,0x00,0x94]
+0x6f 0xb5 0x42 0x21 0x00 0x90
+# CHECK: simd_iadd r45, u129l{{ +}}// encoding: [0x6f,0xb5,0x42,0x21,0x00,0x90]
+0x6f 0xec 0x62 0x21 0x00 0xbc
+# CHECK: simd_iadd r123l, u241l{{ +}}// encoding: [0x6f,0xec,0x62,0x21,0x00,0xbc]
+0x6f 0xd8 0x72 0x21 0x00 0xa0
+# CHECK: simd_iadd r86l, u153l{{ +}}// encoding: [0x6f,0xd8,0x72,0x21,0x00,0xa0]
+
+0x6f 0xde 0x61 0x31 0xc0 0x14
+# CHECK: simd_max.f r55h, u176h{{ +}}// encoding: [0x6f,0xde,0x61,0x31,0xc0,0x14]
+0xef 0x98 0x52 0x31 0xc0 0x18
+# CHECK: simd_max.f r38l.cache, u201l{{ +}}// encoding: [0xef,0x98,0x52,0x31,0xc0,0x18]
+0xef 0xdc 0x17 0x30 0xc0 0x00
+# CHECK: simd_max.f r23l.cache, 23{{ +}}// encoding: [0xef,0xdc,0x17,0x30,0xc0,0x00]
+0x6f 0xa0 0xb0 0x33 0xc0 0x10
+# CHECK: simd_max.f r40l, r24_r25.cache{{ +}}// encoding: [0x6f,0xa0,0xb0,0x33,0xc0,0x10]
+
+0x6f 0x92 0xa7 0x20 0xc0 0xb4
+# CHECK: simd_max.s r100h, r51h.cache{{ +}}// encoding: [0x6f,0x92,0xa7,0x20,0xc0,0xb4]
+0xef 0xaa 0x8a 0x22 0xc0 0xa4
+# CHECK: simd_max.s r74h.cache, r37.cache{{ +}}// encoding: [0xef,0xaa,0x8a,0x22,0xc0,0xa4]
+0x6f 0x86 0x9c 0x21 0xc0 0xac
+# CHECK: simd_max.s r65h, u110{{ +}}// encoding: [0x6f,0x86,0x9c,0x21,0xc0,0xac]
+0xef 0xbe 0x0e 0x21 0xc0 0x8c
+# CHECK: simd_max.s r15h.cache, u103l{{ +}}// encoding: [0xef,0xbe,0x0e,0x21,0xc0,0x8c]
+
+0x6f 0xa9 0x81 0x20 0xc0 0xad
+# CHECK: simd_max.u r74, r96h.cache{{ +}}// encoding: [0x6f,0xa9,0x81,0x20,0xc0,0xad]
+0xef 0xd8 0x37 0x20 0xc0 0x89
+# CHECK: simd_max.u r22l.cache, 183{{ +}}// encoding: [0xef,0xd8,0x37,0x20,0xc0,0x89]
+0xef 0x99 0x97 0x20 0xc0 0xbd
+# CHECK: simd_max.u r102.cache, r107h.cache{{ +}}// encoding: [0xef,0x99,0x97,0x20,0xc0,0xbd]
+0x6f 0x9d 0x6e 0x22 0xc0 0xa5
+# CHECK: simd_max.u r71, r55{{ +}}// encoding: [0x6f,0x9d,0x6e,0x22,0xc0,0xa5]
+
+0x6f 0x96 0x19 0x31 0x80 0x00
+# CHECK: simd_min.f r5h, u12h{{ +}}// encoding: [0x6f,0x96,0x19,0x31,0x80,0x00]
+0x6f 0x90 0x7b 0x31 0x80 0x3c
+# CHECK: simd_min.f r100l, u253h{{ +}}// encoding: [0x6f,0x90,0x7b,0x31,0x80,0x3c]
+0xef 0xa4 0xa2 0x33 0x80 0x2c
+# CHECK: simd_min.f r73l.cache, r113_r114.cache{{ +}}// encoding: [0xef,0xa4,0xa2,0x33,0x80,0x2c]
+0xef 0x9d 0xae 0x33 0x80 0x24
+# CHECK: simd_min.f r71.cache, r55_r56.cache{{ +}}// encoding: [0xef,0x9d,0xae,0x33,0x80,0x24]
+
+0x6f 0xde 0xc6 0x23 0x80 0xa8
+# CHECK: simd_min.s r87h, r67_r68.discard{{ +}}// encoding: [0x6f,0xde,0xc6,0x23,0x80,0xa8]
+0xef 0x88 0x59 0x21 0x80 0xa0
+# CHECK: simd_min.s r66l.cache, u140h{{ +}}// encoding: [0xef,0x88,0x59,0x21,0x80,0xa0]
+0x6f 0x82 0x2a 0x21 0x80 0x94
+# CHECK: simd_min.s r32h, u53l{{ +}}// encoding: [0x6f,0x82,0x2a,0x21,0x80,0x94]
+0xef 0xe2 0x47 0x20 0x80 0x8c
+# CHECK: simd_min.s r24h.cache, r99h{{ +}}// encoding: [0xef,0xe2,0x47,0x20,0x80,0x8c]
+
+0xef 0x99 0x48 0x20 0x80 0x8d
+# CHECK: simd_min.u r6.cache, r100l{{ +}}// encoding: [0xef,0x99,0x48,0x20,0x80,0x8d]
+0x6f 0xa6 0x5a 0x20 0x80 0x81
+# CHECK: simd_min.u r9h, r13l{{ +}}// encoding: [0x6f,0xa6,0x5a,0x20,0x80,0x81]
+0x6f 0xd4 0xe0 0x20 0x80 0xa9
+# CHECK: simd_min.u r85l, r80l.discard{{ +}}// encoding: [0x6f,0xd4,0xe0,0x20,0x80,0xa9]
+0x6f 0xa9 0x2a 0x21 0x80 0xb5
+# CHECK: simd_min.u r106, u53l{{ +}}// encoding: [0x6f,0xa9,0x2a,0x21,0x80,0xb5]
+
+0xef 0xc5 0x62 0x22 0x40 0x18
+# CHECK: simd_or r49.cache, r81{{ +}}// encoding: [0xef,0xc5,0x62,0x22,0x40,0x18]
+0x6f 0xe1 0x76 0x21 0x40 0x1c
+# CHECK: simd_or r56, u251l{{ +}}// encoding: [0x6f,0xe1,0x76,0x21,0x40,0x1c]
+0x6f 0xf8 0xa5 0x20 0x40 0x1c
+# CHECK: simd_or r62l, r114h.cache{{ +}}// encoding: [0x6f,0xf8,0xa5,0x20,0x40,0x1c]
+0x6f 0xfc 0xb4 0x21 0x40 0x30
+# CHECK: simd_or r127l, u26{{ +}}// encoding: [0x6f,0xfc,0xb4,0x21,0x40,0x30]
+
+0x6f 0xfa 0xc2 0x60 0x00 0x08
+# CHECK: simd_prefix_and r30h, r65l.discard{{ +}}// encoding: [0x6f,0xfa,0xc2,0x60,0x00,0x08]
+0x6f 0xf2 0x91 0x60 0x00 0x34
+# CHECK: simd_prefix_and r124h, r40h.cache{{ +}}// encoding: [0x6f,0xf2,0x91,0x60,0x00,0x34]
+0x6f 0xb0 0xa4 0x63 0x00 0x08
+# CHECK: simd_prefix_and r12l, r82_r83.cache{{ +}}// encoding: [0x6f,0xb0,0xa4,0x63,0x00,0x08]
+0x6f 0xa1 0xcc 0x60 0x00 0x14
+# CHECK: simd_prefix_and r40, r38l.discard{{ +}}// encoding: [0x6f,0xa1,0xcc,0x60,0x00,0x14]
+
+0x6f 0xec 0xee 0x70 0x00 0x3c
+# CHECK: simd_prefix_fadd r123l, r119l.discard{{ +}}// encoding: [0x6f,0xec,0xee,0x70,0x00,0x3c]
+0xef 0xf9 0x5a 0x73 0x00 0x34
+# CHECK: simd_prefix_fadd r126.cache, r45_r46{{ +}}// encoding: [0xef,0xf9,0x5a,0x73,0x00,0x34]
+0x6f 0xcd 0xe6 0x70 0x00 0x24
+# CHECK: simd_prefix_fadd r83, r51l.discard{{ +}}// encoding: [0x6f,0xcd,0xe6,0x70,0x00,0x24]
+0xef 0xdd 0x08 0x71 0x00 0x08
+# CHECK: simd_prefix_fadd r23.cache, u68l{{ +}}// encoding: [0xef,0xdd,0x08,0x71,0x00,0x08]
+
+0x6f 0xc5 0xdf 0x70 0x40 0x14
+# CHECK: simd_prefix_fmul r49, r47h.discard{{ +}}// encoding: [0x6f,0xc5,0xdf,0x70,0x40,0x14]
+0xef 0xf6 0x1d 0x70 0x40 0x1c
+# CHECK: simd_prefix_fmul r61h.cache, 221{{ +}}// encoding: [0xef,0xf6,0x1d,0x70,0x40,0x1c]
+0x6f 0x89 0xbc 0x73 0x40 0x24
+# CHECK: simd_prefix_fmul r66, r62_r63.cache{{ +}}// encoding: [0x6f,0x89,0xbc,0x73,0x40,0x24]
+0x6f 0x85 0x42 0x72 0x40 0x10
+# CHECK: simd_prefix_fmul r33, r1{{ +}}// encoding: [0x6f,0x85,0x42,0x72,0x40,0x10]
+
+0xef 0xfc 0xaa 0x63 0x00 0xa8
+# CHECK: simd_prefix_iadd r95l.cache, r85_r86.cache{{ +}}// encoding: [0xef,0xfc,0xaa,0x63,0x00,0xa8]
+0x6f 0x9c 0x9e 0x60 0x00 0x94
+# CHECK: simd_prefix_iadd r39l, r47l.cache{{ +}}// encoding: [0x6f,0x9c,0x9e,0x60,0x00,0x94]
+0x6f 0x9e 0xf6 0x61 0x00 0x80
+# CHECK: simd_prefix_iadd r7h, u155{{ +}}// encoding: [0x6f,0x9e,0xf6,0x61,0x00,0x80]
+0xef 0xad 0x3e 0x61 0x00 0xb4
+# CHECK: simd_prefix_iadd r107.cache, u63l{{ +}}// encoding: [0xef,0xad,0x3e,0x61,0x00,0xb4]
+
+0xef 0xde 0xa4 0x60 0xc0 0xa8
+# CHECK: simd_prefix_max.s r87h.cache, r82l.cache{{ +}}// encoding: [0xef,0xde,0xa4,0x60,0xc0,0xa8]
+0x6f 0x98 0x56 0x62 0xc0 0x9c
+# CHECK: simd_prefix_max.s r38l, r107{{ +}}// encoding: [0x6f,0x98,0x56,0x62,0xc0,0x9c]
+0x6f 0xe4 0x9a 0x61 0xc0 0xbc
+# CHECK: simd_prefix_max.s r121l, u109{{ +}}// encoding: [0x6f,0xe4,0x9a,0x61,0xc0,0xbc]
+0x6f 0xfe 0xb8 0x60 0xc0 0x90
+# CHECK: simd_prefix_max.s r63h, r28l.cache{{ +}}// encoding: [0x6f,0xfe,0xb8,0x60,0xc0,0x90]
+
+0xef 0xc9 0xf8 0x61 0xc0 0xb9
+# CHECK: simd_prefix_max.u r114.cache, u220{{ +}}// encoding: [0xef,0xc9,0xf8,0x61,0xc0,0xb9]
+0xef 0x81 0x75 0x61 0xc0 0x91
+# CHECK: simd_prefix_max.u r32.cache, u154h{{ +}}// encoding: [0xef,0x81,0x75,0x61,0xc0,0x91]
+0x6f 0xee 0x66 0x62 0xc0 0xbd
+# CHECK: simd_prefix_max.u r123h, r115{{ +}}// encoding: [0x6f,0xee,0x66,0x62,0xc0,0xbd]
+0xef 0xd2 0x7e 0x60 0xc0 0x99
+# CHECK: simd_prefix_max.u r52h.cache, r95l{{ +}}// encoding: [0xef,0xd2,0x7e,0x60,0xc0,0x99]
+
+0xef 0x84 0xb5 0x60 0x80 0x88
+# CHECK: simd_prefix_min.s r1l.cache, r90h.cache{{ +}}// encoding: [0xef,0x84,0xb5,0x60,0x80,0x88]
+0x6f 0x8c 0x99 0x60 0x80 0xa0
+# CHECK: simd_prefix_min.s r67l, r12h.cache{{ +}}// encoding: [0x6f,0x8c,0x99,0x60,0x80,0xa0]
+0x6f 0xbd 0xfd 0x60 0x80 0x80
+# CHECK: simd_prefix_min.s r15, r30h.discard{{ +}}// encoding: [0x6f,0xbd,0xfd,0x60,0x80,0x80]
+0x6f 0x85 0xf0 0x60 0x80 0x8c
+# CHECK: simd_prefix_min.s r1, r120l.discard{{ +}}// encoding: [0x6f,0x85,0xf0,0x60,0x80,0x8c]
+
+0x6f 0x86 0xe4 0x61 0x80 0xbd
+# CHECK: simd_prefix_min.u r97h, u242{{ +}}// encoding: [0x6f,0x86,0xe4,0x61,0x80,0xbd]
+0xef 0x8d 0x10 0x60 0x80 0xa9
+# CHECK: simd_prefix_min.u r67.cache, 144{{ +}}// encoding: [0xef,0x8d,0x10,0x60,0x80,0xa9]
+0xef 0xa8 0x59 0x61 0x80 0xa1
+# CHECK: simd_prefix_min.u r74l.cache, u140h{{ +}}// encoding: [0xef,0xa8,0x59,0x61,0x80,0xa1]
+0xef 0xde 0x28 0x60 0x80 0x8d
+# CHECK: simd_prefix_min.u r23h.cache, 232{{ +}}// encoding: [0xef,0xde,0x28,0x60,0x80,0x8d]
+
+0xef 0xb4 0xb8 0x61 0x40 0x28
+# CHECK: simd_prefix_or r77l.cache, u92{{ +}}// encoding: [0xef,0xb4,0xb8,0x61,0x40,0x28]
+0x6f 0x80 0x74 0x61 0x40 0x08
+# CHECK: simd_prefix_or r0l, u218l{{ +}}// encoding: [0x6f,0x80,0x74,0x61,0x40,0x08]
+0x6f 0xc6 0x75 0x60 0x40 0x10
+# CHECK: simd_prefix_or r49h, r26h{{ +}}// encoding: [0x6f,0xc6,0x75,0x60,0x40,0x10]
+0xef 0xb6 0x0e 0x60 0x40 0x0c
+# CHECK: simd_prefix_or r13h.cache, 206{{ +}}// encoding: [0xef,0xb6,0x0e,0x60,0x40,0x0c]
+
+0xef 0xa2 0x3c 0x61 0x80 0x18
+# CHECK: simd_prefix_xor r40h.cache, u94l{{ +}}// encoding: [0xef,0xa2,0x3c,0x61,0x80,0x18]
+0x6f 0xc5 0x61 0x61 0x80 0x20
+# CHECK: simd_prefix_xor r81, u144h{{ +}}// encoding: [0x6f,0xc5,0x61,0x61,0x80,0x20]
+0xef 0xcc 0x18 0x60 0x80 0x14
+# CHECK: simd_prefix_xor r51l.cache, 88{{ +}}// encoding: [0xef,0xcc,0x18,0x60,0x80,0x14]
+0x6f 0xf6 0x79 0x61 0x80 0x14
+# CHECK: simd_prefix_xor r61h, u188h{{ +}}// encoding: [0x6f,0xf6,0x79,0x61,0x80,0x14]
+
+0xef 0x6c 0x4b 0x84 0x09 0x1c
+# CHECK: simd_shuffle r59l.cache, r101h, r12l.cache{{ +}}// encoding: [0xef,0x6c,0x4b,0x84,0x09,0x1c]
+0x6f 0x0c 0x05 0x25 0x0c 0x0b
+# CHECK: simd_shuffle r3l, u66h, r97l.discard{{ +}}// encoding: [0x6f,0x0c,0x05,0x25,0x0c,0x0b]
+0x6f 0x75 0x7c 0x24 0x0d 0x06
+# CHECK: simd_shuffle r29, r62l, r73l.discard{{ +}}// encoding: [0x6f,0x75,0x7c,0x24,0x0d,0x06]
+0x6f 0x10 0x0e 0x54 0x0b 0x3e
+# CHECK: simd_shuffle r100l, 206, r90h.cache{{ +}}// encoding: [0x6f,0x10,0x0e,0x54,0x0b,0x3e]
+
+0x6f 0x1d 0x6c 0x74 0xd3 0x3c
+# CHECK: simd_shuffle_down r103, r118l, u27h{{ +}}// encoding: [0x6f,0x1d,0x6c,0x74,0xd3,0x3c]
+0xef 0x6e 0x17 0x94 0xd6 0x3c
+# CHECK: simd_shuffle_down r123h.cache, 215, u148h{{ +}}// encoding: [0xef,0x6e,0x17,0x94,0xd6,0x3c]
+0xef 0x41 0x40 0xc6 0xc0 0x0d
+# CHECK: simd_shuffle_down r16.cache, r96, 76{{ +}}// encoding: [0xef,0x41,0x40,0xc6,0xc0,0x0d]
+0xef 0x6d 0xd3 0xd4 0xd6 0x16
+# CHECK: simd_shuffle_down r59.cache, r41h.discard, u214h{{ +}}// encoding: [0xef,0x6d,0xd3,0xd4,0xd6,0x16]
+
+0x6f 0x2a 0xe6 0x66 0x96 0xb9
+# CHECK: simd_shuffle_rotate_up r106h, r83.discard, u179l{{ +}}// encoding: [0x6f,0x2a,0xe6,0x66,0x96,0xb9]
+0x6f 0x28 0xfc 0x36 0x8c 0x81
+# CHECK: simd_shuffle_rotate_up r10l, r30.discard, r33h.discard{{ +}}// encoding: [0x6f,0x28,0xfc,0x36,0x8c,0x81]
+0xef 0x2e 0xbf 0x24 0x8c 0x96
+# CHECK: simd_shuffle_rotate_up r43h.cache, r63h.cache, r65l.discard{{ +}}// encoding: [0xef,0x2e,0xbf,0x24,0x8c,0x96]
+0x6f 0x7e 0x4b 0x05 0x9d 0x8e
+# CHECK: simd_shuffle_rotate_up r31h, u229h, u200{{ +}}// encoding: [0x6f,0x7e,0x4b,0x05,0x9d,0x8e]
+
+0x6f 0x19 0x15 0x74 0x8b 0x3b
+# CHECK: simd_shuffle_up r102, 149, r123h.cache{{ +}}// encoding: [0x6f,0x19,0x15,0x74,0x8b,0x3b]
+0xef 0x1e 0x28 0x85 0x95 0x3f
+# CHECK: simd_shuffle_up r103h.cache, u116l, u236l{{ +}}// encoding: [0xef,0x1e,0x28,0x85,0x95,0x3f]
+0x6f 0x7e 0x73 0xe5 0x8c 0x2e
+# CHECK: simd_shuffle_up r95h, u249h, r71l.discard{{ +}}// encoding: [0x6f,0x7e,0x73,0xe5,0x8c,0x2e]
+0xef 0x16 0x6b 0x04 0x8f 0x0e
+# CHECK: simd_shuffle_up r5h.cache, r117h, r88l.discard{{ +}}// encoding: [0xef,0x16,0x6b,0x04,0x8f,0x0e]
+
+0xef 0x15 0x70 0x25 0x4f 0x3f
+# CHECK: simd_shuffle_xor r101.cache, u248l, r121l.discard{{ +}}// encoding: [0xef,0x15,0x70,0x25,0x4f,0x3f]
+0xef 0x42 0xe0 0xe6 0x50 0x38
+# CHECK: simd_shuffle_xor r112h.cache, r80.discard, u7l{{ +}}// encoding: [0xef,0x42,0xe0,0xe6,0x50,0x38]
+0x6f 0x1e 0x3a 0xc5 0x5a 0x18
+# CHECK: simd_shuffle_xor r39h, u93l, u22{{ +}}// encoding: [0x6f,0x1e,0x3a,0xc5,0x5a,0x18]
+0x6f 0x18 0x6c 0x04 0x51 0x19
+# CHECK: simd_shuffle_xor r38l, r86l, u40l{{ +}}// encoding: [0x6f,0x18,0x6c,0x04,0x51,0x19]
+
+0xef 0xd4 0xc2 0x21 0x80 0x24
+# CHECK: simd_xor r85l.cache, u161{{ +}}// encoding: [0xef,0xd4,0xc2,0x21,0x80,0x24]
+0xef 0x8c 0xea 0x22 0x80 0x2c
+# CHECK: simd_xor r67l.cache, r117.discard{{ +}}// encoding: [0xef,0x8c,0xea,0x22,0x80,0x2c]
+0xef 0xc5 0xee 0x22 0x80 0x28
+# CHECK: simd_xor r81.cache, r87.discard{{ +}}// encoding: [0xef,0xc5,0xee,0x22,0x80,0x28]
+0xef 0xa5 0x47 0x20 0x80 0x1c
+# CHECK: simd_xor r41.cache, r99h{{ +}}// encoding: [0xef,0xa5,0x47,0x20,0x80,0x1c]
+
diff --git a/llvm/test/MC/Disassembler/AGX/lit.local.cfg b/llvm/test/MC/Disassembler/AGX/lit.local.cfg
new file mode 100644
index 0000000000000..e7054032f0deb
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AGX/lit.local.cfg
@@ -0,0 +1,2 @@
+if not "AGX" in config.root.targets:
+ config.unsupported = True
>From 8a473f42142db0f6181455ee2e73ad0daed3b7d7 Mon Sep 17 00:00:00 2001
From: Claude <noreply at anthropic.com>
Date: Sun, 18 Oct 2026 20:10:55 +0000
Subject: [PATCH 03/22] [AGX] Fix code generation issues found by testing; add
CodeGen tests
* Emit pop_exec at the start of its block: scheduling could move pure
instructions of a reconvergence block above it, so that they ran with
no thread active (SILowerControlFlow does the same).
* Lay blocks out in the structurizer's order and disable machine block
placement, so that constructs fall through as in Mesa's output.
* Flag the load and store fragments as loads and stores so that their
address space checks are emitted: private accesses were selected as
threadgroup accesses.
* The device memory index signedness field is 1 for zero extension.
* Keep 64-bit shifts by constants until selection so that scaled indices
fold into addresses and iadd; custom lower 64-bit multiplies.
* Fold uniform registers through subregister copies; expand memory
intrinsics inline.
* Teach update_llc_test_checks.py about AGX.
Generated code was cross-checked against the applegpu disassembler.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
Claude-Session: https://claude.ai/code/session_01E56KEecg6edNaS23myRWER
---
.../lib/Target/AGX/AGXAnnotateControlFlow.cpp | 21 ++
llvm/lib/Target/AGX/AGXAsmPrinter.cpp | 2 +-
llvm/lib/Target/AGX/AGXFoldOperands.cpp | 95 +++---
llvm/lib/Target/AGX/AGXISelLowering.cpp | 37 ++-
llvm/lib/Target/AGX/AGXISelLowering.h | 1 +
llvm/lib/Target/AGX/AGXInstrInfo.td | 6 +-
llvm/lib/Target/AGX/AGXInstructions.td | 16 -
llvm/lib/Target/AGX/AGXLowerControlFlow.cpp | 14 +-
llvm/lib/Target/AGX/AGXOperands.td | 3 +-
llvm/lib/Target/AGX/AGXRegisterInfo.td | 4 -
llvm/lib/Target/AGX/AGXTargetMachine.cpp | 4 +
llvm/test/CodeGen/AGX/atomics.ll | 147 +++++++++
llvm/test/CodeGen/AGX/control-flow.ll | 310 ++++++++++++++++++
llvm/test/CodeGen/AGX/errors.ll | 12 +
llvm/test/CodeGen/AGX/fp.ll | 284 ++++++++++++++++
llvm/test/CodeGen/AGX/i64.ll | 176 ++++++++++
llvm/test/CodeGen/AGX/kernel-args.ll | 68 ++++
llvm/test/CodeGen/AGX/lit.local.cfg | 2 +
llvm/test/CodeGen/AGX/memory.ll | 204 ++++++++++++
llvm/test/CodeGen/AGX/simd.ll | 196 +++++++++++
llvm/utils/UpdateTestChecks/asm.py | 19 ++
21 files changed, 1556 insertions(+), 65 deletions(-)
create mode 100644 llvm/test/CodeGen/AGX/atomics.ll
create mode 100644 llvm/test/CodeGen/AGX/control-flow.ll
create mode 100644 llvm/test/CodeGen/AGX/errors.ll
create mode 100644 llvm/test/CodeGen/AGX/fp.ll
create mode 100644 llvm/test/CodeGen/AGX/i64.ll
create mode 100644 llvm/test/CodeGen/AGX/kernel-args.ll
create mode 100644 llvm/test/CodeGen/AGX/lit.local.cfg
create mode 100644 llvm/test/CodeGen/AGX/memory.ll
create mode 100644 llvm/test/CodeGen/AGX/simd.ll
diff --git a/llvm/lib/Target/AGX/AGXAnnotateControlFlow.cpp b/llvm/lib/Target/AGX/AGXAnnotateControlFlow.cpp
index 6c53385af1221..2f1d150975d90 100644
--- a/llvm/lib/Target/AGX/AGXAnnotateControlFlow.cpp
+++ b/llvm/lib/Target/AGX/AGXAnnotateControlFlow.cpp
@@ -25,6 +25,7 @@
#include "AGX.h"
#include "llvm/ADT/DepthFirstIterator.h"
+#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Dominators.h"
@@ -65,6 +66,7 @@ class AGXAnnotateControlFlowImpl {
bool insertElse(CondBrInst *Term);
bool handleLoop(CondBrInst *Term);
bool closeControlFlow(BasicBlock *BB);
+ bool layoutBlocks();
CallInst *createCall(IRBuilder<> &IRB, Intrinsic::ID ID,
ArrayRef<Value *> Args = {}) {
@@ -248,5 +250,24 @@ bool AGXAnnotateControlFlowImpl::run() {
if (!Stack.empty())
report_fatal_error("AGX: failed to annotate the control flow graph (it "
"is not structured)");
+ Changed |= layoutBlocks();
+ return Changed;
+}
+
+/// Execution mask code runs each construct in order, falling through from
+/// the then part into the flow block and from the loop body into the latch.
+/// The reverse post-order of the structured graph is that order; it becomes
+/// the block layout, which machine block placement leaves alone.
+bool AGXAnnotateControlFlowImpl::layoutBlocks() {
+ ReversePostOrderTraversal<Function *> RPOT(&F);
+ bool Changed = false;
+ BasicBlock *Prev = nullptr;
+ for (BasicBlock *BB : RPOT) {
+ if (Prev && Prev->getNextNode() != BB) {
+ BB->moveAfter(Prev);
+ Changed = true;
+ }
+ Prev = BB;
+ }
return Changed;
}
diff --git a/llvm/lib/Target/AGX/AGXAsmPrinter.cpp b/llvm/lib/Target/AGX/AGXAsmPrinter.cpp
index 9c067049c8cb4..f5ff5699995e4 100644
--- a/llvm/lib/Target/AGX/AGXAsmPrinter.cpp
+++ b/llvm/lib/Target/AGX/AGXAsmPrinter.cpp
@@ -87,7 +87,7 @@ void AGXAsmPrinter::emitFunctionBodyStart() {
Twine(getGPRCount(*MF)) + " x 32-bit");
OutStreamer->emitRawComment(" threadgroup memory: " +
Twine(MFI->getThreadgroupSize()) + " bytes");
- OutStreamer->emitRawComment(" stack: " +
+ OutStreamer->emitRawComment(" stack size: " +
Twine(AGXFrameLowering::getStackAdjustUnits(*MF) *
4) +
" bytes");
diff --git a/llvm/lib/Target/AGX/AGXFoldOperands.cpp b/llvm/lib/Target/AGX/AGXFoldOperands.cpp
index e2baf7442091e..a54e97a198661 100644
--- a/llvm/lib/Target/AGX/AGXFoldOperands.cpp
+++ b/llvm/lib/Target/AGX/AGXFoldOperands.cpp
@@ -63,49 +63,68 @@ bool AGXFoldOperands::runOnMachineFunction(MachineFunction &MF) {
const AGXRegisterInfo *TRI = ST.getRegisterInfo();
MachineRegisterInfo &MRI = MF.getRegInfo();
+ auto IsUniformCopy = [&](const MachineInstr &MI) {
+ if (!MI.isCopy())
+ return false;
+ Register Dst = MI.getOperand(0).getReg();
+ Register Src = MI.getOperand(1).getReg();
+ return Dst.isVirtual() && Src.isPhysical() &&
+ !MI.getOperand(1).getSubReg() &&
+ AGX::getRegInfo(Src, *TRI).isUniform();
+ };
+
+ SmallVector<MachineInstr *, 32> Worklist;
+ for (MachineBasicBlock &MBB : MF)
+ for (MachineInstr &MI : MBB)
+ if (IsUniformCopy(MI))
+ Worklist.push_back(&MI);
+
bool Changed = false;
- for (MachineBasicBlock &MBB : MF) {
- for (MachineInstr &Copy : make_early_inc_range(MBB)) {
- if (!Copy.isCopy())
- continue;
- Register Dst = Copy.getOperand(0).getReg();
- Register Src = Copy.getOperand(1).getReg();
- if (!Dst.isVirtual() || !Src.isPhysical() ||
- Copy.getOperand(1).getSubReg())
- continue;
- AGX::RegInfo SrcInfo = AGX::getRegInfo(Src, *TRI);
- if (!SrcInfo.isUniform())
+ while (!Worklist.empty()) {
+ MachineInstr &Copy = *Worklist.pop_back_val();
+ Register Dst = Copy.getOperand(0).getReg();
+ Register Src = Copy.getOperand(1).getReg();
+
+ for (MachineOperand &Use :
+ make_early_inc_range(MRI.use_nodbg_operands(Dst))) {
+ MachineInstr &UseMI = *Use.getParent();
+ MCRegister Folded = Src;
+ if (Use.getSubReg())
+ Folded = TRI->getSubReg(Src, Use.getSubReg());
+ if (!Folded)
continue;
- for (MachineOperand &Use :
- make_early_inc_range(MRI.use_nodbg_operands(Dst))) {
- MachineInstr &UseMI = *Use.getParent();
- if (UseMI.isCopy() || UseMI.isPHI() || UseMI.isRegSequence() ||
- UseMI.isInlineAsm())
- continue;
- unsigned OpIdx = UseMI.getOperandNo(&Use);
- if (OpIdx >= UseMI.getDesc().getNumOperands())
- continue;
- const TargetRegisterClass *RC = TII->getRegClass(UseMI.getDesc(),
- OpIdx);
- MCRegister Folded = Src;
- if (Use.getSubReg())
- Folded = TRI->getSubReg(Src, Use.getSubReg());
- if (!Folded || !RC || !RC->contains(Folded))
- continue;
- if (RC == &AGX::BASE64RegClass &&
- AGX::getRegInfo(Folded, *TRI).HalfIndex >= MaxUniformBaseHalf)
- continue;
- Use.setReg(Folded);
- Use.setSubReg(0);
- Use.setIsKill(false);
- Changed = true;
+ // Copies of parts of a uniform tuple become uniform copies themselves.
+ if (UseMI.isCopy()) {
+ if (Use.getSubReg() && UseMI.getOperand(0).getReg().isVirtual()) {
+ Use.setReg(Folded);
+ Use.setSubReg(0);
+ Use.setIsKill(false);
+ Worklist.push_back(&UseMI);
+ Changed = true;
+ }
+ continue;
}
+ if (UseMI.isPHI() || UseMI.isRegSequence() || UseMI.isInlineAsm())
+ continue;
+ unsigned OpIdx = UseMI.getOperandNo(&Use);
+ if (OpIdx >= UseMI.getDesc().getNumOperands())
+ continue;
+ const TargetRegisterClass *RC = TII->getRegClass(UseMI.getDesc(), OpIdx);
+ if (!RC || !RC->contains(Folded))
+ continue;
+ if (RC == &AGX::BASE64RegClass &&
+ AGX::getRegInfo(Folded, *TRI).HalfIndex >= MaxUniformBaseHalf)
+ continue;
+ Use.setReg(Folded);
+ Use.setSubReg(0);
+ Use.setIsKill(false);
+ Changed = true;
+ }
- if (MRI.use_nodbg_empty(Dst)) {
- Copy.eraseFromParent();
- Changed = true;
- }
+ if (MRI.use_nodbg_empty(Dst)) {
+ Copy.eraseFromParent();
+ Changed = true;
}
}
return Changed;
diff --git a/llvm/lib/Target/AGX/AGXISelLowering.cpp b/llvm/lib/Target/AGX/AGXISelLowering.cpp
index e307cf95c204f..32513fa7fbc05 100644
--- a/llvm/lib/Target/AGX/AGXISelLowering.cpp
+++ b/llvm/lib/Target/AGX/AGXISelLowering.cpp
@@ -102,7 +102,8 @@ AGXTargetLowering::AGXTargetLowering(const TargetMachine &TM,
setOperationAction(ISD::BITREVERSE, MVT::i16, Promote);
setOperationAction({ISD::CTPOP, ISD::CTLZ, ISD::CTTZ}, MVT::i16, Promote);
setOperationAction({ISD::CTLZ, ISD::CTTZ}, MVT::i64, Custom);
- setOperationAction({ISD::SHL, ISD::SRL, ISD::SRA}, MVT::i64, Custom);
+ setOperationAction({ISD::SHL, ISD::SRL, ISD::SRA, ISD::MUL}, MVT::i64,
+ Custom);
setOperationAction(ISD::SETCC, MVT::i64, Custom);
setOperationAction({ISD::UADDSAT, ISD::USUBSAT, ISD::SADDSAT,
ISD::SSUBSAT},
@@ -156,6 +157,12 @@ AGXTargetLowering::AGXTargetLowering(const TargetMachine &TM,
setTruncStoreAction(MVT::f32, MVT::f16, Expand);
setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i32, Expand);
+
+ // There is no library to call: memory intrinsics that AGXCodeGenPrepare
+ // leaves alone are always expanded into loads and stores.
+ MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = ~0U;
+ MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = ~0U;
+ MaxStoresPerMemset = MaxStoresPerMemsetOptSize = ~0U;
}
EVT AGXTargetLowering::getSetCCResultType(const DataLayout &DL,
@@ -348,6 +355,8 @@ SDValue AGXTargetLowering::LowerOperation(SDValue Op,
case ISD::SRL:
case ISD::SRA:
return lowerShift64(Op, DAG);
+ case ISD::MUL:
+ return lowerMUL64(Op, DAG);
case ISD::CTLZ:
case ISD::CTTZ:
return lowerCTLZ_CTTZ64(Op, DAG);
@@ -626,6 +635,32 @@ SDValue AGXTargetLowering::lowerShift64(SDValue Op, SelectionDAG &DAG) const {
return join64(DAG, DL, NewLo, NewHi);
}
+SDValue AGXTargetLowering::lowerMUL64(SDValue Op, SelectionDAG &DAG) const {
+ SDLoc DL(Op);
+ SDValue A = Op.getOperand(0);
+ SDValue B = Op.getOperand(1);
+ auto IsExt32 = [](SDValue V, unsigned Opc) {
+ return V.getOpcode() == Opc && V.getOperand(0).getValueType() == MVT::i32;
+ };
+ // 32 x 32 -> 64-bit products are a single imadd.
+ if ((IsExt32(A, ISD::ZERO_EXTEND) && IsExt32(B, ISD::ZERO_EXTEND)) ||
+ (IsExt32(A, ISD::SIGN_EXTEND) && IsExt32(B, ISD::SIGN_EXTEND)))
+ return Op;
+
+ // lo(a) * lo(b) as a 64-bit product, plus the cross products in the high
+ // half.
+ auto [ALo, AHi] = split64(DAG, DL, A);
+ auto [BLo, BHi] = split64(DAG, DL, B);
+ SDValue P = DAG.getNode(ISD::MUL, DL, MVT::i64,
+ DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, ALo),
+ DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, BLo));
+ auto [PLo, PHi] = split64(DAG, DL, P);
+ SDValue Cross = DAG.getNode(ISD::ADD, DL, MVT::i32,
+ DAG.getNode(ISD::MUL, DL, MVT::i32, ALo, BHi),
+ DAG.getNode(ISD::MUL, DL, MVT::i32, AHi, BLo));
+ return join64(DAG, DL, PLo, DAG.getNode(ISD::ADD, DL, MVT::i32, PHi, Cross));
+}
+
SDValue AGXTargetLowering::lowerCTLZ_CTTZ64(SDValue Op,
SelectionDAG &DAG) const {
SDLoc DL(Op);
diff --git a/llvm/lib/Target/AGX/AGXISelLowering.h b/llvm/lib/Target/AGX/AGXISelLowering.h
index 98efb397e4f89..5dff124131a95 100644
--- a/llvm/lib/Target/AGX/AGXISelLowering.h
+++ b/llvm/lib/Target/AGX/AGXISelLowering.h
@@ -30,6 +30,7 @@ class AGXTargetLowering final : public TargetLowering {
SDValue lowerBRCOND(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerSETCC64(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerShift64(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerMUL64(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerCTLZ_CTTZ64(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFDIV(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFREM(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/AGX/AGXInstrInfo.td b/llvm/lib/Target/AGX/AGXInstrInfo.td
index b744492c1598b..43636df80af73 100644
--- a/llvm/lib/Target/AGX/AGXInstrInfo.td
+++ b/llvm/lib/Target/AGX/AGXInstrInfo.td
@@ -81,13 +81,17 @@ defvar DeviceAS = [0, 1, 4];
defvar LocalAS = [3];
defvar PrivateAS = [5];
+// The address space check is only emitted for fragments flagged as loads or
+// stores.
multiclass AGXLoadFrags<list<int> AS> {
- let AddressSpaces = AS in {
+ let AddressSpaces = AS, IsLoad = 1 in {
def _load : PatFrag<(ops node:$ptr), (load node:$ptr)>;
def _extloadi8 : PatFrag<(ops node:$ptr), (extloadi8 node:$ptr)>;
def _zextloadi8 : PatFrag<(ops node:$ptr), (zextloadi8 node:$ptr)>;
def _extloadi16 : PatFrag<(ops node:$ptr), (extloadi16 node:$ptr)>;
def _zextloadi16 : PatFrag<(ops node:$ptr), (zextloadi16 node:$ptr)>;
+ }
+ let AddressSpaces = AS, IsStore = 1 in {
def _store : PatFrag<(ops node:$val, node:$ptr), (store node:$val, node:$ptr)>;
def _truncstorei8 : PatFrag<(ops node:$val, node:$ptr),
(truncstorei8 node:$val, node:$ptr)>;
diff --git a/llvm/lib/Target/AGX/AGXInstructions.td b/llvm/lib/Target/AGX/AGXInstructions.td
index cb794eccba301..d03153fdc4a41 100644
--- a/llvm/lib/Target/AGX/AGXInstructions.td
+++ b/llvm/lib/Target/AGX/AGXInstructions.td
@@ -322,22 +322,6 @@ def : Pat<(i32 (mulhs i32:$a, i32:$b)),
(EXTRACT_SUBREG (IMADD_I64_FROM32 MOD_SX, $a, MOD_SX, $b, 0,
(i64 0), 0), sub1)>;
-// 64-bit multiply from 32-bit pieces.
-def : Pat<(i64 (mul i64:$a, i64:$b)),
- (REG_SEQUENCE GPR64,
- (EXTRACT_SUBREG
- (IMADD_I64_FROM32 0, (EXTRACT_SUBREG $a, sub0),
- 0, (EXTRACT_SUBREG $b, sub0), 0, (i64 0), 0),
- sub0), sub0,
- (IMADD_I32 0, (EXTRACT_SUBREG $a, sub0), 0,
- (EXTRACT_SUBREG $b, sub1), 0,
- (IMADD_I32 0, (EXTRACT_SUBREG $a, sub1), 0,
- (EXTRACT_SUBREG $b, sub0), 0,
- (EXTRACT_SUBREG
- (IMADD_I64_FROM32 0, (EXTRACT_SUBREG $a, sub0),
- 0, (EXTRACT_SUBREG $b, sub0), 0, (i64 0),
- 0), sub1), 0), 0), sub1)>;
-
//===----------------------------------------------------------------------===//
// Shifts and bit fields
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AGX/AGXLowerControlFlow.cpp b/llvm/lib/Target/AGX/AGXLowerControlFlow.cpp
index 6e16961a5cfbf..07f9165f126b0 100644
--- a/llvm/lib/Target/AGX/AGXLowerControlFlow.cpp
+++ b/llvm/lib/Target/AGX/AGXLowerControlFlow.cpp
@@ -13,7 +13,7 @@
// AGX_ELSE %end else_fcmp r0l, eq, 0.0, 0.0, 1; jmp_exec_none %end
// AGX_LOOP brk, %hdr while_icmp r0l, eq, brk, 0, 1; jmp_exec_any %hdr
// AGX_LOOP_BEGIN if_fcmp r0l, eq, 0.0, 0.0, 1 (push_exec 1)
-// AGX_END_CF pop_exec r0l, 1
+// AGX_END_CF pop_exec r0l, 1, at the start of the block
// AGX_BR %bb jmp_exec_any %bb; jmp_exec_none %bb
//
// The jumps only skip code no thread executes; the execution mask
@@ -122,9 +122,17 @@ bool AGXLowerControlFlow::runOnMachineFunction(MachineFunction &MF) {
.addImm(FPImmZero)
.addImm(1);
break;
- case AGX::AGX_END_CF:
- BuildMI(MBB, I, DL, TII.get(AGX::POP_EXEC)).addImm(0).addImm(1);
+ case AGX::AGX_END_CF: {
+ // The construct reconverges at the start of the block. Scheduling
+ // may have moved instructions of the block above the pseudo, since
+ // they have no dependency on it: the pop goes first (like
+ // SILowerControlFlow::emitEndCf), after any earlier pops.
+ MachineBasicBlock::iterator Pos = MBB.begin();
+ while (Pos != I && Pos->getOpcode() == AGX::POP_EXEC)
+ ++Pos;
+ BuildMI(MBB, Pos, DL, TII.get(AGX::POP_EXEC)).addImm(0).addImm(1);
break;
+ }
case AGX::AGX_BR: {
MachineBasicBlock *Target = MI.getOperand(0).getMBB();
if (!MBB.isLayoutSuccessor(Target)) {
diff --git a/llvm/lib/Target/AGX/AGXOperands.td b/llvm/lib/Target/AGX/AGXOperands.td
index 7644f7a2a298c..d5431627abd52 100644
--- a/llvm/lib/Target/AGX/AGXOperands.td
+++ b/llvm/lib/Target/AGX/AGXOperands.td
@@ -308,5 +308,6 @@ class AGXRegOperand<RegisterClass rc, string type, string ns = "AGX">
// after operand folding, 8-bit immediates.
def Src16Op : AGXRegOperand<SRC16, "OPERAND_SRC">;
def Src32Op : AGXRegOperand<SRC32, "OPERAND_SRC">;
-def Src64Op : AGXRegOperand<SRC64, "OPERAND_SRC">;
+// 64-bit sources cannot be uniform registers.
+def Src64Op : AGXRegOperand<GPR64, "OPERAND_SRC">;
def Base64Op : AGXRegOperand<BASE64, "OPERAND_REGISTER", "MCOI">;
diff --git a/llvm/lib/Target/AGX/AGXRegisterInfo.td b/llvm/lib/Target/AGX/AGXRegisterInfo.td
index cc1791a38ba7e..fbf086e17bcbc 100644
--- a/llvm/lib/Target/AGX/AGXRegisterInfo.td
+++ b/llvm/lib/Target/AGX/AGXRegisterInfo.td
@@ -128,10 +128,6 @@ def SRC32 : RegisterClass<"AGX", [i32, f32, v2i16, v2f16, v2bf16], 32,
(add GPR32, UGPR32)> {
let isAllocatable = 0;
}
-def SRC64 : RegisterClass<"AGX", [i64, f64, v2i32, v2f32], 32,
- (add GPR64, UGPR64)> {
- let isAllocatable = 0;
-}
// 64-bit memory base addresses: general purpose or uniform register pairs.
def BASE64 : RegisterClass<"AGX", [i64], 32, (add GPR32x2, UGPR64)> {
let isAllocatable = 0;
diff --git a/llvm/lib/Target/AGX/AGXTargetMachine.cpp b/llvm/lib/Target/AGX/AGXTargetMachine.cpp
index a472d9ef76fdc..e80d4d26f5484 100644
--- a/llvm/lib/Target/AGX/AGXTargetMachine.cpp
+++ b/llvm/lib/Target/AGX/AGXTargetMachine.cpp
@@ -115,6 +115,10 @@ class AGXPassConfig : public TargetPassConfig {
disablePass(&PatchableFunctionID);
disablePass(&ShrinkWrapID);
disablePass(&PostRAMachineSinkingID);
+ // Keep the block order of the structurizer: execution mask code falls
+ // through from each construct into the next, as the vendor and Mesa
+ // compilers lay it out.
+ disablePass(&MachineBlockPlacementID);
}
AGXTargetMachine &getAGXTargetMachine() const {
diff --git a/llvm/test/CodeGen/AGX/atomics.ll b/llvm/test/CodeGen/AGX/atomics.ll
new file mode 100644
index 0000000000000..c56c08cd3270e
--- /dev/null
+++ b/llvm/test/CodeGen/AGX/atomics.ll
@@ -0,0 +1,147 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -verify-machineinstrs < %s | FileCheck %s
+
+define agx_kernel void @device_rmw(ptr addrspace(1) %p, ptr addrspace(1) %out, i32 %v) {
+; CHECK-LABEL: device_rmw:
+; CHECK: // uniform registers: 16 x 16-bit (threadgroups_per_grid at u5)
+; CHECK-NEXT: // general purpose registers: 2 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: bitop 0101, r1, u4, 0
+; CHECK-NEXT: atomic add, 0, r1, u0_u1, 0, signed, r1, 0
+; CHECK-NEXT: wait 0
+; CHECK-NEXT: atomic umax, 0, r1, u0_u1, 0, signed, r1, 0
+; CHECK-NEXT: wait 0
+; CHECK-NEXT: atomic xchg, 0, r1, u0_u1, 0, signed, r1, 0
+; CHECK-NEXT: wait 0
+; CHECK-NEXT: device_store 0, i32, x, r1, u2_u3, 0, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %a = atomicrmw add ptr addrspace(1) %p, i32 %v monotonic
+ %b = atomicrmw umax ptr addrspace(1) %p, i32 %a monotonic
+ %c = atomicrmw xchg ptr addrspace(1) %p, i32 %b monotonic
+ store i32 %c, ptr addrspace(1) %out
+ ret void
+}
+
+; The data operand of cmpxchg is the pair {new, compare}.
+define agx_kernel void @device_cmpxchg(ptr addrspace(1) %p, ptr addrspace(1) %out, i32 %cmp, i32 %new) {
+; CHECK-LABEL: device_cmpxchg:
+; CHECK: // uniform registers: 18 x 16-bit (threadgroups_per_grid at u6)
+; CHECK-NEXT: // general purpose registers: 4 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: bitop 0101, r2, u5, 0
+; CHECK-NEXT: bitop 0101, r3, u4, 0
+; CHECK-NEXT: atomic cmpxchg, 0, r1, u0_u1, 0, signed, r2_r3, 0
+; CHECK-NEXT: wait 0
+; CHECK-NEXT: device_store 0, i32, x, r1, u2_u3, 0, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %r = cmpxchg ptr addrspace(1) %p, i32 %cmp, i32 %new monotonic monotonic
+ %v = extractvalue {i32, i1} %r, 0
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+ at tg = internal addrspace(3) global i32 undef
+
+define agx_kernel void @threadgroup_rmw(ptr addrspace(1) %out, i32 %v) {
+; CHECK-LABEL: threadgroup_rmw:
+; CHECK: // uniform registers: 12 x 16-bit (threadgroups_per_grid at u3)
+; CHECK-NEXT: // general purpose registers: 3 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 4 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: bitop 0101, r1, u2, 0
+; CHECK-NEXT: mov_imm r2, 0
+; CHECK-NEXT: threadgroup_atomic imin, r1, r2l, 0, r1
+; CHECK-NEXT: device_store 0, i32, x, r1, u0_u1, 0, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %a = atomicrmw min ptr addrspace(3) @tg, i32 %v monotonic
+ store i32 %a, ptr addrspace(1) %out
+ ret void
+}
+
+; Operations without an atomic instruction become compare-and-swap loops.
+define agx_kernel void @fadd(ptr addrspace(1) %p, float %v) {
+; CHECK-LABEL: fadd:
+; CHECK: // uniform registers: 12 x 16-bit (threadgroups_per_grid at u3)
+; CHECK-NEXT: // general purpose registers: 5 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov_imm r0l, 0
+; CHECK-NEXT: mov_imm r2, 0
+; CHECK-NEXT: bitop 0101, r3, r2, 0
+; CHECK-NEXT: atomic cmpxchg, 0, r3, u0_u1, 0, signed, r2_r3, 0
+; CHECK-NEXT: if_fcmp r0l, eq, 0.0, 0.0, 1
+; CHECK-NEXT: wait 0
+; CHECK-NEXT: .LBB3_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: fadd32 r2, r3, u2
+; CHECK-NEXT: atomic cmpxchg, 0, r1, u0_u1, 0, signed, r2_r3, 0
+; CHECK-NEXT: wait 0
+; CHECK-NEXT: icmpsel ueq, r4, r1, r3, 1, 0
+; CHECK-NEXT: bitop 0101, r3, r1, 0
+; CHECK-NEXT: while_icmp r0l, ueq, r4, 0, 1
+; CHECK-NEXT: jmp_exec_any .LBB3_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: pop_exec r0l, 1
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %a = atomicrmw fadd ptr addrspace(1) %p, float %v monotonic
+ ret void
+}
+
+define agx_kernel void @fence() {
+; CHECK-LABEL: fence:
+; CHECK: // uniform registers: 6 x 16-bit (threadgroups_per_grid at u0)
+; CHECK-NEXT: // general purpose registers: 1 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: memory_barrier 1, 2, 9
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ fence seq_cst
+ ret void
+}
diff --git a/llvm/test/CodeGen/AGX/control-flow.ll b/llvm/test/CodeGen/AGX/control-flow.ll
new file mode 100644
index 0000000000000..98a34086a4f68
--- /dev/null
+++ b/llvm/test/CodeGen/AGX/control-flow.ll
@@ -0,0 +1,310 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -verify-machineinstrs < %s | FileCheck %s
+
+; All branches become execution mask operations on r0l: if_icmp / else /
+; pop_exec for if-then-else, and push_exec / while_icmp / jmp_exec_any for
+; loops. The jumps only skip code no thread executes.
+
+define agx_kernel void @if_then_else(ptr addrspace(1) %out, i32 %n) {
+; CHECK-LABEL: if_then_else:
+; CHECK: // uniform registers: 12 x 16-bit (threadgroups_per_grid at u3)
+; CHECK-NEXT: // general purpose registers: 4 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov_imm r0l, 0
+; CHECK-NEXT: get_sr r1, sr80 (thread_position_in_grid.x)
+; CHECK-NEXT: icmpsel ult, r3, r1, u2, 0, 1
+; CHECK-NEXT: // implicit-def: $r2
+; CHECK-NEXT: if_icmp r0l, nueq, r3, 0, 1
+; CHECK-NEXT: jmp_exec_none .LBB0_2
+; CHECK-NEXT: // %bb.1: // %else
+; CHECK-NEXT: iadd r2, r1, 7
+; CHECK-NEXT: .LBB0_2: // %Flow
+; CHECK-NEXT: else_fcmp r0l, eq, 0.0, 0.0, 1
+; CHECK-NEXT: jmp_exec_none .LBB0_4
+; CHECK-NEXT: // %bb.3: // %then
+; CHECK-NEXT: imadd r2, r1, 3, 0
+; CHECK-NEXT: .LBB0_4: // %end
+; CHECK-NEXT: pop_exec r0l, 1
+; CHECK-NEXT: device_store 0, i32, x, r2, u0_u1, r1, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %id = call i32 @llvm.agx.thread.position.in.grid.x()
+ %c = icmp ult i32 %id, %n
+ br i1 %c, label %then, label %else
+then:
+ %a = mul i32 %id, 3
+ br label %end
+else:
+ %b = add i32 %id, 7
+ br label %end
+end:
+ %r = phi i32 [%a, %then], [%b, %else]
+ %p = getelementptr i32, ptr addrspace(1) %out, i32 %id
+ store i32 %r, ptr addrspace(1) %p
+ ret void
+}
+
+define agx_kernel void @early_return(ptr addrspace(1) %out, i32 %n) {
+; CHECK-LABEL: early_return:
+; CHECK: // uniform registers: 12 x 16-bit (threadgroups_per_grid at u3)
+; CHECK-NEXT: // general purpose registers: 3 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov_imm r0l, 0
+; CHECK-NEXT: get_sr r1, sr80 (thread_position_in_grid.x)
+; CHECK-NEXT: icmpsel ult, r2, r1, u2, 1, 0
+; CHECK-NEXT: if_icmp r0l, nueq, r2, 0, 1
+; CHECK-NEXT: jmp_exec_none .LBB1_2
+; CHECK-NEXT: // %bb.1: // %then
+; CHECK-NEXT: mov_imm r2, 1
+; CHECK-NEXT: device_store 0, i32, x, r2, u0_u1, r1, signed
+; CHECK-NEXT: .LBB1_2: // %unified.return
+; CHECK-NEXT: pop_exec r0l, 1
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %id = call i32 @llvm.agx.thread.position.in.grid.x()
+ %c = icmp ult i32 %id, %n
+ br i1 %c, label %then, label %ret
+then:
+ %p = getelementptr i32, ptr addrspace(1) %out, i32 %id
+ store i32 1, ptr addrspace(1) %p
+ ret void
+ret:
+ ret void
+}
+
+define agx_kernel void @loop(ptr addrspace(1) %out) {
+; CHECK-LABEL: loop:
+; CHECK: // uniform registers: 10 x 16-bit (threadgroups_per_grid at u2)
+; CHECK-NEXT: // general purpose registers: 6 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0: // %entry
+; CHECK-NEXT: mov_imm r0l, 0
+; CHECK-NEXT: get_sr r1, sr80 (thread_position_in_grid.x)
+; CHECK-NEXT: if_fcmp r0l, eq, 0.0, 0.0, 1
+; CHECK-NEXT: mov_imm r2, 0
+; CHECK-NEXT: mov_imm r3, 0
+; CHECK-NEXT: .LBB2_1: // %header
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: convert s32_to_f, r4, r3, rte
+; CHECK-NEXT: iadd r3, r3, 1
+; CHECK-NEXT: icmpsel ult, r5, r3, r1, 0, 1
+; CHECK-NEXT: fadd32 r2, r2, r4
+; CHECK-NEXT: while_icmp r0l, ueq, r5, 0, 1
+; CHECK-NEXT: jmp_exec_any .LBB2_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: pop_exec r0l, 1
+; CHECK-NEXT: device_store 0, i32, x, r2, u0_u1, r1, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+entry:
+ %id = call i32 @llvm.agx.thread.position.in.grid.x()
+ br label %header
+header:
+ %i = phi i32 [0, %entry], [%i.next, %header]
+ %acc = phi float [0.0, %entry], [%acc.next, %header]
+ %f = sitofp i32 %i to float
+ %acc.next = fadd float %acc, %f
+ %i.next = add i32 %i, 1
+ %done = icmp uge i32 %i.next, %id
+ br i1 %done, label %exit, label %header
+exit:
+ %p = getelementptr float, ptr addrspace(1) %out, i32 %id
+ store float %acc.next, ptr addrspace(1) %p
+ ret void
+}
+
+; Code of a reconvergence block never runs before its pop_exec.
+define agx_kernel void @nested_loops(ptr addrspace(1) %out, i32 %n, i32 %m) {
+; CHECK-LABEL: nested_loops:
+; CHECK: // uniform registers: 14 x 16-bit (threadgroups_per_grid at u4)
+; CHECK-NEXT: // general purpose registers: 8 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0: // %entry
+; CHECK-NEXT: mov_imm r0l, 0
+; CHECK-NEXT: get_sr r1, sr80 (thread_position_in_grid.x)
+; CHECK-NEXT: if_fcmp r0l, eq, 0.0, 0.0, 1
+; CHECK-NEXT: mov_imm r2, 0
+; CHECK-NEXT: .LBB3_1: // %outer
+; CHECK-NEXT: // =>This Loop Header: Depth=1
+; CHECK-NEXT: // Child Loop BB3_2 Depth 2
+; CHECK-NEXT: if_fcmp r0l, eq, 0.0, 0.0, 1
+; CHECK-NEXT: mov_imm r4, 1
+; CHECK-NEXT: .LBB3_2: // %inner
+; CHECK-NEXT: // Parent Loop BB3_1 Depth=1
+; CHECK-NEXT: // => This Inner Loop Header: Depth=2
+; CHECK-NEXT: bitop 0101, r3, r4, 0
+; CHECK-NEXT: icmpsel ueq, r7, r1, r4, 0, 1
+; CHECK-NEXT: mov_imm r5l, 1
+; CHECK-NEXT: // implicit-def: $r4
+; CHECK-NEXT: mov_imm r6l, 1
+; CHECK-NEXT: if_icmp r0l, nueq, r7, 0, 1
+; CHECK-NEXT: jmp_exec_none .LBB3_4
+; CHECK-NEXT: // %bb.3: // %inner.c
+; CHECK-NEXT: // in Loop: Header=BB3_2 Depth=2
+; CHECK-NEXT: iadd r4, r3, 1
+; CHECK-NEXT: icmpsel ult, r6, r3, u3, 0, 1
+; CHECK-NEXT: mov_imm r5l, 0
+; CHECK-NEXT: .LBB3_4: // %Flow
+; CHECK-NEXT: // in Loop: Header=BB3_2 Depth=2
+; CHECK-NEXT: pop_exec r0l, 1
+; CHECK-NEXT: bitop 0101, r6, r6l, 0
+; CHECK-NEXT: while_icmp r0l, ueq, r6, 0, 1
+; CHECK-NEXT: jmp_exec_any .LBB3_2
+; CHECK-NEXT: // %bb.5: // %loop.exit.guard
+; CHECK-NEXT: // in Loop: Header=BB3_1 Depth=1
+; CHECK-NEXT: pop_exec r0l, 1
+; CHECK-NEXT: xor r4l, r5l, 1
+; CHECK-NEXT: bitop 0101, r5, r4l, 0
+; CHECK-NEXT: mov_imm r4l, 1
+; CHECK-NEXT: if_icmp r0l, nueq, r5, 0, 1
+; CHECK-NEXT: jmp_exec_none .LBB3_7
+; CHECK-NEXT: // %bb.6: // %outer.latch
+; CHECK-NEXT: // in Loop: Header=BB3_1 Depth=1
+; CHECK-NEXT: iadd r2, r2, 1
+; CHECK-NEXT: icmpsel ult, r4, r2, u2, 0, 1
+; CHECK-NEXT: bitop 0101, r3, r2, 0
+; CHECK-NEXT: .LBB3_7: // %Flow3
+; CHECK-NEXT: // in Loop: Header=BB3_1 Depth=1
+; CHECK-NEXT: pop_exec r0l, 1
+; CHECK-NEXT: bitop 0101, r4, r4l, 0
+; CHECK-NEXT: while_icmp r0l, ueq, r4, 0, 1
+; CHECK-NEXT: jmp_exec_any .LBB3_1
+; CHECK-NEXT: // %bb.8: // %exit
+; CHECK-NEXT: pop_exec r0l, 1
+; CHECK-NEXT: device_store 0, i32, x, r3, u0_u1, r1, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+entry:
+ %id = call i32 @llvm.agx.thread.position.in.grid.x()
+ br label %outer
+outer:
+ %i = phi i32 [0, %entry], [%i.n, %outer.latch]
+ br label %inner
+inner:
+ %j = phi i32 [0, %outer], [%j.n, %inner.c]
+ %j.n = add i32 %j, 1
+ %brk = icmp eq i32 %j.n, %id
+ br i1 %brk, label %exit, label %inner.c
+inner.c:
+ %ic = icmp ult i32 %j.n, %m
+ br i1 %ic, label %inner, label %outer.latch
+outer.latch:
+ %i.n = add i32 %i, 1
+ %oc = icmp ult i32 %i.n, %n
+ br i1 %oc, label %outer, label %exit
+exit:
+ %v = phi i32 [%j.n, %inner], [%i.n, %outer.latch]
+ %p = getelementptr i32, ptr addrspace(1) %out, i32 %id
+ store i32 %v, ptr addrspace(1) %p
+ ret void
+}
+
+define agx_kernel void @switch(ptr addrspace(1) %out) {
+; CHECK-LABEL: switch:
+; CHECK: // uniform registers: 10 x 16-bit (threadgroups_per_grid at u2)
+; CHECK-NEXT: // general purpose registers: 5 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0: // %entry
+; CHECK-NEXT: mov_imm r0l, 0
+; CHECK-NEXT: get_sr r1, sr80 (thread_position_in_grid.x)
+; CHECK-NEXT: icmpsel sgt, r4, r1, 4, 1, 0
+; CHECK-NEXT: mov_imm r2l, 0
+; CHECK-NEXT: mov_imm r3l, 0
+; CHECK-NEXT: if_icmp r0l, nueq, r4, 0, 1
+; CHECK-NEXT: jmp_exec_none .LBB4_4
+; CHECK-NEXT: // %bb.1: // %LeafBlock1
+; CHECK-NEXT: icmpsel ueq, r4, r1, 5, 1, 0
+; CHECK-NEXT: mov_imm r3l, 1
+; CHECK-NEXT: if_icmp r0l, nueq, r4, 0, 1
+; CHECK-NEXT: jmp_exec_none .LBB4_3
+; CHECK-NEXT: // %bb.2: // %b
+; CHECK-NEXT: mov_imm r3l, 0
+; CHECK-NEXT: .LBB4_3: // %Flow3
+; CHECK-NEXT: pop_exec r0l, 1
+; CHECK-NEXT: .LBB4_4: // %Flow
+; CHECK-NEXT: else_fcmp r0l, eq, 0.0, 0.0, 1
+; CHECK-NEXT: jmp_exec_none .LBB4_6
+; CHECK-NEXT: // %bb.5: // %LeafBlock
+; CHECK-NEXT: icmpsel ueq, r3, r1, 0, 0, 1
+; CHECK-NEXT: mov_imm r2l, 1
+; CHECK-NEXT: .LBB4_6: // %Flow4
+; CHECK-NEXT: pop_exec r0l, 1
+; CHECK-NEXT: bitop 0101, r4, r3l, 0
+; CHECK-NEXT: mov_imm r3, 2
+; CHECK-NEXT: if_icmp r0l, nueq, r4, 0, 1
+; CHECK-NEXT: jmp_exec_none .LBB4_8
+; CHECK-NEXT: // %bb.7: // %def
+; CHECK-NEXT: mov_imm r2l, 0
+; CHECK-NEXT: mov_imm r3, 3
+; CHECK-NEXT: .LBB4_8: // %Flow5
+; CHECK-NEXT: pop_exec r0l, 1
+; CHECK-NEXT: bitop 0101, r2, r2l, 0
+; CHECK-NEXT: if_icmp r0l, nueq, r2, 0, 1
+; CHECK-NEXT: jmp_exec_none .LBB4_10
+; CHECK-NEXT: // %bb.9: // %a
+; CHECK-NEXT: mov_imm r3, 1
+; CHECK-NEXT: .LBB4_10: // %join
+; CHECK-NEXT: pop_exec r0l, 1
+; CHECK-NEXT: device_store 0, i32, x, r3, u0_u1, r1, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+entry:
+ %id = call i32 @llvm.agx.thread.position.in.grid.x()
+ switch i32 %id, label %def [i32 0, label %a
+ i32 5, label %b]
+a:
+ br label %join
+b:
+ br label %join
+def:
+ br label %join
+join:
+ %v = phi i32 [1, %a], [2, %b], [3, %def]
+ %p = getelementptr i32, ptr addrspace(1) %out, i32 %id
+ store i32 %v, ptr addrspace(1) %p
+ ret void
+}
+
+declare i32 @llvm.agx.thread.position.in.grid.x()
diff --git a/llvm/test/CodeGen/AGX/errors.ll b/llvm/test/CodeGen/AGX/errors.ll
new file mode 100644
index 0000000000000..66ee8c4846f9a
--- /dev/null
+++ b/llvm/test/CodeGen/AGX/errors.ll
@@ -0,0 +1,12 @@
+; RUN: not llc -mtriple=agx -mcpu=agx-g13g < %s 2>&1 | FileCheck %s
+; RUN: not llc -mtriple=agx -mcpu=agx-g17p < %s 2>&1 | FileCheck %s --check-prefix=APPLE9
+
+; CHECK: error: {{.*}}unsupported call to 'external' (AGX has no calls)
+; APPLE9: error: no AGX instruction encoding for {{.*}} on agx-g17p (the Apple9 encoding is not yet supported)
+
+declare void @external()
+
+define agx_kernel void @calls() {
+ call void @external()
+ ret void
+}
diff --git a/llvm/test/CodeGen/AGX/fp.ll b/llvm/test/CodeGen/AGX/fp.ll
new file mode 100644
index 0000000000000..2cbb402d0668b
--- /dev/null
+++ b/llvm/test/CodeGen/AGX/fp.ll
@@ -0,0 +1,284 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -verify-machineinstrs < %s | FileCheck %s
+
+; Source modifiers and 8-bit float immediates are folded into instructions.
+define agx_kernel void @arith(ptr addrspace(1) %out, float %a, float %b, float %c) {
+; CHECK-LABEL: arith:
+; CHECK: // uniform registers: 16 x 16-bit (threadgroups_per_grid at u5)
+; CHECK-NEXT: // general purpose registers: 2 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: fmul32 r1, u2.neg, u3.abs
+; CHECK-NEXT: fadd32 r1, r1, u4.neg
+; CHECK-NEXT: fmadd32 r1, r1, u2, 0.5
+; CHECK-NEXT: fadd32 r1, r1, -2.0
+; CHECK-NEXT: device_store 0, i32, x, r1, u0_u1, 0, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %na = fneg float %a
+ %ab = call float @llvm.fabs.f32(float %b)
+ %m = fmul float %na, %ab
+ %s = fsub float %m, %c
+ %f = call float @llvm.fma.f32(float %s, float %a, float 0.5)
+ %h = fadd float %f, -2.0
+ store float %h, ptr addrspace(1) %out
+ ret void
+}
+
+; Division is a reciprocal and a multiply with one Newton-Raphson step,
+; unless reciprocals are allowed.
+define agx_kernel void @div(ptr addrspace(1) %out, float %a, float %b) {
+; CHECK-LABEL: div:
+; CHECK: // uniform registers: 14 x 16-bit (threadgroups_per_grid at u4)
+; CHECK-NEXT: // general purpose registers: 4 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: rcp r1, u3
+; CHECK-NEXT: fmul32 r2, u2, r1
+; CHECK-NEXT: fmadd32 r3, u3.neg, r2, u2
+; CHECK-NEXT: fmadd32 r2, r3, r1, r2
+; CHECK-NEXT: fmul32 r1, r2, r1
+; CHECK-NEXT: rcp r1, r1
+; CHECK-NEXT: device_store 0, i32, x, r1, u0_u1, 0, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %d = fdiv float %a, %b
+ %e = fdiv arcp float %d, %b
+ %r = fdiv float 1.0, %e
+ store float %r, ptr addrspace(1) %out
+ ret void
+}
+
+define agx_kernel void @transcendental(ptr addrspace(1) %out, float %a, float %b) {
+; CHECK-LABEL: transcendental:
+; CHECK: // uniform registers: 14 x 16-bit (threadgroups_per_grid at u4)
+; CHECK-NEXT: // general purpose registers: 4 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov_imm r1, 1042479491
+; CHECK-NEXT: fmul32 r2, u2, r1
+; CHECK-NEXT: floor r3, r2
+; CHECK-NEXT: fadd32 r2, r2, r3.neg
+; CHECK-NEXT: fmul32 r2, r2, 4.0
+; CHECK-NEXT: sin_pt_1 r2, r2
+; CHECK-NEXT: sin_pt_2 r3, r2
+; CHECK-NEXT: fmul32 r2, r3, r2
+; CHECK-NEXT: fmul32 r1, r2, r1
+; CHECK-NEXT: fadd32 r1, r1, 0.25
+; CHECK-NEXT: floor r2, r1
+; CHECK-NEXT: fadd32 r1, r1, r2.neg
+; CHECK-NEXT: fmul32 r1, r1, 4.0
+; CHECK-NEXT: sin_pt_1 r1, r1
+; CHECK-NEXT: sin_pt_2 r2, r1
+; CHECK-NEXT: fmul32 r1, r2, r1
+; CHECK-NEXT: exp2 r1, r1
+; CHECK-NEXT: log2 r1, r1
+; CHECK-NEXT: log2 r1, r1
+; CHECK-NEXT: fmul32 r1, u3, r1
+; CHECK-NEXT: exp2 r1, r1
+; CHECK-NEXT: rsqrt_special r2, r1
+; CHECK-NEXT: fmul32 r1, r1, r2
+; CHECK-NEXT: device_store 0, i32, x, r1, u0_u1, 0, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %s = call float @llvm.sin.f32(float %a)
+ %c = call float @llvm.cos.f32(float %s)
+ %e = call float @llvm.exp2.f32(float %c)
+ %l = call float @llvm.log2.f32(float %e)
+ %p = call float @llvm.pow.f32(float %l, float %b)
+ %q = call float @llvm.sqrt.f32(float %p)
+ store float %q, ptr addrspace(1) %out
+ ret void
+}
+
+define agx_kernel void @rounding(ptr addrspace(1) %out, float %a, float %b) {
+; CHECK-LABEL: rounding:
+; CHECK: // uniform registers: 14 x 16-bit (threadgroups_per_grid at u4)
+; CHECK-NEXT: // general purpose registers: 2 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: floor r1, u2
+; CHECK-NEXT: ceil r1, r1
+; CHECK-NEXT: rint r1, r1
+; CHECK-NEXT: fcmpsel ltn, r1, r1, u3, r1, u3
+; CHECK-NEXT: fadd32.sat r1, r1, -0.0
+; CHECK-NEXT: device_store 0, i32, x, r1, u0_u1, 0, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %f = call float @llvm.floor.f32(float %a)
+ %c = call float @llvm.ceil.f32(float %f)
+ %t = call float @llvm.trunc.f32(float %c)
+ %r = call float @llvm.roundeven.f32(float %t)
+ %mn = call float @llvm.minnum.f32(float %r, float %b)
+ %mx = call float @llvm.maxnum.f32(float %mn, float 0.0)
+ %sat = call float @llvm.minnum.f32(float %mx, float 1.0)
+ store float %sat, ptr addrspace(1) %out
+ ret void
+}
+
+define agx_kernel void @compare(ptr addrspace(1) %out, float %a, float %b) {
+; CHECK-LABEL: compare:
+; CHECK: // uniform registers: 14 x 16-bit (threadgroups_per_grid at u4)
+; CHECK-NEXT: // general purpose registers: 3 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: fcmpsel lt, r1, u2, u3, 1, 0
+; CHECK-NEXT: mov_imm r2, 1077936128
+; CHECK-NEXT: icmpsel ueq, r1, r1, 0, r2, u2
+; CHECK-NEXT: fcmpsel lt, r2, r1, u3, 0, 1
+; CHECK-NEXT: icmpsel ueq, r1, r2, 0, u3, r1
+; CHECK-NEXT: device_store 0, i32, x, r1, u0_u1, 0, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %c = fcmp olt float %a, %b
+ %s = select i1 %c, float %a, float 3.0
+ %u = fcmp uge float %s, %b
+ %r = select i1 %u, float %s, float %b
+ store float %r, ptr addrspace(1) %out
+ ret void
+}
+
+define agx_kernel void @half(ptr addrspace(1) %out, half %a, half %b) {
+; CHECK-LABEL: half:
+; CHECK: // uniform registers: 12 x 16-bit (threadgroups_per_grid at u3)
+; CHECK-NEXT: // general purpose registers: 2 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: rcp r1l, u2h
+; CHECK-NEXT: fmul16 r1h, u2l, u2h
+; CHECK-NEXT: fmadd16 r1h, r1h, u2l, 1.0
+; CHECK-NEXT: fmul16 r1l, r1h, r1l
+; CHECK-NEXT: fadd16 r1l, r1l, u2l
+; CHECK-NEXT: device_store 0, i16, x, r1l, u0_u1, 0, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %m = fmul half %a, %b
+ %f = call half @llvm.fma.f16(half %m, half %a, half 1.0)
+ %d = fdiv half %f, %b
+ %e = fpext half %d to float
+ %t = fptrunc float %e to half
+ %r = fadd half %t, %a
+ store half %r, ptr addrspace(1) %out
+ ret void
+}
+
+define agx_kernel void @convert(ptr addrspace(1) %out, float %a, i32 %i, i64 %l) {
+; CHECK-LABEL: convert:
+; CHECK: // uniform registers: 18 x 16-bit (threadgroups_per_grid at u6)
+; CHECK-NEXT: // general purpose registers: 13 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: asr r2, u5, 31
+; CHECK-NEXT: bitop 0101, r3, r2, 0
+; CHECK-NEXT: xor r5, u5, r2
+; CHECK-NEXT: xor r4, u4, r2
+; CHECK-NEXT: isub r4_r5, r4_r5, r2_r3
+; CHECK-NEXT: convert f_to_s32, r1, u2, rtz
+; CHECK-NEXT: convert u32_to_f, r6, u3, rte
+; CHECK-NEXT: ffs r7, r4
+; CHECK-NEXT: ffs r8, r5
+; CHECK-NEXT: trunc r6, r6
+; CHECK-NEXT: isub r7, 31, r7
+; CHECK-NEXT: isub r8, 31, r8
+; CHECK-NEXT: iadd r7, r7, 32
+; CHECK-NEXT: mov_imm r9, 796917760
+; CHECK-NEXT: icmpsel ueq, r7, r5, 0, r7, r8
+; CHECK-NEXT: fmul32 r8, r6, r9
+; CHECK-NEXT: and r9, r7, 31
+; CHECK-NEXT: bfeil r10, 0, r4, 1
+; CHECK-NEXT: xor r11, r9, 31
+; CHECK-NEXT: bfi r12, 0, r5, r9
+; CHECK-NEXT: bfeil r10, 0, r10, r11
+; CHECK-NEXT: and r11, r7, 32
+; CHECK-NEXT: bfi r4, 0, r4, r9
+; CHECK-NEXT: bfeil r5, 0, r11, 5
+; CHECK-NEXT: floor r8, r8
+; CHECK-NEXT: or r9, r12, r10
+; CHECK-NEXT: icmpsel ueq, r10, r5, 0, r4, 0
+; CHECK-NEXT: icmpsel ueq, r4, r5, 0, r9, r4
+; CHECK-NEXT: icmpsel ueq, r5, r10, 0, 0, 1
+; CHECK-NEXT: convert f_to_u32, r11, r8, rtz
+; CHECK-NEXT: or r4, r4, r5
+; CHECK-NEXT: isub r5, 159, r7
+; CHECK-NEXT: convert u32_to_f, r4, r4, rte
+; CHECK-NEXT: bfi r5, 0, r5, 23
+; CHECK-NEXT: mov_imm r7, -813694976
+; CHECK-NEXT: fmadd32 r6, r8, r7, r6
+; CHECK-NEXT: fmul32 r4, r4, r5
+; CHECK-NEXT: mov_imm r5, -2147483648
+; CHECK-NEXT: and r2, r2, r5
+; CHECK-NEXT: convert s32_to_f, r1, r1, rte
+; CHECK-NEXT: or r2, r4, r2
+; CHECK-NEXT: convert f_to_u32, r10, r6, rtz
+; CHECK-NEXT: fadd32 r1, r1, r2
+; CHECK-NEXT: device_store 0, i32, x, r1, u0_u1, 0, signed
+; CHECK-NEXT: device_store 0, i32, xy, r10_r11, u0_u1, 2, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %x = fptosi float %a to i32
+ %y = uitofp i32 %i to float
+ %z = fptoui float %y to i64
+ %w = sitofp i64 %l to float
+ %xf = sitofp i32 %x to float
+ %s = fadd float %xf, %w
+ store float %s, ptr addrspace(1) %out
+ %o = getelementptr i64, ptr addrspace(1) %out, i64 1
+ store i64 %z, ptr addrspace(1) %o
+ ret void
+}
diff --git a/llvm/test/CodeGen/AGX/i64.ll b/llvm/test/CodeGen/AGX/i64.ll
new file mode 100644
index 0000000000000..a9328aac50484
--- /dev/null
+++ b/llvm/test/CodeGen/AGX/i64.ll
@@ -0,0 +1,176 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -verify-machineinstrs < %s | FileCheck %s
+
+define agx_kernel void @add_mul(ptr addrspace(1) %out, i64 %a, i64 %b) {
+; CHECK-LABEL: add_mul:
+; CHECK: // uniform registers: 18 x 16-bit (threadgroups_per_grid at u6)
+; CHECK-NEXT: // general purpose registers: 6 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: bitop 0101, r2, u4, 0
+; CHECK-NEXT: bitop 0101, r3, u5, 0
+; CHECK-NEXT: bitop 0101, r4, u2, 0
+; CHECK-NEXT: bitop 0101, r5, u3, 0
+; CHECK-NEXT: iadd r2_r3, r4_r5, r2_r3
+; CHECK-NEXT: imadd r1, r3, u4, 0
+; CHECK-NEXT: imadd r4_r5, r2, u4, 0
+; CHECK-NEXT: imadd r1, r2, u5, r1
+; CHECK-NEXT: iadd r5, r5, r1
+; CHECK-NEXT: device_store 0, i32, xy, r4_r5, u0_u1, 0, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %s = add i64 %a, %b
+ %m = mul i64 %s, %b
+ store i64 %m, ptr addrspace(1) %out
+ ret void
+}
+
+define agx_kernel void @mul_wide(ptr addrspace(1) %out, i32 %a, i32 %b) {
+; CHECK-LABEL: mul_wide:
+; CHECK: // uniform registers: 14 x 16-bit (threadgroups_per_grid at u4)
+; CHECK-NEXT: // general purpose registers: 4 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: imadd r2_r3, u2, u3, 0
+; CHECK-NEXT: device_store 0, i32, xy, r2_r3, u0_u1, 0, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %x = zext i32 %a to i64
+ %y = zext i32 %b to i64
+ %m = mul i64 %x, %y
+ store i64 %m, ptr addrspace(1) %out
+ ret void
+}
+
+define agx_kernel void @shifts(ptr addrspace(1) %out, i64 %a, i64 %s) {
+; CHECK-LABEL: shifts:
+; CHECK: // uniform registers: 18 x 16-bit (threadgroups_per_grid at u6)
+; CHECK-NEXT: // general purpose registers: 6 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov_imm r1, 0
+; CHECK-NEXT: bfeil r2, 0, u3, 5
+; CHECK-NEXT: mov_imm r3, 16777215
+; CHECK-NEXT: and r1, r1, r1
+; CHECK-NEXT: and r2, r2, r3
+; CHECK-NEXT: and r3, u4, 31
+; CHECK-NEXT: extr r2, r2, r1, r3
+; CHECK-NEXT: and r4, u4, 32
+; CHECK-NEXT: bfeil r1, 0, r1, r3
+; CHECK-NEXT: bfeil r3, 0, r4, 5
+; CHECK-NEXT: icmpsel ueq, r4, r3, 0, r2, r1
+; CHECK-NEXT: icmpsel ueq, r5, r3, 0, r1, 0
+; CHECK-NEXT: device_store 0, i32, xy, r4_r5, u0_u1, 0, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %x = shl i64 %a, 3
+ %y = lshr i64 %x, 40
+ %z = ashr i64 %y, %s
+ store i64 %z, ptr addrspace(1) %out
+ ret void
+}
+
+define agx_kernel void @compare(ptr addrspace(1) %out, i64 %a, i64 %b) {
+; CHECK-LABEL: compare:
+; CHECK: // uniform registers: 18 x 16-bit (threadgroups_per_grid at u6)
+; CHECK-NEXT: // general purpose registers: 5 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: icmpsel slt, r1, u3, u5, 1, 0
+; CHECK-NEXT: icmpsel ult, r2, u2, u4, 1, 0
+; CHECK-NEXT: icmpsel ueq, r3, u3, u5, 1, 0
+; CHECK-NEXT: icmpsel ueq, r4, u2, u4, 1, 0
+; CHECK-NEXT: icmpsel ueq, r1, u3, u5, r2, r1
+; CHECK-NEXT: and r2, r4, r3
+; CHECK-NEXT: iadd r1, r1, r2
+; CHECK-NEXT: device_store 0, i32, x, r1, u0_u1, 0, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %c = icmp slt i64 %a, %b
+ %e = icmp eq i64 %a, %b
+ %z = zext i1 %c to i32
+ %w = zext i1 %e to i32
+ %r = add i32 %z, %w
+ store i32 %r, ptr addrspace(1) %out
+ ret void
+}
+
+define agx_kernel void @bitcount(ptr addrspace(1) %out, i64 %a) {
+; CHECK-LABEL: bitcount:
+; CHECK: // uniform registers: 14 x 16-bit (threadgroups_per_grid at u4)
+; CHECK-NEXT: // general purpose registers: 7 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ffs r1, u2
+; CHECK-NEXT: ffs r2, u3
+; CHECK-NEXT: isub r1, 31, r1
+; CHECK-NEXT: isub r2, 31, r2
+; CHECK-NEXT: iadd r1, r1, 32
+; CHECK-NEXT: bitrev r3, u3
+; CHECK-NEXT: icmpsel ueq, r1, u3, 0, r1, r2
+; CHECK-NEXT: bitrev r2, u2
+; CHECK-NEXT: ffs r3, r3
+; CHECK-NEXT: iadd r4_r5, r1, 0
+; CHECK-NEXT: ffs r1, r2
+; CHECK-NEXT: isub r2, 31, r3
+; CHECK-NEXT: popcount r3, u3
+; CHECK-NEXT: popcount r6, u2
+; CHECK-NEXT: isub r1, 31, r1
+; CHECK-NEXT: iadd r2, r2, 32
+; CHECK-NEXT: iadd r3, r6, r3
+; CHECK-NEXT: icmpsel ueq, r1, u2, 0, r2, r1
+; CHECK-NEXT: iadd r2_r3, r3, 0
+; CHECK-NEXT: iadd r4_r5, r4_r5, r1
+; CHECK-NEXT: iadd r2_r3, r4_r5, r2_r3
+; CHECK-NEXT: device_store 0, i32, xy, r2_r3, u0_u1, 0, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %l = call i64 @llvm.ctlz.i64(i64 %a, i1 false)
+ %t = call i64 @llvm.cttz.i64(i64 %a, i1 false)
+ %p = call i64 @llvm.ctpop.i64(i64 %a)
+ %s = add i64 %l, %t
+ %r = add i64 %s, %p
+ store i64 %r, ptr addrspace(1) %out
+ ret void
+}
+
diff --git a/llvm/test/CodeGen/AGX/kernel-args.ll b/llvm/test/CodeGen/AGX/kernel-args.ll
new file mode 100644
index 0000000000000..4d8eb676b8aa3
--- /dev/null
+++ b/llvm/test/CodeGen/AGX/kernel-args.ll
@@ -0,0 +1,68 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -verify-machineinstrs < %s | FileCheck %s
+
+; Kernel arguments are packed into uniform registers in 16-bit units, each
+; aligned to its size, followed by threadgroups_per_grid.xyz.
+
+define agx_kernel void @args(ptr addrspace(1) %out, i16 %a, i32 %b, i64 %c, half %d, float %e) {
+; CHECK-LABEL: args:
+; CHECK: // uniform registers: 22 x 16-bit (threadgroups_per_grid at u8)
+; CHECK-NEXT: // general purpose registers: 3 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: bitop 0101, r1, u2l, 0
+; CHECK-NEXT: iadd r1, r1, u3
+; CHECK-NEXT: fadd32 r2, u6l, -0.0
+; CHECK-NEXT: iadd r1, r1, u4
+; CHECK-NEXT: fadd32 r2, r2, u7
+; CHECK-NEXT: iadd r1, r1, r2
+; CHECK-NEXT: iadd r1, r1, u9
+; CHECK-NEXT: device_store 0, i32, x, r1, u0_u1, 0, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %a32 = zext i16 %a to i32
+ %s0 = add i32 %a32, %b
+ %c32 = trunc i64 %c to i32
+ %s1 = add i32 %s0, %c32
+ %d32 = fpext half %d to float
+ %f = fadd float %d32, %e
+ %fi = bitcast float %f to i32
+ %s2 = add i32 %s1, %fi
+ %g = call i32 @llvm.agx.threadgroups.per.grid.y()
+ %s3 = add i32 %s2, %g
+ store i32 %s3, ptr addrspace(1) %out
+ ret void
+}
+
+; Without kernels in the module, every external function is an entry point.
+define void @not_a_kernel(ptr addrspace(1) %out, i32 %v) {
+; CHECK-LABEL: not_a_kernel:
+; CHECK: // uniform registers: 12 x 16-bit (threadgroups_per_grid at u3)
+; CHECK-NEXT: // general purpose registers: 2 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: bitop 0101, r1, u2, 0
+; CHECK-NEXT: device_store 0, i32, x, r1, u0_u1, 0, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+declare i32 @llvm.agx.threadgroups.per.grid.y()
diff --git a/llvm/test/CodeGen/AGX/lit.local.cfg b/llvm/test/CodeGen/AGX/lit.local.cfg
new file mode 100644
index 0000000000000..e7054032f0deb
--- /dev/null
+++ b/llvm/test/CodeGen/AGX/lit.local.cfg
@@ -0,0 +1,2 @@
+if not "AGX" in config.root.targets:
+ config.unsupported = True
diff --git a/llvm/test/CodeGen/AGX/memory.ll b/llvm/test/CodeGen/AGX/memory.ll
new file mode 100644
index 0000000000000..0dc59f41aa9d7
--- /dev/null
+++ b/llvm/test/CodeGen/AGX/memory.ll
@@ -0,0 +1,204 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -verify-machineinstrs < %s | FileCheck %s
+
+; Device addresses fold base + (extended index << shift) and constant
+; offsets; loads are asynchronous and need a wait before their use.
+
+define agx_kernel void @scaled_index(ptr addrspace(1) %a, ptr addrspace(1) %out) {
+; CHECK-LABEL: scaled_index:
+; CHECK: // uniform registers: 14 x 16-bit (threadgroups_per_grid at u4)
+; CHECK-NEXT: // general purpose registers: 3 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: get_sr r1, sr80 (thread_position_in_grid.x)
+; CHECK-NEXT: device_load 0, i32, x, r2, u0_u1, r1, unsigned
+; CHECK-NEXT: wait 0
+; CHECK-NEXT: fadd32 r2, r2, r2
+; CHECK-NEXT: device_store 0, i32, x, r2, u2_u3, r1, unsigned
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %id = call i32 @llvm.agx.thread.position.in.grid.x()
+ %idx = zext i32 %id to i64
+ %pa = getelementptr float, ptr addrspace(1) %a, i64 %idx
+ %x = load float, ptr addrspace(1) %pa
+ %y = fmul float %x, 2.0
+ %po = getelementptr float, ptr addrspace(1) %out, i64 %idx
+ store float %y, ptr addrspace(1) %po
+ ret void
+}
+
+define agx_kernel void @signed_index(ptr addrspace(1) %a, i32 %i) {
+; CHECK-LABEL: signed_index:
+; CHECK: // uniform registers: 12 x 16-bit (threadgroups_per_grid at u3)
+; CHECK-NEXT: // general purpose registers: 4 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: bitop 0101, r1, u2, 0
+; CHECK-NEXT: device_load 0, i16, x, r1l, u0_u1, r1, signed
+; CHECK-NEXT: bitop 0101, r2, u0, 0
+; CHECK-NEXT: bitop 0101, r3, u1, 0
+; CHECK-NEXT: iadd r2_r3, r2_r3, u2.sx, lsl 1
+; CHECK-NEXT: wait 0
+; CHECK-NEXT: device_store 0, i16, x, r1l, r2_r3, 4, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %p = getelementptr i16, ptr addrspace(1) %a, i32 %i
+ %x = load i16, ptr addrspace(1) %p
+ %q = getelementptr i16, ptr addrspace(1) %p, i32 4
+ store i16 %x, ptr addrspace(1) %q
+ ret void
+}
+
+define agx_kernel void @bytes(ptr addrspace(1) %a, ptr addrspace(1) %out) {
+; CHECK-LABEL: bytes:
+; CHECK: // uniform registers: 14 x 16-bit (threadgroups_per_grid at u4)
+; CHECK-NEXT: // general purpose registers: 4 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: device_load 0, i8, x, r1l, u0_u1, 0, signed
+; CHECK-NEXT: wait 0
+; CHECK-NEXT: bitop 0101, r2, r1l, 0
+; CHECK-NEXT: bfi r3, 0, r2, 24
+; CHECK-NEXT: asr r3, r3, 24
+; CHECK-NEXT: and r2, r2, 255
+; CHECK-NEXT: iadd r2, r3, r2
+; CHECK-NEXT: device_store 0, i32, x, r2, u2_u3, 0, signed
+; CHECK-NEXT: device_store 0, i8, x, r1l, u2_u3, 7, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %x = load i8, ptr addrspace(1) %a
+ %s = sext i8 %x to i32
+ %z = zext i8 %x to i32
+ %r = add i32 %s, %z
+ store i32 %r, ptr addrspace(1) %out
+ %b = getelementptr i8, ptr addrspace(1) %out, i64 7
+ store i8 %x, ptr addrspace(1) %b
+ ret void
+}
+
+define agx_kernel void @wide(ptr addrspace(1) %a, ptr addrspace(1) %out) {
+; CHECK-LABEL: wide:
+; CHECK: // uniform registers: 14 x 16-bit (threadgroups_per_grid at u4)
+; CHECK-NEXT: // general purpose registers: 4 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: device_load 0, i32, xy, r2_r3, u0_u1, 0, signed
+; CHECK-NEXT: wait 0
+; CHECK-NEXT: iadd r2_r3, r2_r3, 1
+; CHECK-NEXT: device_store 0, i32, xy, r2_r3, u2_u3, 0, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %x = load i64, ptr addrspace(1) %a
+ %y = add i64 %x, 1
+ store i64 %y, ptr addrspace(1) %out
+ ret void
+}
+
+ at tg = internal addrspace(3) global [64 x i32] undef
+
+define agx_kernel void @threadgroup(ptr addrspace(1) %out) {
+; CHECK-LABEL: threadgroup:
+; CHECK: // uniform registers: 10 x 16-bit (threadgroups_per_grid at u2)
+; CHECK-NEXT: // general purpose registers: 4 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 256 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: get_sr r1, sr51 (thread_index_in_threadgroup)
+; CHECK-NEXT: bfi r2, 0, r1, 2
+; CHECK-NEXT: xor r3, r1, 1
+; CHECK-NEXT: bfi r3, 0, r3, 2
+; CHECK-NEXT: threadgroup_store i32, x, r1, r2l, 0
+; CHECK-NEXT: threadgroup_barrier
+; CHECK-NEXT: threadgroup_load i32, x, r2, r3l, 0
+; CHECK-NEXT: device_store 0, i32, x, r2, u0_u1, r1, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %tid = call i32 @llvm.agx.thread.index.in.threadgroup()
+ %p = getelementptr [64 x i32], ptr addrspace(3) @tg, i32 0, i32 %tid
+ store i32 %tid, ptr addrspace(3) %p
+ call void @llvm.agx.threadgroup.barrier()
+ %x = xor i32 %tid, 1
+ %q = getelementptr [64 x i32], ptr addrspace(3) @tg, i32 0, i32 %x
+ %v = load i32, ptr addrspace(3) %q
+ %o = getelementptr i32, ptr addrspace(1) %out, i32 %tid
+ store i32 %v, ptr addrspace(1) %o
+ ret void
+}
+
+define agx_kernel void @stack(ptr addrspace(1) %out, i32 %i) {
+; CHECK-LABEL: stack:
+; CHECK: // uniform registers: 12 x 16-bit (threadgroups_per_grid at u3)
+; CHECK-NEXT: // general purpose registers: 4 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 112 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: stack_adjust 28
+; CHECK-NEXT: bfi r1, 0, u2, 2
+; CHECK-NEXT: bitop 0101, r2, u2, 0
+; CHECK-NEXT: mov_imm r3, 0
+; CHECK-NEXT: iadd r1, r3, r1
+; CHECK-NEXT: iadd r3, r3, 12
+; CHECK-NEXT: stack_store 0, i32, x, r2, r1
+; CHECK-NEXT: stack_load 0, i32, x, r1, r3
+; CHECK-NEXT: wait 0
+; CHECK-NEXT: device_store 0, i32, x, r1, u0_u1, 0, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %arr = alloca [16 x i32], addrspace(5)
+ %p = getelementptr [16 x i32], ptr addrspace(5) %arr, i32 0, i32 %i
+ store i32 %i, ptr addrspace(5) %p
+ %q = getelementptr [16 x i32], ptr addrspace(5) %arr, i32 0, i32 3
+ %v = load volatile i32, ptr addrspace(5) %q
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+declare i32 @llvm.agx.thread.position.in.grid.x()
+declare i32 @llvm.agx.thread.index.in.threadgroup()
+declare void @llvm.agx.threadgroup.barrier()
diff --git a/llvm/test/CodeGen/AGX/simd.ll b/llvm/test/CodeGen/AGX/simd.ll
new file mode 100644
index 0000000000000..c328c5be15d30
--- /dev/null
+++ b/llvm/test/CodeGen/AGX/simd.ll
@@ -0,0 +1,196 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -verify-machineinstrs < %s | FileCheck %s
+
+define agx_kernel void @shuffles(ptr addrspace(1) %out, float %x, i32 %i) {
+; CHECK-LABEL: shuffles:
+; CHECK: // uniform registers: 14 x 16-bit (threadgroups_per_grid at u4)
+; CHECK-NEXT: // general purpose registers: 3 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: simd_shuffle r1, u2, u3l
+; CHECK-NEXT: mov_imm r2, 1
+; CHECK-NEXT: simd_shuffle_xor r1, r1, r2l
+; CHECK-NEXT: mov_imm r2, 2
+; CHECK-NEXT: simd_shuffle_down r1, r1, r2l
+; CHECK-NEXT: mov_imm r2, 3
+; CHECK-NEXT: quad_shuffle r1, r1, r2l
+; CHECK-NEXT: device_store 0, i32, x, r1, u0_u1, 0, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %a = call float @llvm.agx.simd.shuffle.f32(float %x, i32 %i)
+ %b = call float @llvm.agx.simd.shuffle.xor.f32(float %a, i32 1)
+ %c = call float @llvm.agx.simd.shuffle.down.f32(float %b, i32 2)
+ %d = call float @llvm.agx.quad.shuffle.f32(float %c, i32 3)
+ store float %d, ptr addrspace(1) %out
+ ret void
+}
+
+define agx_kernel void @reductions(ptr addrspace(1) %out, i32 %x, float %f) {
+ ; 16: iadd, 22: smax, 1: fadd, 7: fmax
+; CHECK-LABEL: reductions:
+; CHECK: // uniform registers: 14 x 16-bit (threadgroups_per_grid at u4)
+; CHECK-NEXT: // general purpose registers: 3 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: simd_iadd r1, u2
+; CHECK-NEXT: simd_prefix_max.s r1, r1
+; CHECK-NEXT: quad_fadd r2, u3
+; CHECK-NEXT: quad_prefix_max.f r2, r2
+; CHECK-NEXT: device_store 0, i32, x, r1, u0_u1, 0, signed
+; CHECK-NEXT: device_store 0, i32, x, r2, u0_u1, 1, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %a = call i32 @llvm.agx.simd.reduce.i32(i32 %x, i32 16)
+ %b = call i32 @llvm.agx.simd.prefix.i32(i32 %a, i32 22)
+ %c = call float @llvm.agx.quad.reduce.f32(float %f, i32 1)
+ %d = call float @llvm.agx.quad.prefix.f32(float %c, i32 7)
+ store i32 %b, ptr addrspace(1) %out
+ %o = getelementptr float, ptr addrspace(1) %out, i32 1
+ store float %d, ptr addrspace(1) %o
+ ret void
+}
+
+define agx_kernel void @ballots(ptr addrspace(1) %out, i32 %x) {
+; CHECK-LABEL: ballots:
+; CHECK: // uniform registers: 12 x 16-bit (threadgroups_per_grid at u3)
+; CHECK-NEXT: // general purpose registers: 4 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: icmpsel ugt, r1, u2, 5, 1, 0
+; CHECK-NEXT: icmp_quad_ballot r2, nueq, r1, 0
+; CHECK-NEXT: icmp_ballot r1, nueq, r1, 0
+; CHECK-NEXT: and r2, r2, 15
+; CHECK-NEXT: get_sr r3, sr58 (active_thread_index_in_simdgroup)
+; CHECK-NEXT: iadd r1, r1, r2
+; CHECK-NEXT: iadd r1, r1, r3
+; CHECK-NEXT: device_store 0, i32, x, r1, u0_u1, 0, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %c = icmp ugt i32 %x, 5
+ %a = call i32 @llvm.agx.simd.ballot(i1 %c)
+ %q = call i32 @llvm.agx.quad.ballot(i1 %c)
+ %n = call i32 @llvm.agx.active.thread.index.in.simdgroup()
+ %s = add i32 %a, %q
+ %r = add i32 %s, %n
+ store i32 %r, ptr addrspace(1) %out
+ ret void
+}
+
+define agx_kernel void @special_registers(ptr addrspace(1) %out) {
+; CHECK-LABEL: special_registers:
+; CHECK: // uniform registers: 10 x 16-bit (threadgroups_per_grid at u2)
+; CHECK-NEXT: // general purpose registers: 4 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: get_sr r1, sr49 (thread_position_in_threadgroup.y)
+; CHECK-NEXT: get_sr r2, sr2 (threadgroup_position_in_grid.z)
+; CHECK-NEXT: get_sr r3, sr4 (threads_per_threadgroup.x)
+; CHECK-NEXT: iadd r1, r1, r2
+; CHECK-NEXT: get_sr r2, sr53 (simdgroup_index_in_threadgroup)
+; CHECK-NEXT: iadd r1, r1, r3
+; CHECK-NEXT: get_sr r3, sr20 (core_index)
+; CHECK-NEXT: iadd r1, r1, r2
+; CHECK-NEXT: iadd r1, r1, r3
+; CHECK-NEXT: device_store 0, i32, x, r1, u0_u1, 0, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %a = call i32 @llvm.agx.thread.position.in.threadgroup.y()
+ %b = call i32 @llvm.agx.threadgroup.position.in.grid.z()
+ %c = call i32 @llvm.agx.threads.per.threadgroup.x()
+ %d = call i32 @llvm.agx.simdgroup.index.in.threadgroup()
+ %e = call i32 @llvm.agx.read.sr(i32 20)
+ %s0 = add i32 %a, %b
+ %s1 = add i32 %s0, %c
+ %s2 = add i32 %s1, %d
+ %s3 = add i32 %s2, %e
+ store i32 %s3, ptr addrspace(1) %out
+ ret void
+}
+
+define agx_kernel void @special_functions(ptr addrspace(1) %out, float %x) {
+; CHECK-LABEL: special_functions:
+; CHECK: // uniform registers: 12 x 16-bit (threadgroups_per_grid at u3)
+; CHECK-NEXT: // general purpose registers: 2 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: rcp r1, u2
+; CHECK-NEXT: rsqrt r1, r1
+; CHECK-NEXT: dfdx r1, r1
+; CHECK-NEXT: dfdy r1, r1
+; CHECK-NEXT: device_store 0, i32, x, r1, u0_u1, 0, signed
+; CHECK-NEXT: memory_barrier 1, 2, 9
+; CHECK-NEXT: wait 1
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %a = call float @llvm.agx.rcp.f32(float %x)
+ %b = call float @llvm.agx.rsqrt.f32(float %a)
+ %c = call float @llvm.agx.dfdx.f32(float %b)
+ %d = call float @llvm.agx.dfdy.f32(float %c)
+ store float %d, ptr addrspace(1) %out
+ call void @llvm.agx.memory.barrier(i32 1, i32 2, i32 9)
+ call void @llvm.agx.wait(i32 1)
+ ret void
+}
+
+declare float @llvm.agx.simd.shuffle.f32(float, i32)
+declare float @llvm.agx.simd.shuffle.xor.f32(float, i32)
+declare float @llvm.agx.simd.shuffle.down.f32(float, i32)
+declare float @llvm.agx.quad.shuffle.f32(float, i32)
+declare i32 @llvm.agx.simd.reduce.i32(i32, i32)
+declare i32 @llvm.agx.simd.prefix.i32(i32, i32)
+declare float @llvm.agx.quad.reduce.f32(float, i32)
+declare float @llvm.agx.quad.prefix.f32(float, i32)
+declare i32 @llvm.agx.simd.ballot(i1)
+declare i32 @llvm.agx.quad.ballot(i1)
+declare i32 @llvm.agx.active.thread.index.in.simdgroup()
+declare i32 @llvm.agx.thread.position.in.threadgroup.y()
+declare i32 @llvm.agx.threadgroup.position.in.grid.z()
+declare i32 @llvm.agx.threads.per.threadgroup.x()
+declare i32 @llvm.agx.simdgroup.index.in.threadgroup()
+declare i32 @llvm.agx.read.sr(i32)
+declare float @llvm.agx.rcp.f32(float)
+declare float @llvm.agx.rsqrt.f32(float)
+declare float @llvm.agx.dfdx.f32(float)
+declare float @llvm.agx.dfdy.f32(float)
+declare void @llvm.agx.memory.barrier(i32, i32, i32)
+declare void @llvm.agx.wait(i32)
diff --git a/llvm/utils/UpdateTestChecks/asm.py b/llvm/utils/UpdateTestChecks/asm.py
index 17aae40ea319b..d81505be748e1 100644
--- a/llvm/utils/UpdateTestChecks/asm.py
+++ b/llvm/utils/UpdateTestChecks/asm.py
@@ -60,6 +60,13 @@ class string:
flags=(re.M | re.S),
)
+ASM_FUNCTION_AGX_RE = re.compile(
+ r'^_?(?P<func>[^:]+):[ \t]*//+[ \t]*@"?(?P=func)"?\n'
+ r"(?P<body>.*?)\n" # (body of the function)
+ r"^\s*\.Lfunc_end[0-9]+:\n",
+ flags=(re.M | re.S),
+)
+
ASM_FUNCTION_BPF_RE = re.compile(
r'^_?(?P<func>[^:]+):[ \t]*#+[ \t]*@"?(?P=func)"?\n'
r"(?:[ \t]+.cfi_startproc\n|.seh_proc[^\n]+\n)?" # drop optional cfi
@@ -455,6 +462,17 @@ def scrub_asm_lanai(asm, args):
return asm
+def scrub_asm_agx(asm, args):
+ # Scrub runs of whitespace out of the assembly, but leave the leading
+ # whitespace in place.
+ asm = common.SCRUB_WHITESPACE_RE.sub(r" ", asm)
+ # Expand the tabs used for indentation.
+ asm = string.expandtabs(asm, 2)
+ # Strip trailing whitespace.
+ asm = common.SCRUB_TRAILING_WHITESPACE_RE.sub(r"", asm)
+ return asm
+
+
def scrub_asm_sparc(asm, args):
# Scrub runs of whitespace out of the assembly, but leave the leading
# whitespace in place.
@@ -605,6 +623,7 @@ def get_run_handler(triple):
"riscv32-apple-none-macho": (scrub_asm_riscv, ASM_FUNCTION_RISCV_MACHO_RE),
"riscv64": (scrub_asm_riscv, ASM_FUNCTION_RISCV_RE),
"lanai": (scrub_asm_lanai, ASM_FUNCTION_LANAI_RE),
+ "agx": (scrub_asm_agx, ASM_FUNCTION_AGX_RE),
"sparc": (scrub_asm_sparc, ASM_FUNCTION_SPARC_RE),
"spirv": (scrub_asm_spirv, ASM_FUNCTION_SPIRV_RE),
"spirv32": (scrub_asm_spirv, ASM_FUNCTION_SPIRV_RE),
>From 0428472b40d8fc60348aeb62f12b8883b27ba6ae Mon Sep 17 00:00:00 2001
From: Claude <noreply at anthropic.com>
Date: Sun, 18 Oct 2026 20:12:52 +0000
Subject: [PATCH 04/22] [AGX] Document the backend; set the coherency bit on
multi-die parts
Add llvm/docs/AGXUsage.rst (processors, features, address spaces, kernel
ABI, execution model and the state of Apple9 support) and set the
coherency bit of device atomics when multi-die-coherency is enabled.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
Claude-Session: https://claude.ai/code/session_01E56KEecg6edNaS23myRWER
---
llvm/docs/AGXUsage.rst | 267 +++++++++++++++++++++++++
llvm/docs/UserGuides.md | 6 +
llvm/lib/Target/AGX/AGXMCInstLower.cpp | 9 +
llvm/test/CodeGen/AGX/atomics.ll | 4 +
4 files changed, 286 insertions(+)
create mode 100644 llvm/docs/AGXUsage.rst
diff --git a/llvm/docs/AGXUsage.rst b/llvm/docs/AGXUsage.rst
new file mode 100644
index 0000000000000..90ba9227ede93
--- /dev/null
+++ b/llvm/docs/AGXUsage.rst
@@ -0,0 +1,267 @@
+=============================================
+User Guide for the AGX Backend (Experimental)
+=============================================
+
+.. contents::
+ :local:
+ :depth: 2
+
+Introduction
+============
+
+The AGX backend generates machine code for the GPUs of Apple silicon (the
+"AGX" architecture of the M1 and later SoCs). It is an experimental target:
+build it with ``-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=AGX``.
+
+Apple does not document the instruction set. Everything in this backend comes
+from public reverse engineering:
+
+* `applegpu <https://github.com/dougallj/applegpu>`_, the reference for the
+ G13 instruction encodings. The MC layer is tested against it: every
+ encoding produced by the assembler and the code generator decodes to the
+ same instruction in applegpu's disassembler.
+* The Mesa ``asahi`` driver, the reference for the code that the hardware
+ expects in practice (control flow on the nesting counter, scoreboarding,
+ memory addressing). Only facts about the hardware were taken from it.
+* Measurements of instruction latencies on G14 (M2 Max), used for the
+ scheduling models.
+* Reverse engineering notes on the G16/G17 ("Apple9") generation. These are
+ incomplete, which is why code generation for those GPUs is not supported
+ yet (see :ref:`agx-apple9`).
+
+The backend is modelled on the AMDGPU backend: the same split between code
+generation pseudo instructions and real, per encoding family instructions,
+the same structurization of control flow and the same layout of the
+sources.
+
+Target triples
+==============
+
+Use ``agx`` as the architecture, for example ``agx-apple-macosx`` or simply
+``-mtriple=agx``. The triple has no effect on code generation beyond selecting
+the target. Object files are ELF (``EM_AGX``, ``0x4147``, a value that is not
+registered); they contain the raw instructions of each kernel in ``.text``.
+
+Processors
+==========
+
+Use ``-mcpu`` (or the ``target-cpu`` function attribute) to select the GPU.
+
+.. table:: AGX processors
+ :name: agx-processors-table
+
+ =============== ========== ================= =====================================
+ Processor Generation Encoding family Products
+ =============== ========== ================= =====================================
+ ``generic`` G13 AGX2 Any G13-G15 GPU
+ ``agx-g13g`` G13 AGX2 M1
+ ``agx-g13s`` G13 AGX2 M1 Pro (``multi-die-coherency``)
+ ``agx-g13c`` G13 AGX2 M1 Max (``multi-die-coherency``)
+ ``agx-g13d`` G13 AGX2 M1 Ultra (``multi-die-coherency``)
+ ``agx-g14g`` G14 AGX2 M2
+ ``agx-g14s`` G14 AGX2 M2 Pro
+ ``agx-g14c`` G14 AGX2 M2 Max
+ ``agx-g14d`` G14 AGX2 M2 Ultra
+ ``agx-g15g`` G15 AGX2 M3
+ ``agx-g15s`` G15 AGX2 M3 Pro
+ ``agx-g15c`` G15 AGX2 M3 Max
+ ``agx-g15d`` G15 AGX2 M3 Ultra
+ ``agx-g16g`` G16 Apple9 M4
+ ``agx-g16s`` G16 Apple9 M4 Pro
+ ``agx-g16c`` G16 Apple9 M4 Max
+ ``agx-g17p`` G17 Apple9 A18 Pro
+ ``agx-g17g`` G17 Apple9 M5
+ =============== ========== ================= =====================================
+
+G15 support assumes the G14 encoding; it has not been validated on hardware.
+
+Target features
+===============
+
+Each generation implies a set of features. Features can also be enabled or
+disabled individually with ``-mattr`` (or the ``target-features`` attribute),
+which allows code for one generation to be restricted to the capabilities of
+another.
+
+.. table:: AGX target features
+ :name: agx-features-table
+
+ ============================== ==============================================
+ Feature Description
+ ============================== ==============================================
+ ``agx2-encoding`` G13/G14 instruction encoding.
+ ``apple9-encoding`` G16/G17 instruction encoding.
+ ``fp16-arith`` Native half precision arithmetic.
+ ``int64-add`` 64-bit ``iadd``/``imadd`` operands.
+ ``simd-shuffle`` SIMD-group and quad shuffles.
+ ``simd-reduce`` SIMD-group and quad reductions and prefixes.
+ ``simd-ballot`` Ballots.
+ ``simd-matrix`` 8x8 SIMD-group matrix multiply-accumulate.
+ ``quad-shuffle-index-merge`` ``simd_shuffle`` merges the low index bits of
+ each quad (informational).
+ ``multi-die-coherency`` Device atomics set the coherency bit (multi-die
+ G13 parts).
+ ``threadgroup-atomics`` Threadgroup memory atomics.
+ ``device-atomics`` Device memory atomics.
+ ``stack-access`` Per-thread stack loads and stores.
+ ``dynamic-caching`` Registers and threadgroup memory are
+ allocated dynamically (G15 and later).
+ ``ray-tracing`` Hardware ray tracing (not used by code
+ generation).
+ ``mesh-shading`` Hardware mesh shading (not used by code
+ generation).
+ ``source-lifetime`` ALU sources carry register release flags
+ (Apple9).
+ ``max-gprs-128``, ``-64`` Limit the general purpose registers used.
+ ============================== ==============================================
+
+Address spaces
+==============
+
+.. table:: AGX address spaces
+ :name: agx-address-spaces-table
+
+ ============= ============ ============= =================================
+ Address space Name Pointer size Memory
+ ============= ============ ============= =================================
+ 0 generic 64 Device memory
+ 1 device 64 Device memory
+ 3 threadgroup 32 Threadgroup (shared) memory
+ 4 constant 64 Device memory, read only
+ 5 private 32 Per-thread stack
+ ============= ============ ============= =================================
+
+Generic, device and constant pointers are interchangeable: casts between them
+are no-ops. There is no generic addressing of threadgroup or private memory.
+
+Kernel ABI
+==========
+
+Kernels use the ``agx_kernel`` calling convention and return ``void``. There
+are no calls: every other function is inlined, and the backend reports an
+error for calls that remain. A module without ``agx_kernel`` functions
+compiles each external function as an entry point with the kernel ABI.
+
+Kernel arguments are passed in uniform registers. They are packed in order in
+16-bit units (``u0l``, ``u0h``, ``u1l`` ...), each argument aligned to its
+size and 64-bit arguments to 32 bits; arguments are split into their legal
+parts first (``i8`` and ``i1`` take 16 bits, vectors one unit per element).
+The 32-bit values ``threadgroups_per_grid.x``, ``.y`` and ``.z`` follow the
+last argument, aligned to 32 bits; ``llvm.agx.threadgroups.per.grid.*``
+read them.
+
+Before each kernel, the assembly printer emits comments with the resources
+the driver must provide::
+
+ // uniform registers: 18 x 16-bit (threadgroups_per_grid at u6)
+ // general purpose registers: 7 x 32-bit
+ // threadgroup memory: 0 bytes
+ // stack size: 0 bytes
+
+Each kernel ends with ``stop`` followed by eight ``trap`` instructions, since
+the instruction prefetcher reads past the end of the program.
+
+Threadgroup variables are globals in address space 3 without an initializer.
+They are allocated per kernel in threadgroup memory, at offsets in order of
+first use.
+
+Execution model
+===============
+
+A SIMD-group has 32 threads. There is no scalar unit and no scalar branch:
+every branch is executed by the SIMD-group as a whole, under an execution
+mask kept in the 16-bit nesting counter ``r0l`` of each thread (a thread is
+active when its counter is zero). The code generator therefore structurizes
+every function, uniform branches included, and annotates it with:
+
+* ``llvm.agx.if``: ``if_icmp r0l, nueq, cond, 0, 1`` deactivates the threads
+ where the condition is false, followed by ``jmp_exec_none`` to skip the
+ region when no thread remains.
+* ``llvm.agx.else``: ``else_fcmp r0l, eq, 0.0, 0.0, 1`` swaps the threads of
+ the innermost level.
+* ``llvm.agx.end.cf``: ``pop_exec r0l, 1`` reactivates the threads of the
+ innermost construct. It is emitted at the start of the reconvergence
+ block.
+* ``llvm.agx.loop.begin``: ``push_exec r0l, 1`` in the loop preheader.
+* ``llvm.agx.loop``: ``while_icmp r0l, ueq, break, 0, 1`` at the latch
+ deactivates the threads leaving the loop, and ``jmp_exec_any`` branches back
+ while any thread remains.
+
+The nesting counter is zeroed at the start of each kernel that uses control
+flow. Blocks are laid out in the reverse post-order of the structured graph,
+so constructs fall through into each other as in Mesa's output.
+
+Memory loads, stack loads and atomics complete asynchronously. A ``wait`` is
+inserted before the first instruction that reads or overwrites a pending
+result, before barriers and at the end of each block.
+
+Registers
+=========
+
+``r0`` to ``r127`` are the 32-bit general purpose registers, with 16-bit
+halves ``rNl`` and ``rNh``. ``r0`` is reserved for the nesting counter. 64-bit
+ALU values use aligned register pairs; memory operations accept any register
+tuple. ``u0`` to ``u255`` are the uniform registers. They are never allocated;
+copies from them are folded into the ALU sources and memory bases that can
+read them (64-bit ALU sources cannot be uniform).
+
+Code generation
+===============
+
+Supported:
+
+* 16, 32 and 64-bit integer arithmetic, including 64-bit multiplication,
+ shifts, comparisons and bit counting. Division is expanded in IR, except
+ by constants.
+* Half and single precision arithmetic, with source modifiers and 8-bit
+ floating point immediates folded. Division uses ``rcp`` and a multiply
+ with a Newton-Raphson step (omitted with ``arcp``/``afn`` and for
+ ``half``); ``sin``/``cos`` reduce to quadrants for ``sin_pt_1`` and
+ ``sin_pt_2``; ``exp``, ``log`` and ``pow`` go through ``exp2``/``log2``.
+ Conversions between 64-bit integers and ``float`` are expanded.
+* Device memory loads and stores of 8 to 64 bits with folded base, scaled
+ index and constant offsets; threadgroup and stack memory; memory
+ intrinsics (inline or as loops).
+* Device and threadgroup 32-bit atomics. Other atomic operations become
+ compare-and-swap loops; atomic loads and stores become atomic operations.
+* SIMD-group and quad shuffles, ballots, reductions and prefixes, derivatives,
+ barriers and special registers through the ``llvm.agx.*`` intrinsics.
+
+Not supported: ``double``, vector memory operations (vectors are
+scalarized), textures and graphics shaders, calls and recursion.
+
+.. _agx-apple9:
+
+Apple9 (G16/G17)
+================
+
+The G16 and G17 GPUs use a different instruction encoding, of which too
+little is known to generate code reliably. The processors are defined so that
+features and scheduling can be selected, but instruction emission reports
+``no AGX instruction encoding for ... (the Apple9 encoding is not yet
+supported)``. Pseudo instructions carry an ``Apple9Real`` field for the
+real instruction of that family, to be filled in as encodings are confirmed.
+
+Intrinsics
+==========
+
+See ``llvm/include/llvm/IR/IntrinsicsAGX.td``. The special register
+intrinsics (``llvm.agx.thread.position.in.grid.x`` and so on) read
+``get_sr`` registers, ``llvm.agx.read.sr`` reads any special register by
+number, and ``llvm.agx.simd.reduce``/``prefix`` and
+``llvm.agx.quad.reduce``/``prefix`` take the operation as an immediate:
+
+.. table:: SIMD-group reduction operations
+ :name: agx-simd-ops-table
+
+ ===== ========= ===== =========
+ Value Operation Value Operation
+ ===== ========= ===== =========
+ 0 and 16 iadd
+ 1 fadd 20 smin
+ 2 or 22 smax
+ 3 fmul 28 umin
+ 4 xor 30 umax
+ 5 fmin
+ 7 fmax
+ ===== ========= ===== =========
diff --git a/llvm/docs/UserGuides.md b/llvm/docs/UserGuides.md
index f7e971915b669..f8f6e4e05dfb8 100644
--- a/llvm/docs/UserGuides.md
+++ b/llvm/docs/UserGuides.md
@@ -61,6 +61,7 @@ AArch64SME
CompileCudaWithLLVM
NVPTXUsage
AMDGPUUsage
+AGXUsage
AMDGPUAsyncOperations
AMDGPUDMAOperations
AMDGPUExecutionSynchronization
@@ -346,6 +347,11 @@ yaml2obj
This document describes using the AMDGPU backend to compile GPU kernels.
+- {doc}`AGXUsage`
+
+ This document describes using the experimental AGX backend to compile
+ kernels for Apple GPUs.
+
- {doc}`AMDGPUAsyncOperations`
Asynchronous operations and the asyncmark mechanism for tracking their
diff --git a/llvm/lib/Target/AGX/AGXMCInstLower.cpp b/llvm/lib/Target/AGX/AGXMCInstLower.cpp
index 38d95ba76990a..89acf5944949f 100644
--- a/llvm/lib/Target/AGX/AGXMCInstLower.cpp
+++ b/llvm/lib/Target/AGX/AGXMCInstLower.cpp
@@ -76,6 +76,15 @@ bool AGXMCInstLower::lower(const MachineInstr *MI, MCInst &OutMI) const {
if (Idx++ == 0 && InsertCache)
OutMI.addOperand(MCOperand::createImm(0));
}
+ // Device atomics on parts with several dies must bypass the per-die
+ // caches.
+ if ((MI->getOpcode() == AGX::DEVICE_ATOMIC ||
+ MI->getOpcode() == AGX::DEVICE_ATOMIC_CMPXCHG) &&
+ ST.hasMultiDieCoherency()) {
+ int Idx = AGX::getNamedOperandIdx(Opcode, AGX::OpName::coherent);
+ OutMI.getOperand(Idx).setImm(1);
+ }
+
assert(OutMI.getNumOperands() == RealDesc.getNumOperands() &&
"pseudo and real instruction operands do not match");
return true;
diff --git a/llvm/test/CodeGen/AGX/atomics.ll b/llvm/test/CodeGen/AGX/atomics.ll
index c56c08cd3270e..0e8249b76ed8c 100644
--- a/llvm/test/CodeGen/AGX/atomics.ll
+++ b/llvm/test/CodeGen/AGX/atomics.ll
@@ -1,5 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=agx -mcpu=agx-g13g -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=agx -mcpu=agx-g13c < %s | FileCheck %s --check-prefix=MULTIDIE
+
+; Multi-die parts set the coherency bit of device atomics.
+; MULTIDIE: atomic add, 0, r{{[0-9]+}}, u0_u1, 0, signed, r{{[0-9]+}}, 1
define agx_kernel void @device_rmw(ptr addrspace(1) %p, ptr addrspace(1) %out, i32 %v) {
; CHECK-LABEL: device_rmw:
>From b719cddec80058189ee8a953a5805dacd3b6d5e0 Mon Sep 17 00:00:00 2001
From: Claude <noreply at anthropic.com>
Date: Sun, 18 Oct 2026 20:35:21 +0000
Subject: [PATCH 05/22] [AGX] Support device globals, traps and constrained FP;
drop unused functions
Found by compiling ZLUDA's PTX test kernels:
* Device and constant global variables live in a global data buffer whose
address kernels receive after threadgroups_per_grid. The initial contents
are emitted in the .agx.globals section, in module order. Threadgroup
variables are no longer emitted as data.
* llvm.trap and llvm.debugtrap select trap instead of calling abort.
* Constrained floating point intrinsics become ordinary operations: there
is no floating point environment.
* bfloat16 conversions are expanded.
* In modules with kernels, all other functions are internalized and the
unused ones removed (for example from linked device libraries).
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
Claude-Session: https://claude.ai/code/session_01E56KEecg6edNaS23myRWER
---
llvm/lib/Target/AGX/AGXAsmPrinter.cpp | 44 +++++++++-
llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp | 86 +++++++++++++++++--
llvm/lib/Target/AGX/AGXISelLowering.cpp | 38 +++++++-
llvm/lib/Target/AGX/AGXInstructions.td | 2 +
.../lib/Target/AGX/AGXMachineFunctionInfo.cpp | 32 +++++++
llvm/lib/Target/AGX/AGXMachineFunctionInfo.h | 37 ++++++++
llvm/test/CodeGen/AGX/always-inline.ll | 25 ++++++
llvm/test/CodeGen/AGX/constrained-fp.ll | 20 +++++
llvm/test/CodeGen/AGX/control-flow.ll | 43 ++++++++++
llvm/test/CodeGen/AGX/entry-points.ll | 27 ++++++
llvm/test/CodeGen/AGX/global-data.ll | 58 +++++++++++++
llvm/test/CodeGen/AGX/kernel-args.ll | 23 -----
12 files changed, 399 insertions(+), 36 deletions(-)
create mode 100644 llvm/test/CodeGen/AGX/always-inline.ll
create mode 100644 llvm/test/CodeGen/AGX/constrained-fp.ll
create mode 100644 llvm/test/CodeGen/AGX/entry-points.ll
create mode 100644 llvm/test/CodeGen/AGX/global-data.ll
diff --git a/llvm/lib/Target/AGX/AGXAsmPrinter.cpp b/llvm/lib/Target/AGX/AGXAsmPrinter.cpp
index f5ff5699995e4..bbf183924bbb8 100644
--- a/llvm/lib/Target/AGX/AGXAsmPrinter.cpp
+++ b/llvm/lib/Target/AGX/AGXAsmPrinter.cpp
@@ -10,7 +10,8 @@
// resources the driver must allocate for it (uniform registers, general
// purpose registers, threadgroup memory and stack), and followed by trap
// padding, since the instruction prefetcher reads past the end of the
-// program.
+// program. The initial contents of the global data buffer are emitted in the
+// .agx.globals section.
//
//===----------------------------------------------------------------------===//
@@ -23,9 +24,13 @@
#include "TargetInfo/AGXTargetInfo.h"
#include "llvm/CodeGen/AsmPrinter.h"
#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/BinaryFormat/ELF.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/Module.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCSectionELF.h"
#include "llvm/MC/MCStreamer.h"
#include "llvm/MC/TargetRegistry.h"
#include "llvm/Support/Compiler.h"
@@ -53,6 +58,8 @@ class AGXAsmPrinter : public AsmPrinter {
return AsmPrinter::runOnMachineFunction(MF);
}
+ void emitGlobalVariable(const GlobalVariable *GV) override;
+ void emitEndOfAsmFile(Module &M) override;
void emitFunctionBodyStart() override;
void emitFunctionBodyEnd() override;
void emitInstruction(const MachineInstr *MI) override;
@@ -85,6 +92,10 @@ void AGXAsmPrinter::emitFunctionBodyStart() {
Twine(MFI->getImplicitArgHalfIndex() / 2) + ")");
OutStreamer->emitRawComment(" general purpose registers: " +
Twine(getGPRCount(*MF)) + " x 32-bit");
+ if (MFI->getGlobalDataVReg())
+ OutStreamer->emitRawComment(
+ " global data buffer at u" + Twine(MFI->getGlobalDataHalfIndex() / 2) +
+ "_u" + Twine(MFI->getGlobalDataHalfIndex() / 2 + 1));
OutStreamer->emitRawComment(" threadgroup memory: " +
Twine(MFI->getThreadgroupSize()) + " bytes");
OutStreamer->emitRawComment(" stack size: " +
@@ -93,6 +104,37 @@ void AGXAsmPrinter::emitFunctionBodyStart() {
" bytes");
}
+void AGXAsmPrinter::emitGlobalVariable(const GlobalVariable *GV) {
+ // Threadgroup variables are allocated per kernel, and device variables are
+ // emitted together by emitEndOfAsmFile.
+ if (GV->getAddressSpace() == AGX::THREADGROUP_ADDRESS ||
+ AGX::isGlobalDataVariable(*GV))
+ return;
+ AsmPrinter::emitGlobalVariable(GV);
+}
+
+void AGXAsmPrinter::emitEndOfAsmFile(Module &M) {
+ AGX::GlobalDataLayout Layout = AGX::computeGlobalDataLayout(M);
+ if (Layout.empty())
+ return;
+ const DataLayout &DL = M.getDataLayout();
+ OutStreamer->switchSection(OutContext.getELFSection(
+ ".agx.globals", ELF::SHT_PROGBITS, ELF::SHF_ALLOC | ELF::SHF_WRITE));
+ OutStreamer->emitValueToAlignment(Layout.MaxAlign);
+ uint64_t End = 0;
+ for (const auto &[GV, Offset] : Layout.Offsets) {
+ OutStreamer->emitZeros(Offset - End);
+ OutStreamer->emitLabel(getSymbol(GV));
+ uint64_t Size = DL.getTypeAllocSize(GV->getValueType());
+ const Constant *Init = GV->hasInitializer() ? GV->getInitializer() : nullptr;
+ if (Init && !isa<UndefValue>(Init))
+ emitGlobalConstant(DL, Init);
+ else
+ OutStreamer->emitZeros(Size);
+ End = Offset + Size;
+ }
+}
+
void AGXAsmPrinter::emitFunctionBodyEnd() {
const auto &ST = MF->getSubtarget<AGXSubtarget>();
if (ST.getEncodingFamily() != AGXSubtarget::AGX2)
diff --git a/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp b/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp
index ccabfe5ce075b..aa29c93cf2a5c 100644
--- a/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp
@@ -9,11 +9,14 @@
// IR transformations that AGX code generation relies on:
//
// * AGXAlwaysInline marks every non-kernel function always_inline (and
-// internal, so it disappears once inlined), since there are no calls.
+// internal, so it disappears once inlined), since there are no calls,
+// and removes the unused ones.
// * AGXCodeGenPrepare expands what the hardware has no instruction or
// library for: integer division, and memory intrinsics that are not
-// small enough to be expanded by instruction selection. It also unifies
-// the function exits, which the control flow structurizer expects.
+// small enough to be expanded by instruction selection. There is no
+// floating point environment, so constrained floating point intrinsics
+// become their ordinary operations. It also unifies the function exits,
+// which the control flow structurizer expects.
//
//===----------------------------------------------------------------------===//
@@ -70,20 +73,32 @@ bool AGXAlwaysInline::runOnModule(Module &M) {
});
bool Changed = false;
+ SmallVector<Function *, 8> Dead;
for (Function &F : M) {
if (F.isDeclaration() || F.getCallingConv() == CallingConv::AGX_KERNEL)
continue;
if (!HasKernels && !F.hasLocalLinkage())
continue;
- if (F.hasFnAttribute(Attribute::NoInline))
- continue;
- F.addFnAttr(Attribute::AlwaysInline);
- if (HasKernels && !F.hasLocalLinkage() && !F.use_empty()) {
+ // With kernels in the module, other functions are only reachable from
+ // them: internalize them so that they disappear once inlined, and drop
+ // the unused ones (for example from linked device libraries).
+ if (HasKernels && !F.hasLocalLinkage()) {
F.setLinkage(GlobalValue::InternalLinkage);
F.setVisibility(GlobalValue::DefaultVisibility);
+ Changed = true;
+ }
+ if (F.use_empty()) {
+ Dead.push_back(&F);
+ continue;
}
+ if (F.hasFnAttribute(Attribute::NoInline))
+ continue;
+ F.addFnAttr(Attribute::AlwaysInline);
Changed = true;
}
+ for (Function *F : Dead)
+ F->eraseFromParent();
+ Changed |= !Dead.empty();
return Changed;
}
@@ -97,6 +112,7 @@ class AGXCodeGenPrepare : public FunctionPass {
bool expandDivisions(Function &F);
bool expandMemIntrinsics(Function &F, const TargetTransformInfo &TTI);
bool unifyExits(Function &F);
+ bool lowerConstrainedFP(Function &F);
public:
static char ID;
@@ -198,6 +214,59 @@ bool AGXCodeGenPrepare::expandMemIntrinsics(Function &F,
return !Worklist.empty();
}
+bool AGXCodeGenPrepare::lowerConstrainedFP(Function &F) {
+ SmallVector<ConstrainedFPIntrinsic *, 8> Worklist;
+ for (Instruction &I : instructions(F))
+ if (auto *CI = dyn_cast<ConstrainedFPIntrinsic>(&I))
+ Worklist.push_back(CI);
+
+ for (ConstrainedFPIntrinsic *CI : Worklist) {
+ IRBuilder<> B(CI);
+ SmallVector<Value *, 4> Args(CI->args().begin(),
+ CI->args().begin() +
+ CI->getNonMetadataArgCount());
+ Value *Res = nullptr;
+ switch (CI->getIntrinsicID()) {
+#define INSTRUCTION(NAME, NARGS, ROUND_MODE, INTRINSIC) \
+ case Intrinsic::INTRINSIC: \
+ if (Instruction::isBinaryOp(Instruction::NAME)) \
+ Res = B.CreateBinOp(Instruction::BinaryOps(Instruction::NAME), Args[0], \
+ Args[1]); \
+ else \
+ Res = B.CreateCast(Instruction::CastOps(Instruction::NAME), Args[0], \
+ CI->getType()); \
+ break;
+#define CMP_INSTRUCTION(NAME, NARGS, ROUND_MODE, INTRINSIC, DAGN) \
+ case Intrinsic::INTRINSIC: \
+ Res = B.CreateFCmp(cast<ConstrainedFPCmpIntrinsic>(CI)->getPredicate(), \
+ Args[0], Args[1]); \
+ break;
+#define FUNCTION(NAME, NARGS, ROUND_MODE, INTRINSIC) \
+ case Intrinsic::INTRINSIC: { \
+ SmallVector<Type *, 4> ArgTys; \
+ for (Value *A : Args) \
+ ArgTys.push_back(A->getType()); \
+ SmallVector<Type *, 2> OverloadTys; \
+ FunctionType *FT = FunctionType::get(CI->getType(), ArgTys, false); \
+ if (Intrinsic::isSignatureValid(Intrinsic::NAME, FT, OverloadTys)) \
+ Res = B.CreateCall(Intrinsic::getOrInsertDeclaration( \
+ F.getParent(), Intrinsic::NAME, OverloadTys), \
+ Args); \
+ break; \
+ }
+#include "llvm/IR/ConstrainedOps.def"
+ default:
+ break;
+ }
+ if (!Res)
+ continue;
+ Res->takeName(CI);
+ CI->replaceAllUsesWith(Res);
+ CI->eraseFromParent();
+ }
+ return !Worklist.empty();
+}
+
/// Replace the returns, and unreachable terminators, by branches to a single
/// return block.
bool AGXCodeGenPrepare::unifyExits(Function &F) {
@@ -226,7 +295,8 @@ bool AGXCodeGenPrepare::runOnFunction(Function &F) {
return false;
const TargetTransformInfo &TTI =
getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
- bool Changed = expandDivisions(F);
+ bool Changed = lowerConstrainedFP(F);
+ Changed |= expandDivisions(F);
Changed |= expandMemIntrinsics(F, TTI);
Changed |= unifyExits(F);
return Changed;
diff --git a/llvm/lib/Target/AGX/AGXISelLowering.cpp b/llvm/lib/Target/AGX/AGXISelLowering.cpp
index 32513fa7fbc05..7a5779c4e618c 100644
--- a/llvm/lib/Target/AGX/AGXISelLowering.cpp
+++ b/llvm/lib/Target/AGX/AGXISelLowering.cpp
@@ -72,6 +72,8 @@ AGXTargetLowering::AGXTargetLowering(const TargetMachine &TM,
ISD::DYNAMIC_STACKALLOC},
{MVT::i32, MVT::Other}, Expand);
setOperationAction(ISD::GlobalAddress, {MVT::i32, MVT::i64}, Custom);
+ // trap is an instruction; there is no debugger to stop in.
+ setOperationAction({ISD::TRAP, ISD::DEBUGTRAP}, MVT::Other, Legal);
setOperationAction({ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_VOID},
MVT::Other, Custom);
// llvm.agx.simd.ballot takes an i1, which is promoted by custom lowering.
@@ -134,6 +136,9 @@ AGXTargetLowering::AGXTargetLowering(const TargetMachine &TM,
ISD::FLOG, ISD::FLOG10, ISD::FPOW},
MVT::f16, Promote);
setOperationAction({ISD::FP16_TO_FP, ISD::FP_TO_FP16}, MVT::f32, Expand);
+ // bfloat16 is not a legal type: conversions are integer operations.
+ setOperationAction({ISD::BF16_TO_FP, ISD::FP_TO_BF16},
+ {MVT::i16, MVT::f16, MVT::f32}, Expand);
// 64-bit conversions are expanded with 32-bit conversions.
setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::SINT_TO_FP,
@@ -285,6 +290,13 @@ SDValue AGXTargetLowering::LowerFormalArguments(
MFI->setImplicitArgVReg(Dim,
MF.addLiveIn(PReg, &AGX::GPR32RegClass));
}
+ // The global data buffer address, if the module has device globals.
+ if (!AGX::computeGlobalDataLayout(*MF.getFunction().getParent()).empty()) {
+ unsigned GlobalHalf = alignTo(Half, 2);
+ MCRegister PReg = Allocate(MVT::i64);
+ if (PReg)
+ MFI->setGlobalData(MF.addLiveIn(PReg, &AGX::GPR64RegClass), GlobalHalf);
+ }
MFI->setUniformHalfCount(Half);
if (Overflow)
@@ -395,10 +407,30 @@ SDValue AGXTargetLowering::lowerGlobalAddress(SDValue Op,
SDLoc DL(Op);
auto *GA = cast<GlobalAddressSDNode>(Op);
const auto *GV = dyn_cast<GlobalVariable>(GA->getGlobal());
+ MachineFunction &MF = DAG.getMachineFunction();
+ auto *MFI = MF.getInfo<AGXMachineFunctionInfo>();
+
+ // Device and constant variables: an offset into the global data buffer.
+ if (GV && AGX::isGlobalDataVariable(*GV)) {
+ std::optional<uint64_t> Offset =
+ AGX::computeGlobalDataLayout(*MF.getFunction().getParent())
+ .getOffset(GV);
+ if (!Offset || !MFI->getGlobalDataVReg()) {
+ diagnose(DAG, DL,
+ "unsupported external global '" + GV->getName() + "'");
+ return DAG.getUNDEF(Op.getValueType());
+ }
+ SDValue Base = DAG.getCopyFromReg(DAG.getEntryNode(), DL,
+ MFI->getGlobalDataVReg(), MVT::i64);
+ SDValue Addr =
+ DAG.getNode(ISD::ADD, DL, MVT::i64, Base,
+ DAG.getConstant(*Offset + GA->getOffset(), DL, MVT::i64));
+ return DAG.getZExtOrTrunc(Addr, DL, Op.getValueType());
+ }
+
if (!GV || GA->getAddressSpace() != AGX::THREADGROUP_ADDRESS) {
diagnose(DAG, DL,
- "unsupported global '" + GA->getGlobal()->getName() +
- "' (only threadgroup variables are supported)");
+ "unsupported global '" + GA->getGlobal()->getName() + "'");
return DAG.getUNDEF(Op.getValueType());
}
if (GV->hasInitializer() && !isa<UndefValue>(GV->getInitializer()))
@@ -406,8 +438,6 @@ SDValue AGXTargetLowering::lowerGlobalAddress(SDValue Op,
// Threadgroup variables are allocated statically per kernel; their
// address is a constant offset into threadgroup memory.
- MachineFunction &MF = DAG.getMachineFunction();
- auto *MFI = MF.getInfo<AGXMachineFunctionInfo>();
unsigned Offset =
MFI->allocateThreadgroupVariable(DAG.getDataLayout(), *GV);
return DAG.getConstant(Offset + GA->getOffset(), DL, Op.getValueType());
diff --git a/llvm/lib/Target/AGX/AGXInstructions.td b/llvm/lib/Target/AGX/AGXInstructions.td
index d03153fdc4a41..98e94d5b08680 100644
--- a/llvm/lib/Target/AGX/AGXInstructions.td
+++ b/llvm/lib/Target/AGX/AGXInstructions.td
@@ -1252,3 +1252,5 @@ let isTrap = 1, isTerminator = 1, isBarrier = 1, hasSideEffects = 1 in
def AGX_TRAP : AGXPseudo<(outs), (ins), [(trap)]> {
let G13Real = TRAP_g13;
}
+
+def : Pat<(debugtrap), (AGX_TRAP)>;
diff --git a/llvm/lib/Target/AGX/AGXMachineFunctionInfo.cpp b/llvm/lib/Target/AGX/AGXMachineFunctionInfo.cpp
index 84eaa2bab76c1..91e1c11d2e0f7 100644
--- a/llvm/lib/Target/AGX/AGXMachineFunctionInfo.cpp
+++ b/llvm/lib/Target/AGX/AGXMachineFunctionInfo.cpp
@@ -9,6 +9,8 @@
#include "AGXMachineFunctionInfo.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/Module.h"
+#include "Utils/AGXBaseInfo.h"
#include "llvm/Support/MathExtras.h"
using namespace llvm;
@@ -25,3 +27,33 @@ AGXMachineFunctionInfo::allocateThreadgroupVariable(const DataLayout &DL,
ThreadgroupSize = Offset + DL.getTypeAllocSize(GV.getValueType());
return Offset;
}
+
+bool AGX::isGlobalDataVariable(const GlobalVariable &GV) {
+ unsigned AS = GV.getAddressSpace();
+ return AS == AGX::GENERIC_ADDRESS || AS == AGX::DEVICE_ADDRESS ||
+ AS == AGX::CONSTANT_ADDRESS;
+}
+
+AGX::GlobalDataLayout AGX::computeGlobalDataLayout(const Module &M) {
+ GlobalDataLayout Layout;
+ const DataLayout &DL = M.getDataLayout();
+ for (const GlobalVariable &GV : M.globals()) {
+ if (!isGlobalDataVariable(GV) || GV.isDeclaration() ||
+ GV.getName().starts_with("llvm."))
+ continue;
+ Align A = DL.getValueOrABITypeAlignment(GV.getAlign(), GV.getValueType());
+ uint64_t Offset = alignTo(Layout.Size, A);
+ Layout.Offsets.push_back({&GV, Offset});
+ Layout.Size = Offset + DL.getTypeAllocSize(GV.getValueType());
+ Layout.MaxAlign = std::max(Layout.MaxAlign, A);
+ }
+ return Layout;
+}
+
+std::optional<uint64_t>
+AGX::GlobalDataLayout::getOffset(const GlobalVariable *GV) const {
+ for (const auto &[V, Offset] : Offsets)
+ if (V == GV)
+ return Offset;
+ return std::nullopt;
+}
diff --git a/llvm/lib/Target/AGX/AGXMachineFunctionInfo.h b/llvm/lib/Target/AGX/AGXMachineFunctionInfo.h
index a6c3d998d07f3..15003c5179666 100644
--- a/llvm/lib/Target/AGX/AGXMachineFunctionInfo.h
+++ b/llvm/lib/Target/AGX/AGXMachineFunctionInfo.h
@@ -10,11 +10,38 @@
#define LLVM_LIB_TARGET_AGX_AGXMACHINEFUNCTIONINFO_H
#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/Alignment.h"
+#include <optional>
#include "llvm/CodeGen/MachineFunction.h"
namespace llvm {
class GlobalVariable;
+class Module;
+
+namespace AGX {
+
+/// Device and constant global variables live in one buffer, the global data
+/// buffer, whose address kernels receive as an implicit argument. The
+/// variables are laid out in module order, each aligned to its alignment;
+/// the assembly printer emits the initial contents in the .agx.globals
+/// section with the same layout.
+struct GlobalDataLayout {
+ SmallVector<std::pair<const GlobalVariable *, uint64_t>, 8> Offsets;
+ uint64_t Size = 0;
+ Align MaxAlign;
+
+ bool empty() const { return Offsets.empty(); }
+ std::optional<uint64_t> getOffset(const GlobalVariable *GV) const;
+};
+
+/// Whether \p GV is stored in the global data buffer.
+bool isGlobalDataVariable(const GlobalVariable &GV);
+
+GlobalDataLayout computeGlobalDataLayout(const Module &M);
+
+} // namespace AGX
/// Per kernel state: the uniform register layout of the arguments and the
/// threadgroup memory allocation.
@@ -26,6 +53,10 @@ class AGXMachineFunctionInfo : public MachineFunctionInfo {
unsigned UniformHalfCount = 0;
/// Virtual registers holding threadgroups_per_grid.xyz.
Register ImplicitArgVRegs[3];
+ /// Virtual register holding the global data buffer address, and its first
+ /// uniform half register.
+ Register GlobalDataVReg;
+ unsigned GlobalDataHalfIndex = 0;
/// Threadgroup memory: offset of each __shared__ style variable.
DenseMap<const GlobalVariable *, unsigned> ThreadgroupOffsets;
@@ -51,6 +82,12 @@ class AGXMachineFunctionInfo : public MachineFunctionInfo {
Register getImplicitArgVReg(unsigned Dim) const {
return ImplicitArgVRegs[Dim];
}
+ void setGlobalData(Register VReg, unsigned HalfIndex) {
+ GlobalDataVReg = VReg;
+ GlobalDataHalfIndex = HalfIndex;
+ }
+ Register getGlobalDataVReg() const { return GlobalDataVReg; }
+ unsigned getGlobalDataHalfIndex() const { return GlobalDataHalfIndex; }
/// Offset of \p GV in threadgroup memory, allocating it on first use.
unsigned allocateThreadgroupVariable(const DataLayout &DL,
diff --git a/llvm/test/CodeGen/AGX/always-inline.ll b/llvm/test/CodeGen/AGX/always-inline.ll
new file mode 100644
index 0000000000000..57f075072dc9a
--- /dev/null
+++ b/llvm/test/CodeGen/AGX/always-inline.ll
@@ -0,0 +1,25 @@
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -verify-machineinstrs < %s | FileCheck %s
+
+; Functions other than kernels are inlined and removed, used or not.
+
+; CHECK-NOT: helper
+; CHECK-NOT: unused
+; CHECK-LABEL: kernel:
+; CHECK: iadd {{r[0-9]+}}, u2, 5
+; CHECK-NOT: helper
+; CHECK-NOT: unused
+
+define i32 @helper(i32 %x) {
+ %r = add i32 %x, 5
+ ret i32 %r
+}
+
+define hidden float @unused(float %x) {
+ ret float %x
+}
+
+define agx_kernel void @kernel(ptr addrspace(1) %out, i32 %x) {
+ %r = call i32 @helper(i32 %x)
+ store i32 %r, ptr addrspace(1) %out
+ ret void
+}
diff --git a/llvm/test/CodeGen/AGX/constrained-fp.ll b/llvm/test/CodeGen/AGX/constrained-fp.ll
new file mode 100644
index 0000000000000..c371d6f42f4a8
--- /dev/null
+++ b/llvm/test/CodeGen/AGX/constrained-fp.ll
@@ -0,0 +1,20 @@
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -verify-machineinstrs < %s | FileCheck %s
+
+; There is no floating point environment: constrained operations are
+; generated like ordinary ones.
+
+; CHECK-LABEL: strict:
+; CHECK: rcp
+; CHECK: fmul32
+; CHECK: fadd32
+; CHECK: fcmpsel lt
+; CHECK: device_store
+define agx_kernel void @strict(ptr addrspace(1) %out, float %a, float %b) strictfp {
+ %d = call float @llvm.experimental.constrained.fdiv.f32(float %a, float %b, metadata !"round.dynamic", metadata !"fpexcept.ignore") strictfp
+ %s = call float @llvm.experimental.constrained.fadd.f32(float %d, float %b, metadata !"round.dynamic", metadata !"fpexcept.ignore") strictfp
+ %r = call float @llvm.experimental.constrained.sqrt.f32(float %s, metadata !"round.dynamic", metadata !"fpexcept.ignore") strictfp
+ %c = call i1 @llvm.experimental.constrained.fcmp.f32(float %r, float %a, metadata !"olt", metadata !"fpexcept.ignore") strictfp
+ %v = select i1 %c, float %r, float %a
+ store float %v, ptr addrspace(1) %out
+ ret void
+}
diff --git a/llvm/test/CodeGen/AGX/control-flow.ll b/llvm/test/CodeGen/AGX/control-flow.ll
index 98a34086a4f68..191891f7d03dc 100644
--- a/llvm/test/CodeGen/AGX/control-flow.ll
+++ b/llvm/test/CodeGen/AGX/control-flow.ll
@@ -308,3 +308,46 @@ join:
}
declare i32 @llvm.agx.thread.position.in.grid.x()
+
+define agx_kernel void @trap(ptr addrspace(1) %out, i32 %n) {
+; CHECK-LABEL: trap:
+; CHECK: // uniform registers: 12 x 16-bit (threadgroups_per_grid at u3)
+; CHECK-NEXT: // general purpose registers: 3 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov_imm r0l, 0
+; CHECK-NEXT: get_sr r1, sr80 (thread_position_in_grid.x)
+; CHECK-NEXT: icmpsel ult, r2, r1, u2, 0, 1
+; CHECK-NEXT: if_icmp r0l, nueq, r2, 0, 1
+; CHECK-NEXT: jmp_exec_none .LBB5_2
+; CHECK-NEXT: // %bb.1: // %bad
+; CHECK-NEXT: trap
+; CHECK-NEXT: .LBB5_2: // %Flow
+; CHECK-NEXT: else_fcmp r0l, eq, 0.0, 0.0, 1
+; CHECK-NEXT: jmp_exec_none .LBB5_4
+; CHECK-NEXT: // %bb.3: // %ok
+; CHECK-NEXT: device_store 0, i32, x, r1, u0_u1, 0, signed
+; CHECK-NEXT: .LBB5_4: // %unified.return
+; CHECK-NEXT: pop_exec r0l, 1
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %id = call i32 @llvm.agx.thread.position.in.grid.x()
+ %c = icmp ult i32 %id, %n
+ br i1 %c, label %ok, label %bad
+bad:
+ call void @llvm.trap()
+ unreachable
+ok:
+ store i32 %id, ptr addrspace(1) %out
+ ret void
+}
+
+declare void @llvm.trap()
diff --git a/llvm/test/CodeGen/AGX/entry-points.ll b/llvm/test/CodeGen/AGX/entry-points.ll
new file mode 100644
index 0000000000000..17a8bb3e81707
--- /dev/null
+++ b/llvm/test/CodeGen/AGX/entry-points.ll
@@ -0,0 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -verify-machineinstrs < %s | FileCheck %s
+
+; Without kernels in the module, every external function is an entry point
+; with the kernel ABI.
+
+define void @not_a_kernel(ptr addrspace(1) %out, i32 %v) {
+; CHECK-LABEL: not_a_kernel:
+; CHECK: // uniform registers: 12 x 16-bit (threadgroups_per_grid at u3)
+; CHECK-NEXT: // general purpose registers: 2 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: bitop 0101, r1, u2, 0
+; CHECK-NEXT: device_store 0, i32, x, r1, u0_u1, 0, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
diff --git a/llvm/test/CodeGen/AGX/global-data.ll b/llvm/test/CodeGen/AGX/global-data.ll
new file mode 100644
index 0000000000000..bf0d0e430bd06
--- /dev/null
+++ b/llvm/test/CodeGen/AGX/global-data.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -verify-machineinstrs < %s | FileCheck %s
+
+; Device and constant variables live in the global data buffer, whose
+; address follows threadgroups_per_grid in the uniform registers. The
+; initial contents are in .agx.globals, in module order.
+
+ at counter = addrspace(1) global i32 7, align 4
+ at table = addrspace(4) constant [4 x float] [float 1.0, float 2.0, float 3.0, float 4.0], align 16
+ at tg = internal addrspace(3) global [8 x i32] undef
+
+define agx_kernel void @kernel(ptr addrspace(1) %out, i32 %i) {
+; CHECK-LABEL: kernel:
+; CHECK: // uniform registers: 16 x 16-bit (threadgroups_per_grid at u3)
+; CHECK-NEXT: // general purpose registers: 4 x 32-bit
+; CHECK-NEXT: // global data buffer at u6_u7
+; CHECK-NEXT: // threadgroup memory: 32 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: bitop 0101, r2, u6, 0
+; CHECK-NEXT: bitop 0101, r3, u7, 0
+; CHECK-NEXT: iadd r2_r3, r2_r3, u2.sx, lsl 2
+; CHECK-NEXT: device_load 0, i32, x, r1, u6_u7, 0, signed
+; CHECK-NEXT: device_load 0, i32, x, r2, r2_r3, 4, signed
+; CHECK-NEXT: wait 0
+; CHECK-NEXT: convert s32_to_f, r3, r1, rte
+; CHECK-NEXT: fadd32 r2, r2, r3
+; CHECK-NEXT: bfi r3, 0, u2, 2
+; CHECK-NEXT: device_store 0, i32, x, r2, u0_u1, 0, signed
+; CHECK-NEXT: threadgroup_store i32, x, r1, r3l, 0
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %p = getelementptr [4 x float], ptr addrspace(4) @table, i32 0, i32 %i
+ %v = load float, ptr addrspace(4) %p
+ %c = load i32, ptr addrspace(1) @counter
+ %cf = sitofp i32 %c to float
+ %s = fadd float %v, %cf
+ store float %s, ptr addrspace(1) %out
+ %t = getelementptr [8 x i32], ptr addrspace(3) @tg, i32 0, i32 %i
+ store i32 %c, ptr addrspace(3) %t
+ ret void
+}
+
+; CHECK-LABEL: .section .agx.globals,"aw", at progbits
+; CHECK-NEXT: .p2align 4, 0x0
+; CHECK-NEXT: counter:
+; CHECK-NEXT: .long 7
+; CHECK-NEXT: .zero 12
+; CHECK-NEXT: table:
+; CHECK-NEXT: .long 0x3f800000
+; CHECK-NOT: tg
diff --git a/llvm/test/CodeGen/AGX/kernel-args.ll b/llvm/test/CodeGen/AGX/kernel-args.ll
index 4d8eb676b8aa3..3286628761752 100644
--- a/llvm/test/CodeGen/AGX/kernel-args.ll
+++ b/llvm/test/CodeGen/AGX/kernel-args.ll
@@ -42,27 +42,4 @@ define agx_kernel void @args(ptr addrspace(1) %out, i16 %a, i32 %b, i64 %c, half
ret void
}
-; Without kernels in the module, every external function is an entry point.
-define void @not_a_kernel(ptr addrspace(1) %out, i32 %v) {
-; CHECK-LABEL: not_a_kernel:
-; CHECK: // uniform registers: 12 x 16-bit (threadgroups_per_grid at u3)
-; CHECK-NEXT: // general purpose registers: 2 x 32-bit
-; CHECK-NEXT: // threadgroup memory: 0 bytes
-; CHECK-NEXT: // stack size: 0 bytes
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: bitop 0101, r1, u2, 0
-; CHECK-NEXT: device_store 0, i32, x, r1, u0_u1, 0, signed
-; CHECK-NEXT: stop
-; CHECK-NEXT: trap
-; CHECK-NEXT: trap
-; CHECK-NEXT: trap
-; CHECK-NEXT: trap
-; CHECK-NEXT: trap
-; CHECK-NEXT: trap
-; CHECK-NEXT: trap
-; CHECK-NEXT: trap
- store i32 %v, ptr addrspace(1) %out
- ret void
-}
-
declare i32 @llvm.agx.threadgroups.per.grid.y()
>From a1bde517cf3c89be437fd8ca30c92e7a5e1213c7 Mon Sep 17 00:00:00 2001
From: Claude <noreply at anthropic.com>
Date: Sun, 18 Oct 2026 20:35:36 +0000
Subject: [PATCH 06/22] [AGX] Document the global data buffer and constrained
floating point
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
Claude-Session: https://claude.ai/code/session_01E56KEecg6edNaS23myRWER
---
llvm/docs/AGXUsage.rst | 16 +++++++++++++++-
1 file changed, 15 insertions(+), 1 deletion(-)
diff --git a/llvm/docs/AGXUsage.rst b/llvm/docs/AGXUsage.rst
index 90ba9227ede93..c470981f929ad 100644
--- a/llvm/docs/AGXUsage.rst
+++ b/llvm/docs/AGXUsage.rst
@@ -154,7 +154,7 @@ Before each kernel, the assembly printer emits comments with the resources
the driver must provide::
// uniform registers: 18 x 16-bit (threadgroups_per_grid at u6)
- // general purpose registers: 7 x 32-bit
+ // general purpose registers: 4 x 32-bit
// threadgroup memory: 0 bytes
// stack size: 0 bytes
@@ -165,6 +165,16 @@ Threadgroup variables are globals in address space 3 without an initializer.
They are allocated per kernel in threadgroup memory, at offsets in order of
first use.
+Device and constant variables (globals in address spaces 0, 1 and 4) live in
+the *global data buffer*. When a module defines any, each kernel receives
+the 64-bit address of that buffer in the uniform registers following
+``threadgroups_per_grid`` (reported in the resource comments as ``global
+data buffer at uN_uN+1``). The variables are laid out in module order, each
+aligned to its alignment, and the initial contents of the buffer, with that
+layout, are the ``.agx.globals`` section of the object. Initializers that
+refer to the addresses of other globals would need relocations and are not
+supported.
+
Execution model
===============
@@ -227,6 +237,10 @@ Supported:
* SIMD-group and quad shuffles, ballots, reductions and prefixes, derivatives,
barriers and special registers through the ``llvm.agx.*`` intrinsics.
+There is no floating point environment: constrained floating point
+intrinsics are compiled as the ordinary operations, with round to nearest
+even.
+
Not supported: ``double``, vector memory operations (vectors are
scalarized), textures and graphics shaders, calls and recursion.
>From a70c3e1cac78d67a38f058108e7833e9bdceacd1 Mon Sep 17 00:00:00 2001
From: Claude <noreply at anthropic.com>
Date: Sun, 18 Oct 2026 21:33:19 +0000
Subject: [PATCH 07/22] [mlir] Add agx and agxir dialects for the AGX backend
Mirror the amdgpu/rocdl split for Apple GPUs:
- agxir: 1:1 wrappers of the llvm.agx.* intrinsics (special registers,
barriers, waits, SIMD-group and quad shuffles/reductions/ballots,
rcp/rsqrt, derivatives), the agx_kernelcc calling convention, a
translation to LLVM IR and the #agxir.target attribute that serializes
GPU modules to AGX assembly, ELF objects or bitcode via ModuleToObject.
- agx: higher level ops in the style of amdgpu (agx.shuffle for any
type and scope, named agx.reduce kinds, agx.threadgroup_barrier,
agx.ballot) with a convert-agx-to-agxir pass.
- convert-gpu-to-agxir lowers gpu dialect index, barrier, shuffle,
ballot, subgroup reduce and kernel ops; gpu-agxir-attach-target
attaches targets to GPU modules.
The AGX target support is built when the AGX backend is enabled
(MLIR_AGX_TARGET_ENABLED); serialization tests require it via the
host-supports-agx lit feature.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
Claude-Session: https://claude.ai/code/session_01E56KEecg6edNaS23myRWER
---
.../mlir/Conversion/AGXToAGXIR/AGXToAGXIR.h | 28 ++
.../Conversion/GPUToAGXIR/GPUToAGXIRPass.h | 39 ++
mlir/include/mlir/Conversion/Passes.h | 2 +
mlir/include/mlir/Conversion/Passes.td | 44 ++
mlir/include/mlir/Dialect/AGX/CMakeLists.txt | 1 +
mlir/include/mlir/Dialect/AGX/IR/AGX.td | 189 +++++++++
mlir/include/mlir/Dialect/AGX/IR/AGXDialect.h | 33 ++
.../mlir/Dialect/AGX/IR/CMakeLists.txt | 9 +
mlir/include/mlir/Dialect/CMakeLists.txt | 1 +
.../mlir/Dialect/GPU/Transforms/Passes.td | 34 ++
.../mlir/Dialect/LLVMIR/AGXIRDialect.h | 34 ++
mlir/include/mlir/Dialect/LLVMIR/AGXIROps.td | 330 +++++++++++++++
.../mlir/Dialect/LLVMIR/CMakeLists.txt | 13 +
mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td | 5 +-
mlir/include/mlir/Target/LLVM/AGXIR/Target.h | 30 ++
.../Dialect/AGXIR/AGXIRToLLVMIRTranslation.h | 31 ++
mlir/include/mlir/Target/LLVMIR/Dialect/All.h | 3 +
mlir/lib/Conversion/AGXToAGXIR/AGXToAGXIR.cpp | 256 ++++++++++++
mlir/lib/Conversion/AGXToAGXIR/CMakeLists.txt | 17 +
mlir/lib/Conversion/CMakeLists.txt | 2 +
mlir/lib/Conversion/GPUToAGXIR/CMakeLists.txt | 20 +
.../GPUToAGXIR/LowerGpuOpsToAGXIROps.cpp | 383 ++++++++++++++++++
mlir/lib/Dialect/AGX/CMakeLists.txt | 1 +
mlir/lib/Dialect/AGX/IR/AGXDialect.cpp | 90 ++++
mlir/lib/Dialect/AGX/IR/CMakeLists.txt | 14 +
mlir/lib/Dialect/CMakeLists.txt | 1 +
mlir/lib/Dialect/GPU/CMakeLists.txt | 3 +
.../GPU/Transforms/AGXIRAttachTarget.cpp | 68 ++++
mlir/lib/Dialect/LLVMIR/CMakeLists.txt | 21 +
mlir/lib/Dialect/LLVMIR/IR/AGXIRDialect.cpp | 121 ++++++
mlir/lib/RegisterAllDialects.cpp | 6 +
mlir/lib/Target/LLVM/AGXIR/Target.cpp | 198 +++++++++
mlir/lib/Target/LLVM/CMakeLists.txt | 34 ++
mlir/lib/Target/LLVMIR/CMakeLists.txt | 1 +
.../AGXIR/AGXIRToLLVMIRTranslation.cpp | 74 ++++
.../LLVMIR/Dialect/AGXIR/CMakeLists.txt | 16 +
mlir/lib/Target/LLVMIR/Dialect/CMakeLists.txt | 1 +
.../Conversion/AGXToAGXIR/agx-to-agxir.mlir | 56 +++
.../Conversion/GPUToAGXIR/gpu-to-agxir.mlir | 86 ++++
mlir/test/Dialect/AGX/invalid.mlir | 31 ++
mlir/test/Dialect/AGX/ops.mlir | 37 ++
.../test/Dialect/GPU/agxir-attach-target.mlir | 10 +
.../Dialect/GPU/module-to-binary-agxir.mlir | 19 +
mlir/test/Dialect/LLVMIR/agxir.mlir | 72 ++++
mlir/test/Dialect/LLVMIR/func.mlir | 2 +-
mlir/test/Target/LLVMIR/agxir.mlir | 37 ++
mlir/test/lit.cfg.py | 3 +
mlir/test/lit.site.cfg.py.in | 2 +
48 files changed, 2506 insertions(+), 2 deletions(-)
create mode 100644 mlir/include/mlir/Conversion/AGXToAGXIR/AGXToAGXIR.h
create mode 100644 mlir/include/mlir/Conversion/GPUToAGXIR/GPUToAGXIRPass.h
create mode 100644 mlir/include/mlir/Dialect/AGX/CMakeLists.txt
create mode 100644 mlir/include/mlir/Dialect/AGX/IR/AGX.td
create mode 100644 mlir/include/mlir/Dialect/AGX/IR/AGXDialect.h
create mode 100644 mlir/include/mlir/Dialect/AGX/IR/CMakeLists.txt
create mode 100644 mlir/include/mlir/Dialect/LLVMIR/AGXIRDialect.h
create mode 100644 mlir/include/mlir/Dialect/LLVMIR/AGXIROps.td
create mode 100644 mlir/include/mlir/Target/LLVM/AGXIR/Target.h
create mode 100644 mlir/include/mlir/Target/LLVMIR/Dialect/AGXIR/AGXIRToLLVMIRTranslation.h
create mode 100644 mlir/lib/Conversion/AGXToAGXIR/AGXToAGXIR.cpp
create mode 100644 mlir/lib/Conversion/AGXToAGXIR/CMakeLists.txt
create mode 100644 mlir/lib/Conversion/GPUToAGXIR/CMakeLists.txt
create mode 100644 mlir/lib/Conversion/GPUToAGXIR/LowerGpuOpsToAGXIROps.cpp
create mode 100644 mlir/lib/Dialect/AGX/CMakeLists.txt
create mode 100644 mlir/lib/Dialect/AGX/IR/AGXDialect.cpp
create mode 100644 mlir/lib/Dialect/AGX/IR/CMakeLists.txt
create mode 100644 mlir/lib/Dialect/GPU/Transforms/AGXIRAttachTarget.cpp
create mode 100644 mlir/lib/Dialect/LLVMIR/IR/AGXIRDialect.cpp
create mode 100644 mlir/lib/Target/LLVM/AGXIR/Target.cpp
create mode 100644 mlir/lib/Target/LLVMIR/Dialect/AGXIR/AGXIRToLLVMIRTranslation.cpp
create mode 100644 mlir/lib/Target/LLVMIR/Dialect/AGXIR/CMakeLists.txt
create mode 100644 mlir/test/Conversion/AGXToAGXIR/agx-to-agxir.mlir
create mode 100644 mlir/test/Conversion/GPUToAGXIR/gpu-to-agxir.mlir
create mode 100644 mlir/test/Dialect/AGX/invalid.mlir
create mode 100644 mlir/test/Dialect/AGX/ops.mlir
create mode 100644 mlir/test/Dialect/GPU/agxir-attach-target.mlir
create mode 100644 mlir/test/Dialect/GPU/module-to-binary-agxir.mlir
create mode 100644 mlir/test/Dialect/LLVMIR/agxir.mlir
create mode 100644 mlir/test/Target/LLVMIR/agxir.mlir
diff --git a/mlir/include/mlir/Conversion/AGXToAGXIR/AGXToAGXIR.h b/mlir/include/mlir/Conversion/AGXToAGXIR/AGXToAGXIR.h
new file mode 100644
index 0000000000000..237c944194979
--- /dev/null
+++ b/mlir/include/mlir/Conversion/AGXToAGXIR/AGXToAGXIR.h
@@ -0,0 +1,28 @@
+//===- AGXToAGXIR.h - Convert AGX to AGXIR dialect --------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef MLIR_CONVERSION_AGXTOAGXIR_AGXTOAGXIR_H_
+#define MLIR_CONVERSION_AGXTOAGXIR_AGXTOAGXIR_H_
+
+#include <memory>
+
+namespace mlir {
+
+class LLVMTypeConverter;
+class RewritePatternSet;
+class Pass;
+
+#define GEN_PASS_DECL_CONVERTAGXTOAGXIRPASS
+#include "mlir/Conversion/Passes.h.inc"
+
+/// Collect the patterns to convert from the AGX dialect to AGXIR.
+void populateAGXToAGXIRConversionPatterns(const LLVMTypeConverter &converter,
+ RewritePatternSet &patterns);
+
+} // namespace mlir
+
+#endif // MLIR_CONVERSION_AGXTOAGXIR_AGXTOAGXIR_H_
diff --git a/mlir/include/mlir/Conversion/GPUToAGXIR/GPUToAGXIRPass.h b/mlir/include/mlir/Conversion/GPUToAGXIR/GPUToAGXIRPass.h
new file mode 100644
index 0000000000000..f27303e1852ad
--- /dev/null
+++ b/mlir/include/mlir/Conversion/GPUToAGXIR/GPUToAGXIRPass.h
@@ -0,0 +1,39 @@
+//===- GPUToAGXIRPass.h - Convert GPU kernel to AGXIR dialect ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef MLIR_CONVERSION_GPUTOAGXIR_GPUTOAGXIRPASS_H_
+#define MLIR_CONVERSION_GPUTOAGXIR_GPUTOAGXIRPASS_H_
+
+#include "mlir/Conversion/LLVMCommon/LoweringOptions.h"
+#include <memory>
+
+namespace mlir {
+class LLVMTypeConverter;
+class ConversionTarget;
+class RewritePatternSet;
+class Pass;
+
+namespace gpu {
+class GPUModuleOp;
+} // namespace gpu
+
+#define GEN_PASS_DECL_CONVERTGPUOPSTOAGXIROPS
+#include "mlir/Conversion/Passes.h.inc"
+
+/// The data layout of the AGX backend.
+extern const char *const kAGXDataLayout;
+
+/// Collect a set of patterns to convert from the GPU dialect to AGXIR.
+void populateGpuToAGXIRConversionPatterns(const LLVMTypeConverter &converter,
+ RewritePatternSet &patterns);
+
+/// Configure target to convert from the GPU dialect to AGXIR.
+void configureGpuToAGXIRConversionLegality(ConversionTarget &target);
+
+} // namespace mlir
+
+#endif // MLIR_CONVERSION_GPUTOAGXIR_GPUTOAGXIRPASS_H_
diff --git a/mlir/include/mlir/Conversion/Passes.h b/mlir/include/mlir/Conversion/Passes.h
index 8758f5c96c3db..f5c8ab869e5de 100644
--- a/mlir/include/mlir/Conversion/Passes.h
+++ b/mlir/include/mlir/Conversion/Passes.h
@@ -9,6 +9,7 @@
#ifndef MLIR_CONVERSION_PASSES_H
#define MLIR_CONVERSION_PASSES_H
+#include "mlir/Conversion/AGXToAGXIR/AGXToAGXIR.h"
#include "mlir/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.h"
#include "mlir/Conversion/AffineToStandard/AffineToStandard.h"
#include "mlir/Conversion/ArithAndMathToAPFloat/ArithToAPFloat.h"
@@ -40,6 +41,7 @@
#include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
#include "mlir/Conversion/GPUToLLVMSPV/GPUToLLVMSPVPass.h"
#include "mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h"
+#include "mlir/Conversion/GPUToAGXIR/GPUToAGXIRPass.h"
#include "mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h"
#include "mlir/Conversion/GPUToSPIRV/GPUToSPIRVPass.h"
#include "mlir/Conversion/IndexToLLVM/IndexToLLVM.h"
diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index 8562cbc11bd45..8759e4ebba450 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -121,6 +121,26 @@ def LowerAffinePass : Pass<"lower-affine"> {
// AMDGPUToROCDL
//===----------------------------------------------------------------------===//
+//===----------------------------------------------------------------------===//
+// AGXToAGXIR
+//===----------------------------------------------------------------------===//
+
+def ConvertAGXToAGXIRPass : Pass<"convert-agx-to-agxir"> {
+ let summary = "Convert AGX dialect to AGXIR dialect";
+ let description = [{
+ This pass converts the AGX dialect operations (Apple GPUs) to AGXIR
+ intrinsic wrappers.
+ }];
+ let dependentDialects = [
+ "LLVM::LLVMDialect",
+ "AGXIR::AGXIRDialect",
+ ];
+}
+
+//===----------------------------------------------------------------------===//
+// AMDGPUToROCDL
+//===----------------------------------------------------------------------===//
+
def ConvertAMDGPUToROCDLPass : Pass<"convert-amdgpu-to-rocdl"> {
let summary = "Convert AMDGPU dialect to ROCDL dialect";
let description = [{
@@ -691,6 +711,30 @@ def ConvertGpuOpsToNVVMOps : Pass<"convert-gpu-to-nvvm", "gpu::GPUModuleOp"> {
// GPUToROCDL
//===----------------------------------------------------------------------===//
+def ConvertGpuOpsToAGXIROps : Pass<"convert-gpu-to-agxir", "gpu::GPUModuleOp"> {
+ let summary = "Generate AGXIR operations for gpu operations";
+ let description = [{
+ Lowers GPU modules for Apple GPUs: GPU and AGX dialect operations become
+ AGXIR intrinsic wrappers, kernels become `agx_kernel` functions, and the
+ other dialects are converted to the LLVM dialect.
+ }];
+ let dependentDialects = [
+ "AGXIR::AGXIRDialect",
+ "agx::AGXDialect",
+ "cf::ControlFlowDialect",
+ "memref::MemRefDialect",
+ ];
+ let options = [
+ Option<"indexBitwidth", "index-bitwidth", "unsigned",
+ /*default=kDeriveIndexBitwidthFromDataLayout*/ "0",
+ "Bitwidth of the index type, 0 to use size of machine word">,
+ Option<"useBarePtrCallConv", "use-bare-ptr-memref-call-conv", "bool",
+ /*default=*/"false",
+ "Replace memref arguments in GPU functions with bare pointers. "
+ "All memrefs must have static shape">,
+ ];
+}
+
def ConvertGpuOpsToROCDLOps : Pass<"convert-gpu-to-rocdl", "gpu::GPUModuleOp"> {
let summary = "Generate ROCDL operations for gpu operations";
let dependentDialects = [
diff --git a/mlir/include/mlir/Dialect/AGX/CMakeLists.txt b/mlir/include/mlir/Dialect/AGX/CMakeLists.txt
new file mode 100644
index 0000000000000..f33061b2d87cf
--- /dev/null
+++ b/mlir/include/mlir/Dialect/AGX/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(IR)
diff --git a/mlir/include/mlir/Dialect/AGX/IR/AGX.td b/mlir/include/mlir/Dialect/AGX/IR/AGX.td
new file mode 100644
index 0000000000000..af93206b4e3cc
--- /dev/null
+++ b/mlir/include/mlir/Dialect/AGX/IR/AGX.td
@@ -0,0 +1,189 @@
+//===-- AGX.td - AGX dialect definitions -------------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef AGX
+#define AGX
+
+include "mlir/IR/EnumAttr.td"
+include "mlir/IR/OpBase.td"
+include "mlir/Interfaces/SideEffectInterfaces.td"
+
+//===----------------------------------------------------------------------===//
+// AGX dialect definition
+//===----------------------------------------------------------------------===//
+
+def AGX_Dialect : Dialect {
+ let name = "agx";
+ let cppNamespace = "::mlir::agx";
+ let description = [{
+ The `agx` dialect provides wrappers around the functionality of Apple
+ GPUs (the AGX architecture) in the style of the `amdgpu` dialect. It is
+ used together with generic dialects such as `gpu` and `vector`, and lowers
+ to the `agxir` dialect, whose operations are 1:1 wrappers of the LLVM AGX
+ backend's intrinsics.
+
+ Operations belong here when they make AGX functionality work better with
+ MLIR types or hide magic constants of the intrinsics:
+
+ - `agx.shuffle` exchanges values of any type, splitting those wider than
+ 32 bits, with one operation for the SIMD-group and quad variants of
+ every shuffle mode.
+ - `agx.reduce` names the reduction instead of the intrinsic's operation
+ number, and extends 16-bit values.
+ - `agx.threadgroup_barrier` is an execution barrier that also orders
+ threadgroup memory, like `amdgpu.lds_barrier`.
+ }];
+ let useDefaultAttributePrinterParser = 1;
+}
+
+//===----------------------------------------------------------------------===//
+// Enums and attributes
+//===----------------------------------------------------------------------===//
+
+def AGX_Scope : I32EnumAttr<"Scope", "The threads an operation spans", [
+ I32EnumAttrCase<"simdgroup", 0>,
+ I32EnumAttrCase<"quad", 1>
+ ]> {
+ let genSpecializedAttr = 0;
+ let cppNamespace = "::mlir::agx";
+}
+def AGX_ScopeAttr : EnumAttr<AGX_Dialect, AGX_Scope, "scope"> {
+ let assemblyFormat = "$value";
+}
+
+def AGX_ShuffleMode : I32EnumAttr<"ShuffleMode", "Lane selection of a shuffle",
+ [
+ I32EnumAttrCase<"idx", 0>,
+ I32EnumAttrCase<"xor_", 1, "xor">,
+ I32EnumAttrCase<"up", 2>,
+ I32EnumAttrCase<"down", 3>
+ ]> {
+ let genSpecializedAttr = 0;
+ let cppNamespace = "::mlir::agx";
+}
+def AGX_ShuffleModeAttr : EnumAttr<AGX_Dialect, AGX_ShuffleMode, "shuffle_mode"> {
+ let assemblyFormat = "$value";
+}
+
+def AGX_ReduceKind : I32EnumAttr<"ReduceKind", "The operation of a reduction",
+ [
+ I32EnumAttrCase<"add", 0>,
+ I32EnumAttrCase<"mul", 1>,
+ I32EnumAttrCase<"minsi", 2>,
+ I32EnumAttrCase<"maxsi", 3>,
+ I32EnumAttrCase<"minui", 4>,
+ I32EnumAttrCase<"maxui", 5>,
+ I32EnumAttrCase<"minnumf", 6>,
+ I32EnumAttrCase<"maxnumf", 7>,
+ I32EnumAttrCase<"and_", 8, "and">,
+ I32EnumAttrCase<"or_", 9, "or">,
+ I32EnumAttrCase<"xor_", 10, "xor">
+ ]> {
+ let genSpecializedAttr = 0;
+ let cppNamespace = "::mlir::agx";
+}
+def AGX_ReduceKindAttr : EnumAttr<AGX_Dialect, AGX_ReduceKind, "reduce_kind"> {
+ let assemblyFormat = "$value";
+}
+
+//===----------------------------------------------------------------------===//
+// Operations
+//===----------------------------------------------------------------------===//
+
+class AGX_Op<string mnemonic, list<Trait> traits = []>
+ : Op<AGX_Dialect, mnemonic, traits>;
+
+def AGX_ThreadgroupBarrierOp : AGX_Op<"threadgroup_barrier"> {
+ let summary = "Barrier that also orders threadgroup memory";
+ let description = [{
+ `agx.threadgroup_barrier` waits for all threads of the threadgroup and
+ makes the threadgroup memory writes before it visible to the reads after
+ it, like `__syncthreads()`.
+
+ ```mlir
+ agx.threadgroup_barrier
+ ```
+ }];
+ let assemblyFormat = "attr-dict";
+}
+
+def AGX_ShuffleOp : AGX_Op<"shuffle", [AllTypesMatch<["value", "result"]>]> {
+ let summary = "Read a value from another thread";
+ let description = [{
+ Reads `value` from another thread of the SIMD-group (the default)
+ or of the quad (`scope quad`). The lane is
+ `index` itself (`idx`), the current lane xor `index` (`xor`), or the
+ current lane minus or plus `index` (`up`, `down`).
+
+ Values of 16 or 32 bits are shuffled directly; wider values and vectors
+ are split into 32-bit parts.
+
+ ```mlir
+ %0 = agx.shuffle xor %v, %c1 : f32
+ %1 = agx.shuffle down %w, %c4 scope quad : vector<2xf32>
+ ```
+ }];
+ let arguments = (ins AGX_ShuffleModeAttr:$mode,
+ AnyTypeOf<[AnyInteger, AnyFloat, AnyVectorOfNonZeroRank]>:$value,
+ I32:$index,
+ DefaultValuedAttr<AGX_ScopeAttr,
+ "::mlir::agx::Scope::simdgroup">:$scope);
+ let results = (outs AnyType:$result);
+ let assemblyFormat = [{
+ $mode $value `,` $index (`scope` $scope^)? attr-dict `:` type($result)
+ }];
+ let hasVerifier = 1;
+}
+
+def AGX_ReduceOp : AGX_Op<"reduce", [AllTypesMatch<["value", "result"]>]> {
+ let summary = "Reduction across the active threads";
+ let description = [{
+ Combines `value` across the active threads of the SIMD-group or of the
+ quad with the operation `kind`. With `prefix`, each thread receives the
+ combination of the threads before it instead.
+
+ Integer reductions take 16 or 32-bit integers (16-bit values are
+ extended according to the signedness of the operation); floating point
+ reductions take `f16` or `f32`.
+
+ ```mlir
+ %sum = agx.reduce add %v : f32
+ %max = agx.reduce maxui %i scope quad : i32
+ %pre = agx.reduce add %v prefix : i32
+ ```
+ }];
+ let arguments = (ins AGX_ReduceKindAttr:$kind,
+ AnyTypeOf<[I16, I32, F16, F32]>:$value,
+ DefaultValuedAttr<AGX_ScopeAttr,
+ "::mlir::agx::Scope::simdgroup">:$scope,
+ UnitAttr:$prefix);
+ let results = (outs AnyTypeOf<[I16, I32, F16, F32]>:$result);
+ let assemblyFormat = [{
+ $kind $value (`prefix` $prefix^)? (`scope` $scope^)? attr-dict `:` type($result)
+ }];
+ let hasVerifier = 1;
+}
+
+def AGX_BallotOp : AGX_Op<"ballot"> {
+ let summary = "Mask of the threads where a predicate is true";
+ let description = [{
+ Returns a mask with bit N set when thread N of the SIMD-group (or of the
+ quad, in bits 0 to 3) is active and `predicate` is true there.
+
+ ```mlir
+ %mask = agx.ballot %pred : i32
+ ```
+ }];
+ let arguments = (ins I1:$predicate,
+ DefaultValuedAttr<AGX_ScopeAttr,
+ "::mlir::agx::Scope::simdgroup">:$scope);
+ let results = (outs I32:$result);
+ let assemblyFormat = "$predicate (`scope` $scope^)? attr-dict `:` type($result)";
+}
+
+#endif // AGX
diff --git a/mlir/include/mlir/Dialect/AGX/IR/AGXDialect.h b/mlir/include/mlir/Dialect/AGX/IR/AGXDialect.h
new file mode 100644
index 0000000000000..304e85ed58670
--- /dev/null
+++ b/mlir/include/mlir/Dialect/AGX/IR/AGXDialect.h
@@ -0,0 +1,33 @@
+//===- AGXDialect.h - MLIR dialect for Apple GPUs ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares a dialect for the functionality of Apple GPUs (AGX),
+// in the style of the AMDGPU dialect.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_AGX_IR_AGXDIALECT_H_
+#define MLIR_DIALECT_AGX_IR_AGXDIALECT_H_
+
+#include "mlir/Bytecode/BytecodeOpInterface.h"
+#include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/Dialect.h"
+#include "mlir/IR/OpDefinition.h"
+#include "mlir/Interfaces/SideEffectInterfaces.h"
+
+#include "mlir/Dialect/AGX/IR/AGXDialect.h.inc"
+
+#include "mlir/Dialect/AGX/IR/AGXEnums.h.inc"
+
+#define GET_ATTRDEF_CLASSES
+#include "mlir/Dialect/AGX/IR/AGXAttrs.h.inc"
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/AGX/IR/AGX.h.inc"
+
+#endif // MLIR_DIALECT_AGX_IR_AGXDIALECT_H_
diff --git a/mlir/include/mlir/Dialect/AGX/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/AGX/IR/CMakeLists.txt
new file mode 100644
index 0000000000000..f96c00d717fad
--- /dev/null
+++ b/mlir/include/mlir/Dialect/AGX/IR/CMakeLists.txt
@@ -0,0 +1,9 @@
+add_mlir_dialect(AGX agx)
+add_mlir_doc(AGX AGX Dialects/ -gen-dialect-doc -dialect agx)
+
+set(LLVM_TARGET_DEFINITIONS AGX.td)
+mlir_tablegen(AGXEnums.h.inc -gen-enum-decls)
+mlir_tablegen(AGXEnums.cpp.inc -gen-enum-defs)
+mlir_tablegen(AGXAttrs.h.inc -gen-attrdef-decls -attrdefs-dialect=agx)
+mlir_tablegen(AGXAttrs.cpp.inc -gen-attrdef-defs -attrdefs-dialect=agx)
+add_mlir_dialect_tablegen_target(MLIRAGXEnumsGen)
diff --git a/mlir/include/mlir/Dialect/CMakeLists.txt b/mlir/include/mlir/Dialect/CMakeLists.txt
index d7ea0cc3df267..32ca67306ff65 100644
--- a/mlir/include/mlir/Dialect/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/CMakeLists.txt
@@ -1,4 +1,5 @@
add_subdirectory(Affine)
+add_subdirectory(AGX)
add_subdirectory(AMDGPU)
add_subdirectory(Arith)
add_subdirectory(ArmNeon)
diff --git a/mlir/include/mlir/Dialect/GPU/Transforms/Passes.td b/mlir/include/mlir/Dialect/GPU/Transforms/Passes.td
index 090bfdba3ce8f..819fa7b1544c0 100644
--- a/mlir/include/mlir/Dialect/GPU/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/GPU/Transforms/Passes.td
@@ -169,6 +169,40 @@ def GpuNVVMAttachTarget: Pass<"nvvm-attach-target", ""> {
];
}
+def GpuAGXIRAttachTarget: Pass<"agxir-attach-target", ""> {
+ let summary = "Attaches an AGXIR target attribute to a GPU Module.";
+ let description = [{
+ This pass searches for all GPU Modules in the immediate regions and
+ attaches an AGXIR target (Apple GPUs) if the module matches the name
+ specified by the `module` argument.
+
+ Example:
+ ```
+ // mlir-opt --agxir-attach-target="module=agx.* chip=agx-g14c" in.mlir
+ gpu.module @agx_module [#agxir.target<chip = "agx-g14c">] {...}
+ ```
+ }];
+ let options = [
+ Option<"moduleMatcher", "module", "std::string",
+ /*default=*/ [{""}],
+ "Regex used to identify the modules to attach the target to.">,
+ Option<"triple", "triple", "std::string",
+ /*default=*/ "\"agx-apple-macosx\"",
+ "Target triple.">,
+ Option<"chip", "chip", "std::string",
+ /*default=*/"\"agx-g14g\"",
+ "Target chip.">,
+ Option<"features", "features", "std::string",
+ /*default=*/"\"\"",
+ "Target features.">,
+ Option<"optLevel", "O", "unsigned",
+ /*default=*/"2",
+ "Optimization level.">,
+ ListOption<"linkLibs", "l", "std::string",
+ "Extra bitcode libraries paths to link to.">,
+ ];
+}
+
def GpuROCDLAttachTarget: Pass<"rocdl-attach-target", ""> {
let summary = "Attaches a ROCDL target attribute to a GPU Module.";
let description = [{
diff --git a/mlir/include/mlir/Dialect/LLVMIR/AGXIRDialect.h b/mlir/include/mlir/Dialect/LLVMIR/AGXIRDialect.h
new file mode 100644
index 0000000000000..49ae78a049206
--- /dev/null
+++ b/mlir/include/mlir/Dialect/LLVMIR/AGXIRDialect.h
@@ -0,0 +1,34 @@
+//===- AGXIRDialect.h - MLIR AGXIR IR dialect -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The AGXIR dialect wraps the intrinsics of the experimental AGX (Apple GPU)
+// LLVM backend.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_LLVMIR_AGXIRDIALECT_H_
+#define MLIR_DIALECT_LLVMIR_AGXIRDIALECT_H_
+
+#include "mlir/Bytecode/BytecodeOpInterface.h"
+#include "mlir/Dialect/GPU/IR/CompilationInterfaces.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/IR/Dialect.h"
+#include "mlir/IR/OpDefinition.h"
+#include "mlir/Interfaces/SideEffectInterfaces.h"
+
+#include "mlir/Dialect/LLVMIR/AGXIROpsDialect.h.inc"
+
+#include "mlir/Dialect/LLVMIR/AGXIROpsEnums.h.inc"
+
+#define GET_ATTRDEF_CLASSES
+#include "mlir/Dialect/LLVMIR/AGXIROpsAttributes.h.inc"
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/LLVMIR/AGXIROps.h.inc"
+
+#endif // MLIR_DIALECT_LLVMIR_AGXIRDIALECT_H_
diff --git a/mlir/include/mlir/Dialect/LLVMIR/AGXIROps.td b/mlir/include/mlir/Dialect/LLVMIR/AGXIROps.td
new file mode 100644
index 0000000000000..bffc5204f7c35
--- /dev/null
+++ b/mlir/include/mlir/Dialect/LLVMIR/AGXIROps.td
@@ -0,0 +1,330 @@
+//===-- AGXIROps.td - AGXIR dialect operation definitions --*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The AGXIR dialect wraps the llvm.agx.* intrinsics of the experimental AGX
+// (Apple GPU) LLVM backend, like ROCDL wraps the AMDGPU intrinsics.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef AGXIR_OPS
+#define AGXIR_OPS
+
+include "mlir/Dialect/GPU/IR/CompilationAttrInterfaces.td"
+include "mlir/Dialect/LLVMIR/LLVMOpBase.td"
+include "mlir/IR/EnumAttr.td"
+include "mlir/Interfaces/SideEffectInterfaces.td"
+
+//===----------------------------------------------------------------------===//
+// AGXIR dialect definition
+//===----------------------------------------------------------------------===//
+
+def AGXIR_Dialect : Dialect {
+ let name = "agxir";
+ let cppNamespace = "::mlir::AGXIR";
+ let dependentDialects = ["LLVM::LLVMDialect"];
+ let summary = "Dialect for wrapping LLVM AGX backend intrinsics";
+ let hasOperationAttrVerify = 1;
+
+ let description = [{
+ The AGXIR dialect contains 1:1 wrappers around the `llvm.agx.*`
+ intrinsics of the experimental AGX LLVM backend, which generates code for
+ the GPUs of Apple silicon, and the infrastructure to compile GPU modules
+ for them (`#agxir.target`). It plays the role ROCDL plays for AMDGPU; the
+ higher level `agx` dialect, modelled on `amdgpu`, lowers to it.
+
+ The semantics of the operations are those of the intrinsics, see
+ `llvm/include/llvm/IR/IntrinsicsAGX.td` and `llvm/docs/AGXUsage.rst`.
+
+ Functions with the `agxir.kernel` attribute are translated to
+ `agx_kernel` functions: kernel entry points whose arguments are passed in
+ uniform registers.
+ }];
+
+ let extraClassDeclaration = [{
+ /// The address spaces of the AGX backend.
+ static constexpr unsigned kGenericMemoryAddressSpace = 0;
+ static constexpr unsigned kDeviceMemoryAddressSpace = 1;
+ static constexpr unsigned kThreadgroupMemoryAddressSpace = 3;
+ static constexpr unsigned kConstantMemoryAddressSpace = 4;
+ static constexpr unsigned kPrivateMemoryAddressSpace = 5;
+ /// The number of threads in a SIMD-group.
+ static constexpr unsigned kSIMDGroupSize = 32;
+ }];
+
+ let discardableAttrs = (ins
+ "::mlir::UnitAttr":$kernel
+ );
+
+ let useDefaultAttributePrinterParser = 1;
+}
+
+class AGXIR_Attr<string attrName, string attrMnemonic, list<Trait> traits = []>
+ : AttrDef<AGXIR_Dialect, attrName, traits> {
+ let mnemonic = attrMnemonic;
+}
+
+class AGXIR_Op<string mnemonic, list<Trait> traits = []> :
+ LLVM_OpBase<AGXIR_Dialect, mnemonic, traits>;
+
+class AGXIR_IntrOp<string mnemonic, list<int> overloadedResults,
+ list<int> overloadedOperands, list<Trait> traits,
+ int numResults, list<int> immArgPositions = [],
+ list<string> immArgAttrNames = []> :
+ LLVM_IntrOpBase<AGXIR_Dialect, mnemonic,
+ "agx_" # !subst(".", "_", mnemonic), overloadedResults,
+ overloadedOperands, traits, numResults, 0, 0, 0, 0, 0,
+ immArgPositions, immArgAttrNames>;
+
+//===----------------------------------------------------------------------===//
+// Special registers
+//===----------------------------------------------------------------------===//
+
+class AGXIR_SpecialRegisterOp<string mnemonic, list<Trait> traits = [Pure]> :
+ AGXIR_IntrOp<mnemonic, [], [], traits, 1>,
+ Arguments<(ins OptionalAttr<LLVM_ConstantRangeAttr>:$range)> {
+ let results = (outs I32:$res);
+ string llvmBuilder = baseLlvmBuilder # setRangeRetAttrCode #
+ baseLlvmBuilderCoda;
+ string mlirBuilder = baseMlirBuilder # importRangeRetAttrCode #
+ baseMlirBuilderCoda;
+ let assemblyFormat = "(`range` $range^)? attr-dict `:` type($res)";
+ let builders = [
+ OpBuilder<(ins "Type":$resultType), [{
+ build($_builder, $_state, resultType, ::mlir::LLVM::ConstantRangeAttr{});
+ }]>
+ ];
+}
+
+multiclass AGXIR_SpecialRegisterXYZ<string mnemonic> {
+ def XOp : AGXIR_SpecialRegisterOp<mnemonic # ".x">;
+ def YOp : AGXIR_SpecialRegisterOp<mnemonic # ".y">;
+ def ZOp : AGXIR_SpecialRegisterOp<mnemonic # ".z">;
+}
+
+defm AGXIR_ThreadPositionInGrid
+ : AGXIR_SpecialRegisterXYZ<"thread.position.in.grid">;
+defm AGXIR_ThreadPositionInThreadgroup
+ : AGXIR_SpecialRegisterXYZ<"thread.position.in.threadgroup">;
+defm AGXIR_ThreadgroupPositionInGrid
+ : AGXIR_SpecialRegisterXYZ<"threadgroup.position.in.grid">;
+defm AGXIR_ThreadsPerThreadgroup
+ : AGXIR_SpecialRegisterXYZ<"threads.per.threadgroup">;
+defm AGXIR_ThreadgroupsPerGrid
+ : AGXIR_SpecialRegisterXYZ<"threadgroups.per.grid">;
+
+def AGXIR_ThreadIndexInThreadgroupOp
+ : AGXIR_SpecialRegisterOp<"thread.index.in.threadgroup">;
+def AGXIR_ThreadIndexInSIMDGroupOp
+ : AGXIR_SpecialRegisterOp<"thread.index.in.simdgroup">;
+def AGXIR_SIMDGroupIndexInThreadgroupOp
+ : AGXIR_SpecialRegisterOp<"simdgroup.index.in.threadgroup">;
+
+// These depend on the active threads, so they cannot be moved across control
+// flow.
+def AGXIR_ActiveThreadIndexInSIMDGroupOp
+ : AGXIR_SpecialRegisterOp<"active.thread.index.in.simdgroup", []>;
+def AGXIR_TotalActiveThreadsInSIMDGroupOp
+ : AGXIR_SpecialRegisterOp<"total.active.threads.in.simdgroup", []>;
+
+//===----------------------------------------------------------------------===//
+// Synchronization
+//===----------------------------------------------------------------------===//
+
+def AGXIR_ThreadgroupBarrierOp : AGXIR_IntrOp<"threadgroup.barrier", [], [],
+ [], 0> {
+ let summary = "Wait for all threads of the threadgroup";
+ let description = [{
+ Execution barrier for the threadgroup. It does not order memory: combine
+ it with a fence, or use `agxir.memory.barrier`.
+
+ ```mlir
+ agxir.threadgroup.barrier
+ ```
+ }];
+ let assemblyFormat = "attr-dict";
+}
+
+def AGXIR_MemoryBarrierOp : AGXIR_IntrOp<"memory.barrier", [], [], [], 0,
+ [0, 1, 2], ["a", "b", "c"]>,
+ Arguments<(ins I32Attr:$a, I32Attr:$b, I32Attr:$c)> {
+ let summary = "Memory barrier";
+ let description = [{
+ The `memory_barrier` instruction with its three fields; `1, 2, 9` orders
+ device memory, as emitted for fences.
+
+ ```mlir
+ agxir.memory.barrier 1, 2, 9
+ ```
+ }];
+ let assemblyFormat = "$a `,` $b `,` $c attr-dict";
+}
+
+def AGXIR_WaitOp : AGXIR_IntrOp<"wait", [], [], [], 0, [0], ["slot"]>,
+ Arguments<(ins I32Attr:$slot)> {
+ let summary = "Wait for the asynchronous operations of a scoreboard slot";
+ let assemblyFormat = "$slot attr-dict";
+}
+
+//===----------------------------------------------------------------------===//
+// SIMD-group operations
+//===----------------------------------------------------------------------===//
+
+class AGXIR_BallotOp<string mnemonic> :
+ AGXIR_IntrOp<mnemonic, [], [], [], 1>,
+ Arguments<(ins I1:$pred)> {
+ let results = (outs I32:$res);
+ let assemblyFormat = "$pred attr-dict `:` type($res)";
+}
+
+def AGXIR_SIMDBallotOp : AGXIR_BallotOp<"simd.ballot"> {
+ let summary = "Mask of the active threads of the SIMD-group where the "
+ "predicate is true";
+}
+def AGXIR_QuadBallotOp : AGXIR_BallotOp<"quad.ballot"> {
+ let summary = "Mask of the threads of the quad where the predicate is "
+ "true";
+}
+
+class AGXIR_ShuffleOp<string mnemonic> :
+ AGXIR_IntrOp<mnemonic, [0], [], [AllTypesMatch<["res", "value"]>], 1>,
+ Arguments<(ins AnyTypeOf<[I16, I32, F16, F32]>:$value, I32:$index)> {
+ let results = (outs AnyTypeOf<[I16, I32, F16, F32]>:$res);
+ let assemblyFormat = "$value `,` $index attr-dict `:` type($res)";
+}
+
+def AGXIR_SIMDShuffleOp : AGXIR_ShuffleOp<"simd.shuffle"> {
+ let summary = "Read a value from another thread of the SIMD-group";
+ let description = [{
+ ```mlir
+ %r = agxir.simd.shuffle %v, %lane : f32
+ ```
+ }];
+}
+def AGXIR_SIMDShuffleXorOp : AGXIR_ShuffleOp<"simd.shuffle.xor">;
+def AGXIR_SIMDShuffleUpOp : AGXIR_ShuffleOp<"simd.shuffle.up">;
+def AGXIR_SIMDShuffleDownOp : AGXIR_ShuffleOp<"simd.shuffle.down">;
+def AGXIR_QuadShuffleOp : AGXIR_ShuffleOp<"quad.shuffle">;
+def AGXIR_QuadShuffleXorOp : AGXIR_ShuffleOp<"quad.shuffle.xor">;
+def AGXIR_QuadShuffleUpOp : AGXIR_ShuffleOp<"quad.shuffle.up">;
+def AGXIR_QuadShuffleDownOp : AGXIR_ShuffleOp<"quad.shuffle.down">;
+
+// The operation numbers of reductions and prefixes.
+def AGXIR_ReduceAnd : I32EnumAttrCase<"and_", 0, "and">;
+def AGXIR_ReduceFAdd : I32EnumAttrCase<"fadd", 1>;
+def AGXIR_ReduceOr : I32EnumAttrCase<"or_", 2, "or">;
+def AGXIR_ReduceFMul : I32EnumAttrCase<"fmul", 3>;
+def AGXIR_ReduceXor : I32EnumAttrCase<"xor_", 4, "xor">;
+def AGXIR_ReduceFMin : I32EnumAttrCase<"fmin", 5>;
+def AGXIR_ReduceFMax : I32EnumAttrCase<"fmax", 7>;
+def AGXIR_ReduceIAdd : I32EnumAttrCase<"iadd", 16>;
+def AGXIR_ReduceSMin : I32EnumAttrCase<"smin", 20>;
+def AGXIR_ReduceSMax : I32EnumAttrCase<"smax", 22>;
+def AGXIR_ReduceUMin : I32EnumAttrCase<"umin", 28>;
+def AGXIR_ReduceUMax : I32EnumAttrCase<"umax", 30>;
+
+def AGXIR_ReduceKind : I32EnumAttr<"ReduceKind",
+ "Operation of a SIMD-group or quad reduction",
+ [AGXIR_ReduceAnd, AGXIR_ReduceFAdd, AGXIR_ReduceOr, AGXIR_ReduceFMul,
+ AGXIR_ReduceXor, AGXIR_ReduceFMin, AGXIR_ReduceFMax, AGXIR_ReduceIAdd,
+ AGXIR_ReduceSMin, AGXIR_ReduceSMax, AGXIR_ReduceUMin,
+ AGXIR_ReduceUMax]> {
+ let cppNamespace = "::mlir::AGXIR";
+}
+
+class AGXIR_ReduceOp<string mnemonic> :
+ AGXIR_IntrOp<mnemonic, [0], [], [AllTypesMatch<["res", "value"]>], 1,
+ [1], ["kind"]>,
+ Arguments<(ins AnyTypeOf<[I32, F32]>:$value, AGXIR_ReduceKind:$kind)> {
+ let results = (outs AnyTypeOf<[I32, F32]>:$res);
+ let assemblyFormat = "$kind $value attr-dict `:` type($res)";
+ let hasVerifier = 1;
+}
+
+def AGXIR_SIMDReduceOp : AGXIR_ReduceOp<"simd.reduce"> {
+ let summary = "Reduce a value over the active threads of the SIMD-group";
+ let description = [{
+ ```mlir
+ %sum = agxir.simd.reduce fadd %v : f32
+ ```
+ }];
+}
+def AGXIR_SIMDPrefixOp : AGXIR_ReduceOp<"simd.prefix"> {
+ let summary = "Prefix of a value over the active threads of the "
+ "SIMD-group";
+}
+def AGXIR_QuadReduceOp : AGXIR_ReduceOp<"quad.reduce">;
+def AGXIR_QuadPrefixOp : AGXIR_ReduceOp<"quad.prefix">;
+
+//===----------------------------------------------------------------------===//
+// Special functions
+//===----------------------------------------------------------------------===//
+
+class AGXIR_FloatUnaryOp<string mnemonic, list<Trait> traits = [Pure]> :
+ AGXIR_IntrOp<mnemonic, [0], [],
+ !listconcat(traits, [AllTypesMatch<["res", "value"]>]), 1>,
+ Arguments<(ins AnyTypeOf<[F16, F32]>:$value)> {
+ let results = (outs AnyTypeOf<[F16, F32]>:$res);
+ let assemblyFormat = "$value attr-dict `:` type($res)";
+}
+
+def AGXIR_RcpOp : AGXIR_FloatUnaryOp<"rcp"> {
+ let summary = "Hardware reciprocal";
+}
+def AGXIR_RsqrtOp : AGXIR_FloatUnaryOp<"rsqrt"> {
+ let summary = "Hardware reciprocal square root";
+}
+def AGXIR_DfdxOp : AGXIR_FloatUnaryOp<"dfdx", []> {
+ let summary = "Derivative across the quad in x";
+}
+def AGXIR_DfdyOp : AGXIR_FloatUnaryOp<"dfdy", []> {
+ let summary = "Derivative across the quad in y";
+}
+
+//===----------------------------------------------------------------------===//
+// Target attribute
+//===----------------------------------------------------------------------===//
+
+def AGXIR_TargetAttr : AGXIR_Attr<"AGXIRTarget", "target"> {
+ let description = [{
+ Target attribute for compiling GPU modules for Apple GPUs with the AGX
+ LLVM backend. All parameters decay into default values if not present.
+
+ ```mlir
+ gpu.module @kernels [#agxir.target<chip = "agx-g13c">] {
+ ...
+ }
+ ```
+
+ Serialization produces the backend's assembly (`format=isa`), an ELF
+ object with the machine code of each kernel in `.text` (`format=bin`) or
+ LLVM bitcode (`format=llvm`).
+ }];
+ let parameters = (ins
+ DefaultValuedParameter<"int", "2", "Optimization level to apply.">:$O,
+ StringRefParameter<"Target triple.", "\"agx-apple-macosx\"">:$triple,
+ StringRefParameter<"Target chip.", "\"agx-g14g\"">:$chip,
+ StringRefParameter<"Target chip features.", "\"\"">:$features,
+ OptionalParameter<"ArrayAttr", "Files to link to the LLVM module.">:$link
+ );
+ let assemblyFormat = [{
+ (`<` struct($O, $triple, $chip, $features, $link)^ `>`)?
+ }];
+ let builders = [
+ AttrBuilder<(ins CArg<"int", "2">:$optLevel,
+ CArg<"StringRef", "\"agx-apple-macosx\"">:$triple,
+ CArg<"StringRef", "\"agx-g14g\"">:$chip,
+ CArg<"StringRef", "\"\"">:$features,
+ CArg<"ArrayAttr", "nullptr">:$linkFiles), [{
+ return Base::get($_ctxt, optLevel, triple, chip, features, linkFiles);
+ }]>
+ ];
+ let skipDefaultBuilders = 1;
+ let genVerifyDecl = 1;
+}
+
+#endif // AGXIR_OPS
diff --git a/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt b/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
index 1fc410ef075b8..340d06072b7ee 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
@@ -103,6 +103,19 @@ mlir_tablegen(ROCDLOpsAttributes.h.inc -gen-attrdef-decls -attrdefs-dialect=rocd
mlir_tablegen(ROCDLOpsAttributes.cpp.inc -gen-attrdef-defs -attrdefs-dialect=rocdl)
add_mlir_dialect_tablegen_target(MLIRROCDLConversionsIncGen)
+set(LLVM_TARGET_DEFINITIONS AGXIROps.td)
+mlir_tablegen(AGXIROps.h.inc -gen-op-decls)
+mlir_tablegen(AGXIROps.cpp.inc -gen-op-defs)
+mlir_tablegen(AGXIROpsDialect.h.inc -gen-dialect-decls -dialect=agxir)
+mlir_tablegen(AGXIROpsDialect.cpp.inc -gen-dialect-defs -dialect=agxir)
+mlir_tablegen(AGXIROpsEnums.h.inc -gen-enum-decls)
+mlir_tablegen(AGXIROpsEnums.cpp.inc -gen-enum-defs)
+mlir_tablegen(AGXIROpsAttributes.h.inc -gen-attrdef-decls -attrdefs-dialect=agxir)
+mlir_tablegen(AGXIROpsAttributes.cpp.inc -gen-attrdef-defs -attrdefs-dialect=agxir)
+mlir_tablegen(AGXIRConversions.inc -gen-llvmir-conversions)
+add_mlir_dialect_tablegen_target(MLIRAGXIROpsIncGen)
+add_mlir_doc(AGXIROps AGXIRDialect Dialects/ -gen-dialect-doc -dialect=agxir)
+
add_mlir_dialect(VCIXOps vcix)
add_mlir_doc(VCIXOps VCIXDialect Dialects/ -gen-dialect-doc -dialect=vcix)
set(LLVM_TARGET_DEFINITIONS VCIXOps.td)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td
index 803dbdaabfd91..37c8d869a21d7 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td
@@ -272,6 +272,8 @@ def CConvAMDGPU_Gfx : LLVM_EnumAttrCase<"AMDGPU_Gfx", "amdgpu_gfxcc",
"AMDGPU_Gfx", 100>;
def CConvM68k_INTR : LLVM_EnumAttrCase<"M68k_INTR", "m68k_intrcc", "M68k_INTR",
101>;
+def CConvAGX_KERNEL : LLVM_EnumAttrCase<"AGX_KERNEL", "agx_kernelcc",
+ "AGX_KERNEL", 128>;
def CConvEnum : LLVM_CEnumAttr<
"CConv",
@@ -288,7 +290,8 @@ def CConvEnum : LLVM_CEnumAttr<
CConvAMDGPU_VS, CConvAMDGPU_GS, CConvAMDGPU_CS, CConvAMDGPU_KERNEL,
CConvX86_RegCall, CConvAMDGPU_HS, CConvMSP430_BUILTIN, CConvAMDGPU_LS,
CConvAMDGPU_ES, CConvAArch64_VectorCall, CConvAArch64_SVE_VectorCall,
- CConvWASM_EmscriptenInvoke, CConvAMDGPU_Gfx, CConvM68k_INTR
+ CConvWASM_EmscriptenInvoke, CConvAMDGPU_Gfx, CConvM68k_INTR,
+ CConvAGX_KERNEL
]> {
let cppNamespace = "::mlir::LLVM::cconv";
}
diff --git a/mlir/include/mlir/Target/LLVM/AGXIR/Target.h b/mlir/include/mlir/Target/LLVM/AGXIR/Target.h
new file mode 100644
index 0000000000000..4e9d1cfee0331
--- /dev/null
+++ b/mlir/include/mlir/Target/LLVM/AGXIR/Target.h
@@ -0,0 +1,30 @@
+//===- Target.h - MLIR AGXIR target registration ----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This provides registration calls for attaching the AGXIR target interface.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TARGET_LLVM_AGXIR_TARGET_H
+#define MLIR_TARGET_LLVM_AGXIR_TARGET_H
+
+namespace mlir {
+class DialectRegistry;
+class MLIRContext;
+namespace AGXIR {
+/// Registers the `TargetAttrInterface` for the `#agxir.target` attribute in
+/// the given registry.
+void registerAGXIRTargetInterfaceExternalModels(DialectRegistry ®istry);
+
+/// Registers the `TargetAttrInterface` for the `#agxir.target` attribute in
+/// the registry associated with the given context.
+void registerAGXIRTargetInterfaceExternalModels(MLIRContext &context);
+} // namespace AGXIR
+} // namespace mlir
+
+#endif // MLIR_TARGET_LLVM_AGXIR_TARGET_H
diff --git a/mlir/include/mlir/Target/LLVMIR/Dialect/AGXIR/AGXIRToLLVMIRTranslation.h b/mlir/include/mlir/Target/LLVMIR/Dialect/AGXIR/AGXIRToLLVMIRTranslation.h
new file mode 100644
index 0000000000000..07e5fe6d6ca3c
--- /dev/null
+++ b/mlir/include/mlir/Target/LLVMIR/Dialect/AGXIR/AGXIRToLLVMIRTranslation.h
@@ -0,0 +1,31 @@
+//===- AGXIRToLLVMIRTranslation.h - AGXIR to LLVM IR ------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This provides registration calls for AGXIR dialect to LLVM IR translation.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TARGET_LLVMIR_DIALECT_AGXIR_AGXIRTOLLVMIRTRANSLATION_H
+#define MLIR_TARGET_LLVMIR_DIALECT_AGXIR_AGXIRTOLLVMIRTRANSLATION_H
+
+namespace mlir {
+
+class DialectRegistry;
+class MLIRContext;
+
+/// Register the AGXIR dialect and the translation from it to the LLVM IR in
+/// the given registry;
+void registerAGXIRDialectTranslation(DialectRegistry ®istry);
+
+/// Register the AGXIR dialect and the translation from it in the registry
+/// associated with the given context.
+void registerAGXIRDialectTranslation(MLIRContext &context);
+
+} // namespace mlir
+
+#endif // MLIR_TARGET_LLVMIR_DIALECT_AGXIR_AGXIRTOLLVMIRTRANSLATION_H
diff --git a/mlir/include/mlir/Target/LLVMIR/Dialect/All.h b/mlir/include/mlir/Target/LLVMIR/Dialect/All.h
index 95e90789e9580..557c271ee6af9 100644
--- a/mlir/include/mlir/Target/LLVMIR/Dialect/All.h
+++ b/mlir/include/mlir/Target/LLVMIR/Dialect/All.h
@@ -17,6 +17,7 @@
#include "mlir/Target/LLVMIR/Dialect/ArmNeon/ArmNeonToLLVMIRTranslation.h"
#include "mlir/Target/LLVMIR/Dialect/ArmSME/ArmSMEToLLVMIRTranslation.h"
#include "mlir/Target/LLVMIR/Dialect/ArmSVE/ArmSVEToLLVMIRTranslation.h"
+#include "mlir/Target/LLVMIR/Dialect/AGXIR/AGXIRToLLVMIRTranslation.h"
#include "mlir/Target/LLVMIR/Dialect/Builtin/BuiltinToLLVMIRTranslation.h"
#include "mlir/Target/LLVMIR/Dialect/GPU/GPUToLLVMIRTranslation.h"
#include "mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.h"
@@ -36,6 +37,7 @@ class DialectRegistry;
/// Registers all dialects that can be translated to LLVM IR and the
/// corresponding translation interfaces.
static inline void registerAllToLLVMIRTranslations(DialectRegistry ®istry) {
+ registerAGXIRDialectTranslation(registry);
registerArmNeonDialectTranslation(registry);
registerArmSMEDialectTranslation(registry);
registerArmSVEDialectTranslation(registry);
@@ -59,6 +61,7 @@ static inline void registerAllToLLVMIRTranslations(DialectRegistry ®istry) {
/// mechanism is implemented, see D157703.
static inline void
registerAllGPUToLLVMIRTranslations(DialectRegistry ®istry) {
+ registerAGXIRDialectTranslation(registry);
registerBuiltinDialectTranslation(registry);
registerGPUDialectTranslation(registry);
registerLLVMDialectTranslation(registry);
diff --git a/mlir/lib/Conversion/AGXToAGXIR/AGXToAGXIR.cpp b/mlir/lib/Conversion/AGXToAGXIR/AGXToAGXIR.cpp
new file mode 100644
index 0000000000000..046a6c01498e7
--- /dev/null
+++ b/mlir/lib/Conversion/AGXToAGXIR/AGXToAGXIR.cpp
@@ -0,0 +1,256 @@
+//===- AGXToAGXIR.cpp - AGX to AGXIR dialect conversion -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Lowers the `agx` dialect to the AGXIR intrinsic wrappers, like
+// AMDGPUToROCDL: shuffles of any type are split into 32-bit parts,
+// reductions get their intrinsic operation numbers and 16-bit values are
+// extended.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Conversion/AGXToAGXIR/AGXToAGXIR.h"
+
+#include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
+#include "mlir/Conversion/LLVMCommon/Pattern.h"
+#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
+#include "mlir/Dialect/AGX/IR/AGXDialect.h"
+#include "mlir/Dialect/LLVMIR/AGXIRDialect.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Transforms/DialectConversion.h"
+
+namespace mlir {
+#define GEN_PASS_DEF_CONVERTAGXTOAGXIRPASS
+#include "mlir/Conversion/Passes.h.inc"
+} // namespace mlir
+
+using namespace mlir;
+using namespace mlir::agx;
+
+namespace {
+
+struct ThreadgroupBarrierOpLowering
+ : ConvertOpToLLVMPattern<ThreadgroupBarrierOp> {
+ using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
+
+ LogicalResult
+ matchAndRewrite(ThreadgroupBarrierOp op, OpAdaptor adaptor,
+ ConversionPatternRewriter &rewriter) const override {
+ Location loc = op.getLoc();
+ LLVM::FenceOp::create(rewriter, loc, LLVM::AtomicOrdering::release,
+ "workgroup");
+ AGXIR::ThreadgroupBarrierOp::create(rewriter, loc);
+ rewriter.replaceOpWithNewOp<LLVM::FenceOp>(
+ op, LLVM::AtomicOrdering::acquire, "workgroup");
+ return success();
+ }
+};
+
+/// Create the AGXIR shuffle for the mode and scope.
+static Value createShuffle(ConversionPatternRewriter &rewriter, Location loc,
+ ShuffleMode mode, Scope scope, Value value,
+ Value index) {
+ Type type = value.getType();
+ bool quad = scope == Scope::quad;
+ switch (mode) {
+ case ShuffleMode::idx:
+ return quad ? AGXIR::QuadShuffleOp::create(rewriter, loc, type, value,
+ index)
+ .getResult()
+ : AGXIR::SIMDShuffleOp::create(rewriter, loc, type, value,
+ index)
+ .getResult();
+ case ShuffleMode::xor_:
+ return quad ? AGXIR::QuadShuffleXorOp::create(rewriter, loc, type, value,
+ index)
+ .getResult()
+ : AGXIR::SIMDShuffleXorOp::create(rewriter, loc, type, value,
+ index)
+ .getResult();
+ case ShuffleMode::up:
+ return quad ? AGXIR::QuadShuffleUpOp::create(rewriter, loc, type, value,
+ index)
+ .getResult()
+ : AGXIR::SIMDShuffleUpOp::create(rewriter, loc, type, value,
+ index)
+ .getResult();
+ case ShuffleMode::down:
+ return quad ? AGXIR::QuadShuffleDownOp::create(rewriter, loc, type, value,
+ index)
+ .getResult()
+ : AGXIR::SIMDShuffleDownOp::create(rewriter, loc, type, value,
+ index)
+ .getResult();
+ }
+ llvm_unreachable("unknown shuffle mode");
+}
+
+struct ShuffleOpLowering : ConvertOpToLLVMPattern<ShuffleOp> {
+ using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
+
+ LogicalResult
+ matchAndRewrite(ShuffleOp op, OpAdaptor adaptor,
+ ConversionPatternRewriter &rewriter) const override {
+ Location loc = op.getLoc();
+ Value value = adaptor.getValue();
+ Type type = value.getType();
+ // 16 and 32-bit integers and floats are shuffled directly.
+ if (type.isInteger(16) || type.isInteger(32) || type.isF16() ||
+ type.isF32()) {
+ rewriter.replaceOp(op, createShuffle(rewriter, loc, op.getMode(),
+ op.getScope(), value,
+ adaptor.getIndex()));
+ return success();
+ }
+ if (type.isBF16()) {
+ Value bits = LLVM::BitcastOp::create(rewriter, loc,
+ rewriter.getI16Type(), value);
+ Value shuffled = createShuffle(rewriter, loc, op.getMode(),
+ op.getScope(), bits, adaptor.getIndex());
+ rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, type, shuffled);
+ return success();
+ }
+ // Everything else, in 32-bit parts.
+ Type i32 = rewriter.getI32Type();
+ SmallVector<Value> parts;
+ if (failed(LLVM::decomposeValue(rewriter, loc, value, i32, parts)))
+ return rewriter.notifyMatchFailure(op, "cannot split value into i32");
+ SmallVector<Value> shuffled;
+ for (Value part : parts)
+ shuffled.push_back(createShuffle(rewriter, loc, op.getMode(),
+ op.getScope(), part,
+ adaptor.getIndex()));
+ rewriter.replaceOp(op, LLVM::composeValue(rewriter, loc, shuffled, type));
+ return success();
+ }
+};
+
+static AGXIR::ReduceKind getIntrinsicKind(ReduceKind kind, bool isFloat) {
+ switch (kind) {
+ case ReduceKind::add:
+ return isFloat ? AGXIR::ReduceKind::fadd : AGXIR::ReduceKind::iadd;
+ case ReduceKind::mul:
+ return AGXIR::ReduceKind::fmul;
+ case ReduceKind::minsi:
+ return AGXIR::ReduceKind::smin;
+ case ReduceKind::maxsi:
+ return AGXIR::ReduceKind::smax;
+ case ReduceKind::minui:
+ return AGXIR::ReduceKind::umin;
+ case ReduceKind::maxui:
+ return AGXIR::ReduceKind::umax;
+ case ReduceKind::minnumf:
+ return AGXIR::ReduceKind::fmin;
+ case ReduceKind::maxnumf:
+ return AGXIR::ReduceKind::fmax;
+ case ReduceKind::and_:
+ return AGXIR::ReduceKind::and_;
+ case ReduceKind::or_:
+ return AGXIR::ReduceKind::or_;
+ case ReduceKind::xor_:
+ return AGXIR::ReduceKind::xor_;
+ }
+ llvm_unreachable("unknown reduction");
+}
+
+struct ReduceOpLowering : ConvertOpToLLVMPattern<ReduceOp> {
+ using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
+
+ LogicalResult
+ matchAndRewrite(ReduceOp op, OpAdaptor adaptor,
+ ConversionPatternRewriter &rewriter) const override {
+ Location loc = op.getLoc();
+ Value value = adaptor.getValue();
+ Type type = value.getType();
+ bool isFloat = isa<FloatType>(type);
+
+ // The hardware reduces 32-bit values: extend 16-bit ones.
+ Type wideType = isFloat ? Type(rewriter.getF32Type())
+ : Type(rewriter.getI32Type());
+ if (type != wideType) {
+ bool isSigned = op.getKind() == ReduceKind::minsi ||
+ op.getKind() == ReduceKind::maxsi;
+ if (isFloat)
+ value = LLVM::FPExtOp::create(rewriter, loc, wideType, value);
+ else if (isSigned)
+ value = LLVM::SExtOp::create(rewriter, loc, wideType, value);
+ else
+ value = LLVM::ZExtOp::create(rewriter, loc, wideType, value);
+ }
+
+ AGXIR::ReduceKind kind = getIntrinsicKind(op.getKind(), isFloat);
+ Value result;
+ bool quad = op.getScope() == Scope::quad;
+ if (op.getPrefix())
+ result = quad ? AGXIR::QuadPrefixOp::create(rewriter, loc, wideType,
+ value, kind)
+ .getResult()
+ : AGXIR::SIMDPrefixOp::create(rewriter, loc, wideType,
+ value, kind)
+ .getResult();
+ else
+ result = quad ? AGXIR::QuadReduceOp::create(rewriter, loc, wideType,
+ value, kind)
+ .getResult()
+ : AGXIR::SIMDReduceOp::create(rewriter, loc, wideType,
+ value, kind)
+ .getResult();
+
+ if (type != wideType) {
+ if (isFloat)
+ result = LLVM::FPTruncOp::create(rewriter, loc, type, result);
+ else
+ result = LLVM::TruncOp::create(rewriter, loc, type, result);
+ }
+ rewriter.replaceOp(op, result);
+ return success();
+ }
+};
+
+struct BallotOpLowering : ConvertOpToLLVMPattern<BallotOp> {
+ using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
+
+ LogicalResult
+ matchAndRewrite(BallotOp op, OpAdaptor adaptor,
+ ConversionPatternRewriter &rewriter) const override {
+ Type i32 = rewriter.getI32Type();
+ if (op.getScope() == Scope::quad)
+ rewriter.replaceOpWithNewOp<AGXIR::QuadBallotOp>(
+ op, i32, adaptor.getPredicate());
+ else
+ rewriter.replaceOpWithNewOp<AGXIR::SIMDBallotOp>(
+ op, i32, adaptor.getPredicate());
+ return success();
+ }
+};
+
+struct ConvertAGXToAGXIRPass
+ : public impl::ConvertAGXToAGXIRPassBase<ConvertAGXToAGXIRPass> {
+ using Base::Base;
+
+ void runOnOperation() override {
+ MLIRContext *ctx = &getContext();
+ LLVMTypeConverter converter(ctx);
+ RewritePatternSet patterns(ctx);
+ populateAGXToAGXIRConversionPatterns(converter, patterns);
+ LLVMConversionTarget target(getContext());
+ target.addIllegalDialect<agx::AGXDialect>();
+ target.addLegalDialect<AGXIR::AGXIRDialect>();
+ if (failed(applyPartialConversion(getOperation(), target,
+ std::move(patterns))))
+ signalPassFailure();
+ }
+};
+
+} // namespace
+
+void mlir::populateAGXToAGXIRConversionPatterns(
+ const LLVMTypeConverter &converter, RewritePatternSet &patterns) {
+ patterns.add<ThreadgroupBarrierOpLowering, ShuffleOpLowering,
+ ReduceOpLowering, BallotOpLowering>(converter);
+}
diff --git a/mlir/lib/Conversion/AGXToAGXIR/CMakeLists.txt b/mlir/lib/Conversion/AGXToAGXIR/CMakeLists.txt
new file mode 100644
index 0000000000000..44b7a14cb0322
--- /dev/null
+++ b/mlir/lib/Conversion/AGXToAGXIR/CMakeLists.txt
@@ -0,0 +1,17 @@
+add_mlir_conversion_library(MLIRAGXToAGXIR
+ AGXToAGXIR.cpp
+
+ ADDITIONAL_HEADER_DIRS
+ ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/AGXToAGXIR
+
+ DEPENDS
+ MLIRConversionPassIncGen
+
+ LINK_LIBS PUBLIC
+ MLIRAGXDialect
+ MLIRAGXIRDialect
+ MLIRLLVMCommonConversion
+ MLIRLLVMDialect
+ MLIRPass
+ MLIRTransforms
+ )
diff --git a/mlir/lib/Conversion/CMakeLists.txt b/mlir/lib/Conversion/CMakeLists.txt
index cee1a069ad3b1..bc0a62866472e 100644
--- a/mlir/lib/Conversion/CMakeLists.txt
+++ b/mlir/lib/Conversion/CMakeLists.txt
@@ -1,4 +1,5 @@
add_subdirectory(AffineToStandard)
+add_subdirectory(AGXToAGXIR)
add_subdirectory(AMDGPUToROCDL)
add_subdirectory(ArithCommon)
add_subdirectory(ArithToAMDGPU)
@@ -30,6 +31,7 @@ add_subdirectory(FuncToSPIRV)
add_subdirectory(GPUCommon)
add_subdirectory(GPUToLLVMSPV)
add_subdirectory(GPUToNVVM)
+add_subdirectory(GPUToAGXIR)
add_subdirectory(GPUToROCDL)
add_subdirectory(GPUToSPIRV)
add_subdirectory(IndexToLLVM)
diff --git a/mlir/lib/Conversion/GPUToAGXIR/CMakeLists.txt b/mlir/lib/Conversion/GPUToAGXIR/CMakeLists.txt
new file mode 100644
index 0000000000000..a91b6738bb15b
--- /dev/null
+++ b/mlir/lib/Conversion/GPUToAGXIR/CMakeLists.txt
@@ -0,0 +1,20 @@
+add_mlir_conversion_library(MLIRGPUToAGXIRTransforms
+ LowerGpuOpsToAGXIROps.cpp
+
+ DEPENDS
+ MLIRConversionPassIncGen
+
+ LINK_LIBS PUBLIC
+ MLIRAGXDialect
+ MLIRAGXIRDialect
+ MLIRAGXToAGXIR
+ MLIRArithToLLVM
+ MLIRFuncToLLVM
+ MLIRGPUDialect
+ MLIRGPUToGPURuntimeTransforms
+ MLIRGPUTransforms
+ MLIRLLVMCommonConversion
+ MLIRLLVMDialect
+ MLIRMemRefToLLVM
+ MLIRPass
+ )
diff --git a/mlir/lib/Conversion/GPUToAGXIR/LowerGpuOpsToAGXIROps.cpp b/mlir/lib/Conversion/GPUToAGXIR/LowerGpuOpsToAGXIROps.cpp
new file mode 100644
index 0000000000000..b8cc860ad39dc
--- /dev/null
+++ b/mlir/lib/Conversion/GPUToAGXIR/LowerGpuOpsToAGXIROps.cpp
@@ -0,0 +1,383 @@
+//===- LowerGpuOpsToAGXIROps.cpp - MLIR GPU to AGXIR lowering passes ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a pass to generate AGXIR IR operations for higher-level
+// GPU operations, for Apple GPUs. It follows the GPU to ROCDL lowering:
+//
+// gpu.thread_id agxir.thread.position.in.threadgroup.*
+// gpu.block_id agxir.threadgroup.position.in.grid.*
+// gpu.block_dim agxir.threads.per.threadgroup.*
+// gpu.grid_dim agxir.threadgroups.per.grid.*
+// gpu.global_id agxir.thread.position.in.grid.*
+// gpu.lane_id agxir.thread.index.in.simdgroup
+// gpu.subgroup_id agxir.simdgroup.index.in.threadgroup
+// gpu.subgroup_size 32
+// gpu.barrier fences and agxir.threadgroup.barrier
+// gpu.shuffle agxir.simd.shuffle
+// gpu.ballot agxir.simd.ballot
+// gpu.subgroup_reduce agxir.simd.reduce
+// gpu.func agx_kernel functions (agxir.kernel)
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Conversion/GPUToAGXIR/GPUToAGXIRPass.h"
+
+#include "mlir/Conversion/AGXToAGXIR/AGXToAGXIR.h"
+#include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
+#include "mlir/Conversion/ConvertToLLVM/ToLLVMPass.h"
+#include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
+#include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
+#include "mlir/Conversion/LLVMCommon/LoweringOptions.h"
+#include "mlir/Conversion/LLVMCommon/Pattern.h"
+#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
+#include "mlir/Dialect/AGX/IR/AGXDialect.h"
+#include "mlir/Dialect/ControlFlow/IR/ControlFlow.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/Dialect/GPU/IR/GPUDialect.h"
+#include "mlir/Dialect/GPU/Transforms/Passes.h"
+#include "mlir/Dialect/LLVMIR/AGXIRDialect.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Dialect/MemRef/IR/MemRef.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Transforms/DialectConversion.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+
+#include "../GPUCommon/GPUOpsLowering.h"
+#include "../GPUCommon/IndexIntrinsicsOpLowering.h"
+
+namespace mlir {
+#define GEN_PASS_DEF_CONVERTGPUOPSTOAGXIROPS
+#include "mlir/Conversion/Passes.h.inc"
+} // namespace mlir
+
+using namespace mlir;
+
+const char *const mlir::kAGXDataLayout =
+ "e-m:e-p:64:64-p1:64:64-p3:32:32-p4:64:64-p5:32:32-i1:8-i8:8-i16:16-"
+ "i32:32-i64:64-f16:16-f32:32-f64:64-v16:16-v32:32-v64:64-v96:32-"
+ "v128:128-n16:32:64-S32-A5-G1";
+
+/// Extend or truncate an i32 to the index type.
+static Value toIndexBitwidth(ConversionPatternRewriter &rewriter, Location loc,
+ Value value, unsigned indexBitwidth) {
+ Type indexType = rewriter.getIntegerType(indexBitwidth);
+ if (indexBitwidth > 32)
+ return LLVM::SExtOp::create(rewriter, loc, indexType, value);
+ if (indexBitwidth < 32)
+ return LLVM::TruncOp::create(rewriter, loc, indexType, value);
+ return value;
+}
+
+namespace {
+
+template <typename SourceOp, typename AGXIROp>
+struct GPUScalarIndexOpLowering : ConvertOpToLLVMPattern<SourceOp> {
+ using ConvertOpToLLVMPattern<SourceOp>::ConvertOpToLLVMPattern;
+
+ LogicalResult
+ matchAndRewrite(SourceOp op, typename SourceOp::Adaptor adaptor,
+ ConversionPatternRewriter &rewriter) const override {
+ Location loc = op->getLoc();
+ Value value = AGXIROp::create(rewriter, loc, rewriter.getI32Type());
+ rewriter.replaceOp(
+ op, toIndexBitwidth(rewriter, loc, value,
+ this->getTypeConverter()->getIndexTypeBitwidth()));
+ return success();
+ }
+};
+
+struct GPUSubgroupSizeOpLowering
+ : ConvertOpToLLVMPattern<gpu::SubgroupSizeOp> {
+ using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
+
+ LogicalResult
+ matchAndRewrite(gpu::SubgroupSizeOp op, OpAdaptor adaptor,
+ ConversionPatternRewriter &rewriter) const override {
+ Type indexType = rewriter.getIntegerType(
+ getTypeConverter()->getIndexTypeBitwidth());
+ rewriter.replaceOpWithNewOp<LLVM::ConstantOp>(
+ op, indexType, AGXIR::AGXIRDialect::kSIMDGroupSize);
+ return success();
+ }
+};
+
+/// gpu.barrier: fence threadgroup memory around the execution barrier.
+struct GPUBarrierOpLowering : ConvertOpToLLVMPattern<gpu::BarrierOp> {
+ using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
+
+ LogicalResult
+ matchAndRewrite(gpu::BarrierOp op, OpAdaptor adaptor,
+ ConversionPatternRewriter &rewriter) const override {
+ Location loc = op->getLoc();
+ LLVM::FenceOp::create(rewriter, loc, LLVM::AtomicOrdering::release,
+ "workgroup");
+ AGXIR::ThreadgroupBarrierOp::create(rewriter, loc);
+ rewriter.replaceOpWithNewOp<LLVM::FenceOp>(
+ op, LLVM::AtomicOrdering::acquire, "workgroup");
+ return success();
+ }
+};
+
+/// gpu.shuffle: compute the source lane like the ROCDL lowering, falling back
+/// to the current lane outside of the width, and shuffle each 32-bit part.
+struct GPUShuffleOpLowering : ConvertOpToLLVMPattern<gpu::ShuffleOp> {
+ using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
+
+ LogicalResult
+ matchAndRewrite(gpu::ShuffleOp op, OpAdaptor adaptor,
+ ConversionPatternRewriter &rewriter) const override {
+ Location loc = op->getLoc();
+ Type i32 = rewriter.getI32Type();
+ Value lane = AGXIR::ThreadIndexInSIMDGroupOp::create(rewriter, loc, i32);
+ Value width = adaptor.getWidth();
+ Value zero = LLVM::ConstantOp::create(rewriter, loc, i32, 0);
+ Value negWidth = LLVM::SubOp::create(rewriter, loc, i32, zero, width);
+ Value add = LLVM::AddOp::create(rewriter, loc, i32, lane, width);
+ // The end of the lane's segment of `width` lanes.
+ Value segmentEnd = LLVM::AndOp::create(rewriter, loc, i32, add, negWidth);
+ Value srcLane;
+ switch (op.getMode()) {
+ case gpu::ShuffleMode::UP:
+ srcLane = LLVM::SubOp::create(rewriter, loc, i32, lane,
+ adaptor.getOffset());
+ break;
+ case gpu::ShuffleMode::DOWN:
+ srcLane = LLVM::AddOp::create(rewriter, loc, i32, lane,
+ adaptor.getOffset());
+ break;
+ case gpu::ShuffleMode::XOR:
+ srcLane = LLVM::XOrOp::create(rewriter, loc, i32, lane,
+ adaptor.getOffset());
+ break;
+ case gpu::ShuffleMode::IDX:
+ srcLane = adaptor.getOffset();
+ break;
+ }
+ Value valid = LLVM::ICmpOp::create(rewriter, loc, LLVM::ICmpPredicate::slt,
+ srcLane, segmentEnd);
+ Value selected =
+ LLVM::SelectOp::create(rewriter, loc, valid, srcLane, lane);
+
+ SmallVector<Value> parts;
+ if (failed(LLVM::decomposeValue(rewriter, loc, adaptor.getValue(), i32,
+ parts)))
+ return rewriter.notifyMatchFailure(op, "cannot split value into i32");
+ SmallVector<Value> shuffled;
+ for (Value part : parts)
+ shuffled.push_back(
+ AGXIR::SIMDShuffleOp::create(rewriter, loc, i32, part, selected));
+ Value result = LLVM::composeValue(rewriter, loc, shuffled,
+ adaptor.getValue().getType());
+ rewriter.replaceOp(op, {result, valid});
+ return success();
+ }
+};
+
+struct GPUBallotOpLowering : ConvertOpToLLVMPattern<gpu::BallotOp> {
+ using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
+
+ LogicalResult
+ matchAndRewrite(gpu::BallotOp op, OpAdaptor adaptor,
+ ConversionPatternRewriter &rewriter) const override {
+ Location loc = op->getLoc();
+ auto intType = dyn_cast<IntegerType>(op.getType());
+ if (!intType || intType.getWidth() < 32)
+ return rewriter.notifyMatchFailure(op, "result must be at least 32 bits");
+ Value mask = AGXIR::SIMDBallotOp::create(rewriter, loc,
+ rewriter.getI32Type(),
+ adaptor.getPredicate());
+ if (intType.getWidth() > 32)
+ mask = LLVM::ZExtOp::create(rewriter, loc, intType, mask);
+ rewriter.replaceOp(op, mask);
+ return success();
+ }
+};
+
+/// gpu.subgroup_reduce of 32-bit values over the whole SIMD-group.
+struct GPUSubgroupReduceOpLowering
+ : ConvertOpToLLVMPattern<gpu::SubgroupReduceOp> {
+ using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
+
+ LogicalResult
+ matchAndRewrite(gpu::SubgroupReduceOp op, OpAdaptor adaptor,
+ ConversionPatternRewriter &rewriter) const override {
+ if (op.getClusterSize())
+ return rewriter.notifyMatchFailure(op, "clustered reductions");
+ Type type = adaptor.getValue().getType();
+ bool isFloat = type.isF32();
+ if (!isFloat && !type.isInteger(32))
+ return rewriter.notifyMatchFailure(op, "only i32 and f32 are supported");
+ std::optional<AGXIR::ReduceKind> kind;
+ switch (op.getOp()) {
+ case gpu::AllReduceOperation::ADD:
+ kind = isFloat ? AGXIR::ReduceKind::fadd : AGXIR::ReduceKind::iadd;
+ break;
+ case gpu::AllReduceOperation::MUL:
+ if (isFloat)
+ kind = AGXIR::ReduceKind::fmul;
+ break;
+ case gpu::AllReduceOperation::MINSI:
+ kind = AGXIR::ReduceKind::smin;
+ break;
+ case gpu::AllReduceOperation::MAXSI:
+ kind = AGXIR::ReduceKind::smax;
+ break;
+ case gpu::AllReduceOperation::MINUI:
+ kind = AGXIR::ReduceKind::umin;
+ break;
+ case gpu::AllReduceOperation::MAXUI:
+ kind = AGXIR::ReduceKind::umax;
+ break;
+ case gpu::AllReduceOperation::MINNUMF:
+ kind = AGXIR::ReduceKind::fmin;
+ break;
+ case gpu::AllReduceOperation::MAXNUMF:
+ kind = AGXIR::ReduceKind::fmax;
+ break;
+ case gpu::AllReduceOperation::AND:
+ kind = AGXIR::ReduceKind::and_;
+ break;
+ case gpu::AllReduceOperation::OR:
+ kind = AGXIR::ReduceKind::or_;
+ break;
+ case gpu::AllReduceOperation::XOR:
+ kind = AGXIR::ReduceKind::xor_;
+ break;
+ default:
+ break;
+ }
+ if (!kind)
+ return rewriter.notifyMatchFailure(op, "unsupported reduction");
+ rewriter.replaceOpWithNewOp<AGXIR::SIMDReduceOp>(op, type,
+ adaptor.getValue(), *kind);
+ return success();
+ }
+};
+
+struct LowerGpuOpsToAGXIROpsPass final
+ : public impl::ConvertGpuOpsToAGXIROpsBase<LowerGpuOpsToAGXIROpsPass> {
+ using Base::Base;
+
+ void getDependentDialects(DialectRegistry ®istry) const override {
+ Base::getDependentDialects(registry);
+ registerConvertToLLVMDependentDialectLoading(registry);
+ }
+
+ void runOnOperation() override {
+ gpu::GPUModuleOp m = getOperation();
+ MLIRContext *ctx = m.getContext();
+
+ auto llvmDataLayout = m->getAttrOfType<StringAttr>(
+ LLVM::LLVMDialect::getDataLayoutAttrName());
+ if (!llvmDataLayout) {
+ llvmDataLayout = StringAttr::get(ctx, kAGXDataLayout);
+ m->setAttr(LLVM::LLVMDialect::getDataLayoutAttrName(), llvmDataLayout);
+ }
+
+ LowerToLLVMOptions options(
+ ctx, DataLayout(cast<DataLayoutOpInterface>(m.getOperation())));
+ options.dataLayout = llvm::DataLayout(llvmDataLayout.getValue());
+ if (indexBitwidth != kDeriveIndexBitwidthFromDataLayout)
+ options.overrideIndexBitwidth(indexBitwidth);
+ options.useBarePtrCallConv = useBarePtrCallConv;
+
+ // In-dialect lowering of gpu.all_reduce and friends first.
+ {
+ RewritePatternSet patterns(ctx);
+ populateGpuRewritePatterns(patterns);
+ (void)applyPatternsGreedily(m, std::move(patterns));
+ }
+
+ LLVMTypeConverter converter(ctx, options);
+ populateGpuMemorySpaceAttributeConversions(
+ converter, [](gpu::AddressSpace space) -> unsigned {
+ switch (space) {
+ case gpu::AddressSpace::Global:
+ return AGXIR::AGXIRDialect::kDeviceMemoryAddressSpace;
+ case gpu::AddressSpace::Workgroup:
+ return AGXIR::AGXIRDialect::kThreadgroupMemoryAddressSpace;
+ case gpu::AddressSpace::Private:
+ return AGXIR::AGXIRDialect::kPrivateMemoryAddressSpace;
+ case gpu::AddressSpace::Constant:
+ return AGXIR::AGXIRDialect::kConstantMemoryAddressSpace;
+ }
+ llvm_unreachable("unknown address space enum value");
+ });
+
+ RewritePatternSet llvmPatterns(ctx);
+ LLVMConversionTarget target(getContext());
+ for (Dialect *dialect : ctx->getLoadedDialects())
+ if (auto *iface = dyn_cast<ConvertToLLVMPatternInterface>(dialect))
+ iface->populateConvertToLLVMConversionPatterns(target, converter,
+ llvmPatterns);
+ populateAGXToAGXIRConversionPatterns(converter, llvmPatterns);
+ populateGpuToAGXIRConversionPatterns(converter, llvmPatterns);
+ configureGpuToAGXIRConversionLegality(target);
+ if (failed(applyPartialConversion(m, target, std::move(llvmPatterns))))
+ signalPassFailure();
+ }
+};
+
+} // namespace
+
+void mlir::configureGpuToAGXIRConversionLegality(ConversionTarget &target) {
+ target.addIllegalOp<func::FuncOp>();
+ target.addLegalDialect<LLVM::LLVMDialect>();
+ target.addLegalDialect<AGXIR::AGXIRDialect>();
+ target.addIllegalDialect<gpu::GPUDialect>();
+ target.addIllegalDialect<agx::AGXDialect>();
+ // TODO: Remove once we support replacing non-root ops.
+ target.addLegalOp<gpu::YieldOp, gpu::GPUModuleOp>();
+}
+
+void mlir::populateGpuToAGXIRConversionPatterns(
+ const LLVMTypeConverter &converter, RewritePatternSet &patterns) {
+ using gpu::index_lowering::IndexKind;
+ using gpu::index_lowering::IntrType;
+ using gpu::index_lowering::OpLowering;
+ auto *agxirDialect =
+ converter.getContext().getLoadedDialect<AGXIR::AGXIRDialect>();
+
+ patterns.add<OpLowering<gpu::ThreadIdOp, AGXIR::ThreadPositionInThreadgroupXOp,
+ AGXIR::ThreadPositionInThreadgroupYOp,
+ AGXIR::ThreadPositionInThreadgroupZOp>>(
+ converter, IndexKind::Block, IntrType::Id);
+ patterns.add<OpLowering<gpu::BlockIdOp, AGXIR::ThreadgroupPositionInGridXOp,
+ AGXIR::ThreadgroupPositionInGridYOp,
+ AGXIR::ThreadgroupPositionInGridZOp>>(
+ converter, IndexKind::Grid, IntrType::Id);
+ patterns.add<OpLowering<gpu::BlockDimOp, AGXIR::ThreadsPerThreadgroupXOp,
+ AGXIR::ThreadsPerThreadgroupYOp,
+ AGXIR::ThreadsPerThreadgroupZOp>>(
+ converter, IndexKind::Block, IntrType::Dim);
+ patterns.add<OpLowering<gpu::GridDimOp, AGXIR::ThreadgroupsPerGridXOp,
+ AGXIR::ThreadgroupsPerGridYOp,
+ AGXIR::ThreadgroupsPerGridZOp>>(
+ converter, IndexKind::Grid, IntrType::Dim);
+ patterns.add<OpLowering<gpu::GlobalIdOp, AGXIR::ThreadPositionInGridXOp,
+ AGXIR::ThreadPositionInGridYOp,
+ AGXIR::ThreadPositionInGridZOp>>(converter);
+ patterns.add<
+ GPUScalarIndexOpLowering<gpu::LaneIdOp, AGXIR::ThreadIndexInSIMDGroupOp>,
+ GPUScalarIndexOpLowering<gpu::SubgroupIdOp,
+ AGXIR::SIMDGroupIndexInThreadgroupOp>,
+ GPUSubgroupSizeOpLowering, GPUBarrierOpLowering, GPUShuffleOpLowering,
+ GPUBallotOpLowering, GPUSubgroupReduceOpLowering>(converter);
+ patterns.add<GPUReturnOpLowering>(converter);
+ patterns.add<GPUFuncOpLowering>(
+ converter,
+ GPUFuncOpLoweringOptions{
+ /*allocaAddrSpace=*/
+ AGXIR::AGXIRDialect::kPrivateMemoryAddressSpace,
+ /*workgroupAddrSpace=*/
+ AGXIR::AGXIRDialect::kThreadgroupMemoryAddressSpace,
+ agxirDialect->getKernelAttrHelper().getName(),
+ /*kernelBlockSizeAttributeName=*/{},
+ /*kernelClusterSizeAttributeName=*/{},
+ /*kernelCallingConvention=*/LLVM::CConv::AGX_KERNEL});
+}
diff --git a/mlir/lib/Dialect/AGX/CMakeLists.txt b/mlir/lib/Dialect/AGX/CMakeLists.txt
new file mode 100644
index 0000000000000..f33061b2d87cf
--- /dev/null
+++ b/mlir/lib/Dialect/AGX/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(IR)
diff --git a/mlir/lib/Dialect/AGX/IR/AGXDialect.cpp b/mlir/lib/Dialect/AGX/IR/AGXDialect.cpp
new file mode 100644
index 0000000000000..a77a68ed34d3d
--- /dev/null
+++ b/mlir/lib/Dialect/AGX/IR/AGXDialect.cpp
@@ -0,0 +1,90 @@
+//===- AGXDialect.cpp - MLIR AGX dialect implementation -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the AGX dialect and its operations.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/AGX/IR/AGXDialect.h"
+
+#include "mlir/IR/Builders.h"
+#include "mlir/IR/TypeUtilities.h"
+#include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/OpImplementation.h"
+#include "llvm/ADT/TypeSwitch.h"
+
+using namespace mlir;
+using namespace mlir::agx;
+
+#include "mlir/Dialect/AGX/IR/AGXDialect.cpp.inc"
+#include "mlir/Dialect/AGX/IR/AGXEnums.cpp.inc"
+
+void AGXDialect::initialize() {
+ addOperations<
+#define GET_OP_LIST
+#include "mlir/Dialect/AGX/IR/AGX.cpp.inc"
+ >();
+ addAttributes<
+#define GET_ATTRDEF_LIST
+#include "mlir/Dialect/AGX/IR/AGXAttrs.cpp.inc"
+ >();
+}
+
+//===----------------------------------------------------------------------===//
+// ShuffleOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult ShuffleOp::verify() {
+ Type type = getValue().getType();
+ Type elementType = getElementTypeOrSelf(type);
+ if (!elementType.isIntOrFloat())
+ return emitOpError() << "requires integer or floating point values, got "
+ << type;
+ unsigned bits = elementType.getIntOrFloatBitWidth();
+ if (auto vectorType = dyn_cast<VectorType>(type)) {
+ if (vectorType.isScalable())
+ return emitOpError() << "does not support scalable vectors";
+ bits *= vectorType.getNumElements();
+ }
+ // The value is split into 32-bit parts; 16-bit values are shuffled
+ // directly.
+ if (bits != 16 && bits % 32 != 0)
+ return emitOpError() << "requires a 16-bit value or a multiple of 32 bits, "
+ "got "
+ << type;
+ return success();
+}
+
+//===----------------------------------------------------------------------===//
+// ReduceOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult ReduceOp::verify() {
+ bool isFloatKind = getKind() == ReduceKind::minnumf ||
+ getKind() == ReduceKind::maxnumf;
+ bool isArithKind = getKind() == ReduceKind::add ||
+ getKind() == ReduceKind::mul;
+ Type type = getValue().getType();
+ if (isa<FloatType>(type)) {
+ if (!isFloatKind && !isArithKind)
+ return emitOpError() << "reduction '" << stringifyReduceKind(getKind())
+ << "' does not apply to floating point values";
+ } else if (isFloatKind) {
+ return emitOpError() << "reduction '" << stringifyReduceKind(getKind())
+ << "' requires floating point values";
+ } else if (getKind() == ReduceKind::mul) {
+ return emitOpError() << "there is no integer multiplication reduction";
+ }
+ return success();
+}
+
+#define GET_ATTRDEF_CLASSES
+#include "mlir/Dialect/AGX/IR/AGXAttrs.cpp.inc"
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/AGX/IR/AGX.cpp.inc"
diff --git a/mlir/lib/Dialect/AGX/IR/CMakeLists.txt b/mlir/lib/Dialect/AGX/IR/CMakeLists.txt
new file mode 100644
index 0000000000000..c09c643eb7629
--- /dev/null
+++ b/mlir/lib/Dialect/AGX/IR/CMakeLists.txt
@@ -0,0 +1,14 @@
+add_mlir_dialect_library(MLIRAGXDialect
+ AGXDialect.cpp
+
+ ADDITIONAL_HEADER_DIRS
+ ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/AGX
+
+ DEPENDS
+ MLIRAGXEnumsGen
+ MLIRAGXIncGen
+
+ LINK_LIBS PUBLIC
+ MLIRIR
+ MLIRSideEffectInterfaces
+ )
diff --git a/mlir/lib/Dialect/CMakeLists.txt b/mlir/lib/Dialect/CMakeLists.txt
index 55e75f3f0b9ce..98087e292ed91 100644
--- a/mlir/lib/Dialect/CMakeLists.txt
+++ b/mlir/lib/Dialect/CMakeLists.txt
@@ -1,4 +1,5 @@
add_subdirectory(Affine)
+add_subdirectory(AGX)
add_subdirectory(AMDGPU)
add_subdirectory(Arith)
add_subdirectory(ArmNeon)
diff --git a/mlir/lib/Dialect/GPU/CMakeLists.txt b/mlir/lib/Dialect/GPU/CMakeLists.txt
index 6726bf0cab1d2..ff40bcb3628a5 100644
--- a/mlir/lib/Dialect/GPU/CMakeLists.txt
+++ b/mlir/lib/Dialect/GPU/CMakeLists.txt
@@ -31,6 +31,7 @@ add_mlir_dialect_library(MLIRGPUDialect
)
add_mlir_dialect_library(MLIRGPUTransforms
+ Transforms/AGXIRAttachTarget.cpp
Transforms/AllReduceLowering.cpp
Transforms/AsyncRegionRewriter.cpp
Transforms/BufferDeallocationOpInterfaceImpl.cpp
@@ -77,6 +78,8 @@ add_mlir_dialect_library(MLIRGPUTransforms
MLIRMemRefDialect
MLIRNVVMTarget
MLIRPass
+ MLIRAGXIRDialect
+ MLIRAGXIRTarget
MLIRROCDLDialect
MLIRROCDLTarget
MLIRSCFDialect
diff --git a/mlir/lib/Dialect/GPU/Transforms/AGXIRAttachTarget.cpp b/mlir/lib/Dialect/GPU/Transforms/AGXIRAttachTarget.cpp
new file mode 100644
index 0000000000000..bbb2915eff99a
--- /dev/null
+++ b/mlir/lib/Dialect/GPU/Transforms/AGXIRAttachTarget.cpp
@@ -0,0 +1,68 @@
+//===- AGXIRAttachTarget.cpp - Attach an AGXIR target ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the `GpuAGXIRAttachTarget` pass, attaching
+// `#agxir.target` attributes to GPU modules.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/GPU/Transforms/Passes.h"
+
+#include "mlir/Dialect/GPU/IR/GPUDialect.h"
+#include "mlir/Dialect/LLVMIR/AGXIRDialect.h"
+#include "mlir/IR/Builders.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Target/LLVM/AGXIR/Target.h"
+#include "llvm/Support/Regex.h"
+
+namespace mlir {
+#define GEN_PASS_DEF_GPUAGXIRATTACHTARGET
+#include "mlir/Dialect/GPU/Transforms/Passes.h.inc"
+} // namespace mlir
+
+using namespace mlir;
+using namespace mlir::AGXIR;
+
+namespace {
+struct AGXIRAttachTarget
+ : public impl::GpuAGXIRAttachTargetBase<AGXIRAttachTarget> {
+ using Base::Base;
+
+ void runOnOperation() override;
+
+ void getDependentDialects(DialectRegistry ®istry) const override {
+ registry.insert<AGXIR::AGXIRDialect>();
+ }
+};
+} // namespace
+
+void AGXIRAttachTarget::runOnOperation() {
+ OpBuilder builder(&getContext());
+ ArrayRef<std::string> libs(linkLibs);
+ SmallVector<StringRef> filesToLink(libs);
+ auto target = builder.getAttr<AGXIRTargetAttr>(
+ optLevel, triple, chip, features,
+ filesToLink.empty() ? nullptr : builder.getStrArrayAttr(filesToLink));
+ llvm::Regex matcher(moduleMatcher);
+ for (Region ®ion : getOperation()->getRegions())
+ for (Block &block : region.getBlocks())
+ for (auto module : block.getOps<gpu::GPUModuleOp>()) {
+ // Check if the name of the module matches.
+ if (!moduleMatcher.empty() && !matcher.match(module.getName()))
+ continue;
+ // Create the target array.
+ SmallVector<Attribute> targets;
+ if (std::optional<ArrayAttr> attrs = module.getTargets())
+ targets.append(attrs->getValue().begin(), attrs->getValue().end());
+ targets.push_back(target);
+ // Remove any duplicate targets.
+ targets.erase(llvm::unique(targets), targets.end());
+ // Update the target attribute array.
+ module.setTargetsAttr(builder.getArrayAttr(targets));
+ }
+}
diff --git a/mlir/lib/Dialect/LLVMIR/CMakeLists.txt b/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
index 6b7827d5ab12b..a181889f5ed5e 100644
--- a/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
+++ b/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
@@ -139,6 +139,27 @@ add_mlir_dialect_library(MLIRROCDLDialect
MLIRSideEffectInterfaces
)
+add_mlir_dialect_library(MLIRAGXIRDialect
+ IR/AGXIRDialect.cpp
+
+ ADDITIONAL_HEADER_DIRS
+ ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/LLVMIR
+
+ DEPENDS
+ MLIRGPUCompilationAttrInterfacesIncGen
+ MLIRAGXIROpsIncGen
+ intrinsics_gen
+
+ LINK_COMPONENTS
+ AsmParser
+ Core
+
+ LINK_LIBS PUBLIC
+ MLIRIR
+ MLIRLLVMDialect
+ MLIRSideEffectInterfaces
+ )
+
add_mlir_dialect_library(MLIRVCIXDialect
IR/VCIXDialect.cpp
diff --git a/mlir/lib/Dialect/LLVMIR/IR/AGXIRDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/AGXIRDialect.cpp
new file mode 100644
index 0000000000000..c5ac0714e8dfc
--- /dev/null
+++ b/mlir/lib/Dialect/LLVMIR/IR/AGXIRDialect.cpp
@@ -0,0 +1,121 @@
+//===- AGXIRDialect.cpp - AGXIR IR Ops and Dialect registration -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The AGXIR dialect: operations wrapping the llvm.agx.* intrinsics and the
+// #agxir.target attribute.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/LLVMIR/AGXIRDialect.h"
+
+#include "mlir/Dialect/GPU/IR/CompilationInterfaces.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/MLIRContext.h"
+#include "mlir/IR/Operation.h"
+#include "mlir/Transforms/InliningUtils.h"
+#include "llvm/ADT/TypeSwitch.h"
+
+using namespace mlir;
+using namespace mlir::AGXIR;
+
+#include "mlir/Dialect/LLVMIR/AGXIROpsDialect.cpp.inc"
+#include "mlir/Dialect/LLVMIR/AGXIROpsEnums.cpp.inc"
+
+namespace {
+struct AGXIRInlinerInterface final : DialectInlinerInterface {
+ using DialectInlinerInterface::DialectInlinerInterface;
+ bool isLegalToInline(Operation *, Region *, bool, IRMapping &) const final {
+ return true;
+ }
+};
+} // namespace
+
+void AGXIRDialect::initialize() {
+ addOperations<
+#define GET_OP_LIST
+#include "mlir/Dialect/LLVMIR/AGXIROps.cpp.inc"
+ >();
+
+ addAttributes<
+#define GET_ATTRDEF_LIST
+#include "mlir/Dialect/LLVMIR/AGXIROpsAttributes.cpp.inc"
+ >();
+
+ addInterfaces<AGXIRInlinerInterface>();
+ declarePromisedInterface<gpu::TargetAttrInterface, AGXIRTargetAttr>();
+}
+
+LogicalResult AGXIRDialect::verifyOperationAttribute(Operation *op,
+ NamedAttribute attr) {
+ if (kernelAttrName.getName() == attr.getName() &&
+ !isa<LLVM::LLVMFuncOp>(op))
+ return op->emitError() << "'" << kernelAttrName.getName()
+ << "' attribute attached to unexpected op";
+ return success();
+}
+
+//===----------------------------------------------------------------------===//
+// Operations
+//===----------------------------------------------------------------------===//
+
+/// Floating point reductions take f32 values, integer and bitwise ones i32.
+static LogicalResult verifyReduceKind(Operation *op, ReduceKind kind,
+ Type type) {
+ bool isFloatKind = kind == ReduceKind::fadd || kind == ReduceKind::fmul ||
+ kind == ReduceKind::fmin || kind == ReduceKind::fmax;
+ if (isFloatKind != isa<FloatType>(type))
+ return op->emitOpError() << "reduction '" << stringifyReduceKind(kind)
+ << "' does not apply to " << type;
+ return success();
+}
+
+LogicalResult SIMDReduceOp::verify() {
+ return verifyReduceKind(*this, getKind(), getType());
+}
+LogicalResult SIMDPrefixOp::verify() {
+ return verifyReduceKind(*this, getKind(), getType());
+}
+LogicalResult QuadReduceOp::verify() {
+ return verifyReduceKind(*this, getKind(), getType());
+}
+LogicalResult QuadPrefixOp::verify() {
+ return verifyReduceKind(*this, getKind(), getType());
+}
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/LLVMIR/AGXIROps.cpp.inc"
+
+//===----------------------------------------------------------------------===//
+// Target attribute
+//===----------------------------------------------------------------------===//
+
+LogicalResult
+AGXIRTargetAttr::verify(function_ref<InFlightDiagnostic()> emitError,
+ int optLevel, StringRef triple, StringRef chip,
+ StringRef features, ArrayAttr files) {
+ if (optLevel < 0 || optLevel > 3)
+ return emitError() << "the optimization level must be a number between 0 "
+ "and 3";
+ if (triple.empty())
+ return emitError() << "the target triple cannot be empty";
+ if (!chip.starts_with("agx-") && chip != "generic")
+ return emitError() << "the target chip must be an AGX processor, such as "
+ "agx-g14g";
+ if (files && !llvm::all_of(files, [](Attribute attr) {
+ return attr && mlir::isa<StringAttr>(attr);
+ }))
+ return emitError() << "all the elements in the `link` array must be "
+ "strings";
+ return success();
+}
+
+#define GET_ATTRDEF_CLASSES
+#include "mlir/Dialect/LLVMIR/AGXIROpsAttributes.cpp.inc"
diff --git a/mlir/lib/RegisterAllDialects.cpp b/mlir/lib/RegisterAllDialects.cpp
index 11005ab706cd3..b18c63621d113 100644
--- a/mlir/lib/RegisterAllDialects.cpp
+++ b/mlir/lib/RegisterAllDialects.cpp
@@ -13,6 +13,7 @@
#include "mlir/InitAllDialects.h"
+#include "mlir/Dialect/AGX/IR/AGXDialect.h"
#include "mlir/Dialect/AMDGPU/IR/AMDGPUDialectDecl.h"
#include "mlir/Dialect/AMDGPU/Transforms/MemoryAccessOpInterfacesImpl.h"
#include "mlir/Dialect/Affine/IR/AffineDialect.h"
@@ -46,6 +47,7 @@
#include "mlir/Dialect/Index/IR/IndexDialect.h"
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
#include "mlir/Dialect/LLVMIR/NVVMDialectDecl.h"
+#include "mlir/Dialect/LLVMIR/AGXIRDialect.h"
#include "mlir/Dialect/LLVMIR/ROCDLDialectDecl.h"
#include "mlir/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.h"
#include "mlir/Dialect/LLVMIR/XeVMDialect.h"
@@ -107,6 +109,7 @@
#include "mlir/Dialect/XeGPU/IR/XeGPU.h"
#include "mlir/IR/Dialect.h"
#include "mlir/Interfaces/CastInterfaces.h"
+#include "mlir/Target/LLVM/AGXIR/Target.h"
#include "mlir/Target/LLVM/NVVM/Target.h"
#include "mlir/Target/LLVM/ROCDL/Target.h"
#include "mlir/Target/LLVM/XeVM/Target.h"
@@ -116,6 +119,8 @@
void mlir::registerAllDialects(DialectRegistry ®istry) {
// clang-format off
registry.insert<acc::OpenACCDialect,
+ agx::AGXDialect,
+ AGXIR::AGXIRDialect,
affine::AffineDialect,
amdgpu::AMDGPUDialect,
arith::ArithDialect,
@@ -211,6 +216,7 @@ void mlir::registerAllDialects(DialectRegistry ®istry) {
vector::registerSubsetOpInterfaceExternalModels(registry);
vector::registerValueBoundsOpInterfaceExternalModels(registry);
NVVM::registerNVVMTargetInterfaceExternalModels(registry);
+ AGXIR::registerAGXIRTargetInterfaceExternalModels(registry);
ROCDL::registerROCDLTargetInterfaceExternalModels(registry);
spirv::registerSPIRVTargetInterfaceExternalModels(registry);
xevm::registerXeVMTargetInterfaceExternalModels(registry);
diff --git a/mlir/lib/Target/LLVM/AGXIR/Target.cpp b/mlir/lib/Target/LLVM/AGXIR/Target.cpp
new file mode 100644
index 0000000000000..fbdfdc2ce16f4
--- /dev/null
+++ b/mlir/lib/Target/LLVM/AGXIR/Target.cpp
@@ -0,0 +1,198 @@
+//===- Target.cpp - MLIR LLVM AGXIR target compilation ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This files defines AGXIR target related functions including registration
+// calls for the `#agxir.target` compilation attribute: GPU modules are
+// compiled with the experimental AGX LLVM backend.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Target/LLVM/AGXIR/Target.h"
+
+#include "mlir/Dialect/GPU/IR/GPUDialect.h"
+#include "mlir/Dialect/LLVMIR/AGXIRDialect.h"
+#include "mlir/Target/LLVM/ModuleToObject.h"
+#include "mlir/Target/LLVMIR/Dialect/AGXIR/AGXIRToLLVMIRTranslation.h"
+#include "mlir/Target/LLVMIR/Export.h"
+
+#include "llvm/IR/LegacyPassManager.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/Threading.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetMachine.h"
+
+#include <optional>
+
+using namespace mlir;
+using namespace mlir::AGXIR;
+
+#if MLIR_AGX_TARGET_ENABLED
+extern "C" {
+void LLVMInitializeAGXTargetInfo();
+void LLVMInitializeAGXTarget();
+void LLVMInitializeAGXTargetMC();
+void LLVMInitializeAGXAsmParser();
+void LLVMInitializeAGXAsmPrinter();
+}
+#endif
+
+namespace {
+class AGXIRTargetAttrImpl
+ : public gpu::TargetAttrInterface::FallbackModel<AGXIRTargetAttrImpl> {
+public:
+ std::optional<gpu::SerializedObject>
+ serializeToObject(Attribute attribute, Operation *module,
+ const gpu::TargetOptions &options) const;
+
+ Attribute createObject(Attribute attribute, Operation *module,
+ const gpu::SerializedObject &object,
+ const gpu::TargetOptions &options) const;
+};
+} // namespace
+
+void mlir::AGXIR::registerAGXIRTargetInterfaceExternalModels(
+ DialectRegistry ®istry) {
+ registry.addExtension(+[](MLIRContext *ctx, AGXIR::AGXIRDialect *dialect) {
+ AGXIRTargetAttr::attachInterface<AGXIRTargetAttrImpl>(*ctx);
+ });
+}
+
+void mlir::AGXIR::registerAGXIRTargetInterfaceExternalModels(
+ MLIRContext &context) {
+ DialectRegistry registry;
+ registerAGXIRTargetInterfaceExternalModels(registry);
+ context.appendDialectRegistry(registry);
+}
+
+#if MLIR_AGX_TARGET_ENABLED
+namespace {
+/// Serializes a GPU module to AGX assembly, an ELF object or LLVM bitcode.
+class AGXSerializer : public LLVM::ModuleToObject {
+public:
+ AGXSerializer(Operation &module, AGXIRTargetAttr target,
+ const gpu::TargetOptions &targetOptions)
+ : ModuleToObject(module, target.getTriple(), target.getChip(),
+ target.getFeatures(), target.getO(),
+ targetOptions.getInitialLlvmIRCallback(),
+ targetOptions.getLinkedLlvmIRCallback(),
+ targetOptions.getOptimizedLlvmIRCallback(),
+ targetOptions.getISACallback()),
+ targetOptions(targetOptions),
+ librariesToLink(targetOptions.getLibrariesToLink()) {
+ if (target.getLink())
+ librariesToLink.append(target.getLink().begin(), target.getLink().end());
+ }
+
+ static void init() {
+ static llvm::once_flag initializeBackendOnce;
+ llvm::call_once(initializeBackendOnce, []() {
+ LLVMInitializeAGXTargetInfo();
+ LLVMInitializeAGXTarget();
+ LLVMInitializeAGXTargetMC();
+ LLVMInitializeAGXAsmParser();
+ LLVMInitializeAGXAsmPrinter();
+ });
+ }
+
+protected:
+ std::optional<SmallVector<std::unique_ptr<llvm::Module>>>
+ loadBitcodeFiles(llvm::Module &module) override {
+ SmallVector<std::unique_ptr<llvm::Module>> bcFiles;
+ if (failed(loadBitcodeFilesFromList(module.getContext(), librariesToLink,
+ bcFiles, true)))
+ return std::nullopt;
+ return std::move(bcFiles);
+ }
+
+ FailureOr<SmallVector<char, 0>>
+ moduleToObject(llvm::Module &llvmModule) override {
+ if (targetOptions.getCompilationTarget() ==
+ gpu::CompilationTarget::Offload)
+ return ModuleToObject::moduleToObject(llvmModule);
+
+ FailureOr<llvm::TargetMachine *> targetMachine =
+ getOrCreateTargetMachine();
+ if (failed(targetMachine))
+ return getOperation().emitError()
+ << "target machine unavailable for triple " << triple
+ << ", can't compile with LLVM";
+
+ if (targetOptions.getCompilationTarget() ==
+ gpu::CompilationTarget::Assembly) {
+ FailureOr<SmallString<0>> isa =
+ translateModuleToISA(llvmModule, **targetMachine,
+ [&]() { return getOperation().emitError(); });
+ if (failed(isa))
+ return failure();
+ if (isaCallback)
+ isaCallback(*isa);
+ return SmallVector<char, 0>(isa->begin(), isa->end());
+ }
+
+ // The ELF object, with the machine code of each kernel in .text.
+ SmallVector<char, 0> object;
+ {
+ llvm::raw_svector_ostream stream(object);
+ llvm::legacy::PassManager codegenPasses;
+ if ((*targetMachine)
+ ->addPassesToEmitFile(codegenPasses, stream, nullptr,
+ llvm::CodeGenFileType::ObjectFile))
+ return getOperation().emitError()
+ << "the AGX target cannot emit object files";
+ codegenPasses.run(llvmModule);
+ }
+ return object;
+ }
+
+private:
+ gpu::TargetOptions targetOptions;
+ SmallVector<Attribute> librariesToLink;
+};
+} // namespace
+#endif // MLIR_AGX_TARGET_ENABLED
+
+std::optional<gpu::SerializedObject>
+AGXIRTargetAttrImpl::serializeToObject(Attribute attribute, Operation *module,
+ const gpu::TargetOptions &options) const {
+ assert(module && "The module must be non null.");
+ if (!module)
+ return std::nullopt;
+ if (!mlir::isa<gpu::GPUModuleOp>(module)) {
+ module->emitError("module must be a GPU module");
+ return std::nullopt;
+ }
+#if MLIR_AGX_TARGET_ENABLED
+ AGXSerializer serializer(*module, cast<AGXIRTargetAttr>(attribute),
+ options);
+ AGXSerializer::init();
+ std::optional<SmallVector<char, 0>> binary = serializer.run();
+ if (!binary)
+ return std::nullopt;
+ return gpu::SerializedObject{std::move(*binary)};
+#else
+ module->emitError("the `AGX` target was not built. Please enable it when "
+ "building LLVM (LLVM_EXPERIMENTAL_TARGETS_TO_BUILD=AGX)");
+ return std::nullopt;
+#endif // MLIR_AGX_TARGET_ENABLED
+}
+
+Attribute
+AGXIRTargetAttrImpl::createObject(Attribute attribute, Operation *module,
+ const gpu::SerializedObject &object,
+ const gpu::TargetOptions &options) const {
+ gpu::CompilationTarget format = options.getCompilationTarget();
+ // There is no fat binary format: use the ELF object.
+ if (format > gpu::CompilationTarget::Binary)
+ format = gpu::CompilationTarget::Binary;
+ Builder builder(attribute.getContext());
+ return builder.getAttr<gpu::ObjectAttr>(
+ attribute, format,
+ builder.getStringAttr(
+ StringRef(object.getObject().data(), object.getObject().size())),
+ DictionaryAttr{}, gpu::KernelTableAttr{});
+}
diff --git a/mlir/lib/Target/LLVM/CMakeLists.txt b/mlir/lib/Target/LLVM/CMakeLists.txt
index d59138cd9db23..ce24243f179a7 100644
--- a/mlir/lib/Target/LLVM/CMakeLists.txt
+++ b/mlir/lib/Target/LLVM/CMakeLists.txt
@@ -160,6 +160,40 @@ else()
endif()
+if ("AGX" IN_LIST LLVM_TARGETS_TO_BUILD)
+ set(AGX_LIBS
+ AGXAsmParser
+ AGXCodeGen
+ AGXDesc
+ AGXInfo
+ )
+endif()
+
+add_mlir_dialect_library(MLIRAGXIRTarget
+ AGXIR/Target.cpp
+
+ OBJECT
+
+ LINK_COMPONENTS
+ MCParser
+ ${AGX_LIBS}
+
+ LINK_LIBS PUBLIC
+ MLIRIR
+ MLIRExecutionEngineUtils
+ MLIRSupport
+ MLIRGPUDialect
+ MLIRTargetLLVM
+ MLIRAGXIRToLLVMIRTranslation
+ )
+
+if ("AGX" IN_LIST LLVM_TARGETS_TO_BUILD)
+ target_compile_definitions(obj.MLIRAGXIRTarget
+ PRIVATE
+ MLIR_AGX_TARGET_ENABLED=1
+ )
+endif()
+
if ("AMDGPU" IN_LIST LLVM_TARGETS_TO_BUILD)
set(AMDGPU_LIBS
AMDGPUAsmParser
diff --git a/mlir/lib/Target/LLVMIR/CMakeLists.txt b/mlir/lib/Target/LLVMIR/CMakeLists.txt
index ed775bc47d7bf..6843d0c601f5e 100644
--- a/mlir/lib/Target/LLVMIR/CMakeLists.txt
+++ b/mlir/lib/Target/LLVMIR/CMakeLists.txt
@@ -50,6 +50,7 @@ add_mlir_translation_library(MLIRToLLVMIRTranslationRegistration
ConvertToLLVMIR.cpp
LINK_LIBS PUBLIC
+ MLIRAGXIRToLLVMIRTranslation
MLIRArmNeonToLLVMIRTranslation
MLIRArmSMEToLLVMIRTranslation
MLIRArmSVEToLLVMIRTranslation
diff --git a/mlir/lib/Target/LLVMIR/Dialect/AGXIR/AGXIRToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/AGXIR/AGXIRToLLVMIRTranslation.cpp
new file mode 100644
index 0000000000000..b9219c3c82606
--- /dev/null
+++ b/mlir/lib/Target/LLVMIR/Dialect/AGXIR/AGXIRToLLVMIRTranslation.cpp
@@ -0,0 +1,74 @@
+//===- AGXIRToLLVMIRTranslation.cpp - Translate AGXIR to LLVM IR ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a translation between the MLIR AGXIR dialect and
+// LLVM IR.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Target/LLVMIR/Dialect/AGXIR/AGXIRToLLVMIRTranslation.h"
+#include "mlir/Dialect/LLVMIR/AGXIRDialect.h"
+#include "mlir/IR/Operation.h"
+#include "mlir/Target/LLVMIR/ModuleTranslation.h"
+
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/IntrinsicsAGX.h"
+
+using namespace mlir;
+using namespace mlir::LLVM;
+using mlir::LLVM::detail::createIntrinsicCall;
+
+namespace {
+class AGXIRDialectLLVMIRTranslationInterface
+ : public LLVMTranslationDialectInterface {
+public:
+ using LLVMTranslationDialectInterface::LLVMTranslationDialectInterface;
+
+ /// Translates the given operation to LLVM IR using the provided IR builder
+ /// and saving the state in `moduleTranslation`.
+ LogicalResult
+ convertOperation(Operation *op, llvm::IRBuilderBase &builder,
+ LLVM::ModuleTranslation &moduleTranslation) const final {
+ Operation &opInst = *op;
+#include "mlir/Dialect/LLVMIR/AGXIRConversions.inc"
+
+ return failure();
+ }
+
+ /// Functions marked as kernels become agx_kernel entry points.
+ LogicalResult
+ amendOperation(Operation *op, ArrayRef<llvm::Instruction *> instructions,
+ NamedAttribute attribute,
+ LLVM::ModuleTranslation &moduleTranslation) const final {
+ auto *dialect = dyn_cast<AGXIR::AGXIRDialect>(attribute.getNameDialect());
+ if (dialect->getKernelAttrHelper().getName() == attribute.getName()) {
+ auto func = dyn_cast<LLVM::LLVMFuncOp>(op);
+ if (!func)
+ return op->emitOpError(Twine(attribute.getName()) +
+ " is only supported on `llvm.func` operations");
+ llvm::Function *llvmFunc =
+ moduleTranslation.lookupFunction(func.getName());
+ llvmFunc->setCallingConv(llvm::CallingConv::AGX_KERNEL);
+ }
+ return success();
+ }
+};
+} // namespace
+
+void mlir::registerAGXIRDialectTranslation(DialectRegistry ®istry) {
+ registry.insert<AGXIR::AGXIRDialect>();
+ registry.addExtension(+[](MLIRContext *ctx, AGXIR::AGXIRDialect *dialect) {
+ dialect->addInterfaces<AGXIRDialectLLVMIRTranslationInterface>();
+ });
+}
+
+void mlir::registerAGXIRDialectTranslation(MLIRContext &context) {
+ DialectRegistry registry;
+ registerAGXIRDialectTranslation(registry);
+ context.appendDialectRegistry(registry);
+}
diff --git a/mlir/lib/Target/LLVMIR/Dialect/AGXIR/CMakeLists.txt b/mlir/lib/Target/LLVMIR/Dialect/AGXIR/CMakeLists.txt
new file mode 100644
index 0000000000000..82bd2106f6f9f
--- /dev/null
+++ b/mlir/lib/Target/LLVMIR/Dialect/AGXIR/CMakeLists.txt
@@ -0,0 +1,16 @@
+add_mlir_translation_library(MLIRAGXIRToLLVMIRTranslation
+ AGXIRToLLVMIRTranslation.cpp
+
+ DEPENDS
+ MLIRAGXIROpsIncGen
+
+ LINK_COMPONENTS
+ Core
+
+ LINK_LIBS PUBLIC
+ MLIRIR
+ MLIRAGXIRDialect
+ MLIRLLVMDialect
+ MLIRSupport
+ MLIRTargetLLVMIRExport
+ )
diff --git a/mlir/lib/Target/LLVMIR/Dialect/CMakeLists.txt b/mlir/lib/Target/LLVMIR/Dialect/CMakeLists.txt
index d590b236cfc91..ed20a4f2e917b 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/CMakeLists.txt
+++ b/mlir/lib/Target/LLVMIR/Dialect/CMakeLists.txt
@@ -1,3 +1,4 @@
+add_subdirectory(AGXIR)
add_subdirectory(ArmNeon)
add_subdirectory(ArmSME)
add_subdirectory(ArmSVE)
diff --git a/mlir/test/Conversion/AGXToAGXIR/agx-to-agxir.mlir b/mlir/test/Conversion/AGXToAGXIR/agx-to-agxir.mlir
new file mode 100644
index 0000000000000..993d91d7c78a9
--- /dev/null
+++ b/mlir/test/Conversion/AGXToAGXIR/agx-to-agxir.mlir
@@ -0,0 +1,56 @@
+// RUN: mlir-opt %s -convert-agx-to-agxir | FileCheck %s
+
+// CHECK-LABEL: @barrier
+func.func @barrier() {
+ // CHECK: llvm.fence syncscope("workgroup") release
+ // CHECK-NEXT: agxir.threadgroup.barrier
+ // CHECK-NEXT: llvm.fence syncscope("workgroup") acquire
+ agx.threadgroup_barrier
+ return
+}
+
+// CHECK-LABEL: @shuffle_scalar
+func.func @shuffle_scalar(%f: f32, %h: f16, %i: i32) -> (f32, f16) {
+ // CHECK: agxir.simd.shuffle.xor %{{.*}}, %{{.*}} : f32
+ %0 = agx.shuffle xor %f, %i : f32
+ // CHECK: agxir.quad.shuffle %{{.*}}, %{{.*}} : f16
+ %1 = agx.shuffle idx %h, %i scope quad : f16
+ return %0, %1 : f32, f16
+}
+
+// CHECK-LABEL: @shuffle_wide
+func.func @shuffle_wide(%d: f64, %i: i32) -> f64 {
+ // 64-bit values are shuffled in two 32-bit parts.
+ // CHECK: agxir.simd.shuffle.down %{{.*}}, %{{.*}} : i32
+ // CHECK: agxir.simd.shuffle.down %{{.*}}, %{{.*}} : i32
+ // CHECK-NOT: agxir.simd.shuffle
+ %0 = agx.shuffle down %d, %i : f64
+ return %0 : f64
+}
+
+// CHECK-LABEL: @reduce
+func.func @reduce(%f: f32, %i: i32, %s: i16, %h: f16) -> (f32, i32, i16, f16) {
+ // CHECK: agxir.simd.reduce fadd %{{.*}} : f32
+ %0 = agx.reduce add %f : f32
+ // CHECK: agxir.quad.prefix umax %{{.*}} : i32
+ %1 = agx.reduce maxui %i prefix scope quad : i32
+ // 16-bit values are extended according to the signedness.
+ // CHECK: llvm.sext %{{.*}} : i16 to i32
+ // CHECK: agxir.simd.reduce smin %{{.*}} : i32
+ // CHECK: llvm.trunc %{{.*}} : i32 to i16
+ %2 = agx.reduce minsi %s : i16
+ // CHECK: llvm.fpext %{{.*}} : f16 to f32
+ // CHECK: agxir.simd.reduce fmax %{{.*}} : f32
+ // CHECK: llvm.fptrunc %{{.*}} : f32 to f16
+ %3 = agx.reduce maxnumf %h : f16
+ return %0, %1, %2, %3 : f32, i32, i16, f16
+}
+
+// CHECK-LABEL: @ballot
+func.func @ballot(%p: i1) -> (i32, i32) {
+ // CHECK: agxir.simd.ballot %{{.*}} : i32
+ %0 = agx.ballot %p : i32
+ // CHECK: agxir.quad.ballot %{{.*}} : i32
+ %1 = agx.ballot %p scope quad : i32
+ return %0, %1 : i32, i32
+}
diff --git a/mlir/test/Conversion/GPUToAGXIR/gpu-to-agxir.mlir b/mlir/test/Conversion/GPUToAGXIR/gpu-to-agxir.mlir
new file mode 100644
index 0000000000000..db988d3381b6e
--- /dev/null
+++ b/mlir/test/Conversion/GPUToAGXIR/gpu-to-agxir.mlir
@@ -0,0 +1,86 @@
+// RUN: mlir-opt %s -convert-gpu-to-agxir -split-input-file | FileCheck %s
+// RUN: mlir-opt %s -convert-gpu-to-agxir='index-bitwidth=32' -split-input-file | FileCheck --check-prefix=CHECK32 %s
+
+gpu.module @test_module {
+ // CHECK-LABEL: llvm.func @indices
+ // CHECK32-LABEL: llvm.func @indices
+ func.func @indices() -> (index, index, index, index, index, index, index) {
+ // CHECK: agxir.thread.position.in.threadgroup.x : i32
+ // CHECK: llvm.sext %{{.*}} : i32 to i64
+ // CHECK32: agxir.thread.position.in.threadgroup.x : i32
+ // CHECK32-NOT: llvm.sext
+ %tid = gpu.thread_id x
+ // CHECK: agxir.threadgroup.position.in.grid.y
+ %bid = gpu.block_id y
+ // CHECK: agxir.threads.per.threadgroup.z
+ %bdim = gpu.block_dim z
+ // CHECK: agxir.threadgroups.per.grid.x
+ %gdim = gpu.grid_dim x
+ // CHECK: agxir.threadgroup.position.in.grid.x
+ // CHECK: agxir.threads.per.threadgroup.x
+ // CHECK: llvm.mul
+ // CHECK: agxir.thread.position.in.threadgroup.x
+ // CHECK: llvm.add
+ %gid = gpu.global_id x
+ // CHECK: agxir.thread.index.in.simdgroup
+ %lane = gpu.lane_id
+ // CHECK: llvm.mlir.constant(32 : i64) : i64
+ %size = gpu.subgroup_size : index
+ func.return %tid, %bid, %bdim, %gdim, %gid, %lane, %size
+ : index, index, index, index, index, index, index
+ }
+}
+
+// -----
+
+gpu.module @test_module {
+ // CHECK-LABEL: llvm.func @barrier
+ func.func @barrier() {
+ // CHECK: llvm.fence syncscope("workgroup") release
+ // CHECK-NEXT: agxir.threadgroup.barrier
+ // CHECK-NEXT: llvm.fence syncscope("workgroup") acquire
+ gpu.barrier
+ func.return
+ }
+}
+
+// -----
+
+gpu.module @test_module {
+ // CHECK-LABEL: llvm.func @shuffle
+ func.func @shuffle(%v: f32, %o: i32, %w: i32) -> (f32, i1) {
+ // CHECK: %[[LANE:.*]] = agxir.thread.index.in.simdgroup : i32
+ // CHECK: %[[SRC:.*]] = llvm.xor %[[LANE]], %{{.*}} : i32
+ // CHECK: %[[VALID:.*]] = llvm.icmp "slt" %[[SRC]], %{{.*}} : i32
+ // CHECK: %[[SEL:.*]] = llvm.select %[[VALID]], %[[SRC]], %[[LANE]] : i1, i32
+ // CHECK: agxir.simd.shuffle %{{.*}}, %[[SEL]] : i32
+ %r, %valid = gpu.shuffle xor %v, %o, %w : f32
+ func.return %r, %valid : f32, i1
+ }
+
+ // CHECK-LABEL: llvm.func @ballot_reduce
+ func.func @ballot_reduce(%p: i1, %x: f32) -> (i32, f32) {
+ // CHECK: agxir.simd.ballot %{{.*}} : i32
+ %b = gpu.ballot %p : i32
+ // CHECK: agxir.simd.reduce fadd %{{.*}} : f32
+ %r = gpu.subgroup_reduce add %x : (f32) -> f32
+ func.return %b, %r : i32, f32
+ }
+}
+
+// -----
+
+gpu.module @test_module {
+ // Kernels use the agx_kernel calling convention; workgroup attributions
+ // are threadgroup memory.
+ // CHECK: llvm.mlir.global internal @__wg_kernel_0() {addr_space = 3 : i32} : !llvm.array<32 x f32>
+ // CHECK-LABEL: llvm.func agx_kernelcc @kernel
+ // CHECK-SAME: agxir.kernel
+ gpu.func @kernel(%arg0: memref<32xf32, #gpu.address_space<global>>)
+ workgroup(%tile: memref<32xf32, #gpu.address_space<workgroup>>) kernel {
+ %c0 = arith.constant 0 : index
+ %v = memref.load %arg0[%c0] : memref<32xf32, #gpu.address_space<global>>
+ memref.store %v, %tile[%c0] : memref<32xf32, #gpu.address_space<workgroup>>
+ gpu.return
+ }
+}
diff --git a/mlir/test/Dialect/AGX/invalid.mlir b/mlir/test/Dialect/AGX/invalid.mlir
new file mode 100644
index 0000000000000..a8dff62adc702
--- /dev/null
+++ b/mlir/test/Dialect/AGX/invalid.mlir
@@ -0,0 +1,31 @@
+// RUN: mlir-opt %s -split-input-file -verify-diagnostics
+
+func.func @shuffle_i8(%v: i8, %i: i32) -> i8 {
+ // expected-error @below {{requires a 16-bit value or a multiple of 32 bits}}
+ %0 = agx.shuffle xor %v, %i : i8
+ return %0 : i8
+}
+
+// -----
+
+func.func @reduce_int_mul(%v: i32) -> i32 {
+ // expected-error @below {{there is no integer multiplication reduction}}
+ %0 = agx.reduce mul %v : i32
+ return %0 : i32
+}
+
+// -----
+
+func.func @reduce_float_and(%v: f32) -> f32 {
+ // expected-error @below {{reduction 'and' does not apply to floating point values}}
+ %0 = agx.reduce and %v : f32
+ return %0 : f32
+}
+
+// -----
+
+func.func @reduce_int_minnumf(%v: i32) -> i32 {
+ // expected-error @below {{reduction 'minnumf' requires floating point values}}
+ %0 = agx.reduce minnumf %v : i32
+ return %0 : i32
+}
diff --git a/mlir/test/Dialect/AGX/ops.mlir b/mlir/test/Dialect/AGX/ops.mlir
new file mode 100644
index 0000000000000..ff783d03903f9
--- /dev/null
+++ b/mlir/test/Dialect/AGX/ops.mlir
@@ -0,0 +1,37 @@
+// RUN: mlir-opt %s | mlir-opt | FileCheck %s
+// RUN: mlir-opt %s --mlir-print-op-generic | mlir-opt | FileCheck %s
+
+// CHECK-LABEL: @shuffles
+func.func @shuffles(%f: f32, %v: vector<2xf32>, %h: f16, %d: f64, %i: i32) {
+ // CHECK: agx.shuffle xor %{{.*}}, %{{.*}} : f32
+ %0 = agx.shuffle xor %f, %i : f32
+ // CHECK: agx.shuffle down %{{.*}}, %{{.*}} scope quad : vector<2xf32>
+ %1 = agx.shuffle down %v, %i scope quad : vector<2xf32>
+ // CHECK: agx.shuffle idx %{{.*}}, %{{.*}} : f16
+ %2 = agx.shuffle idx %h, %i : f16
+ // CHECK: agx.shuffle up %{{.*}}, %{{.*}} : f64
+ %3 = agx.shuffle up %d, %i : f64
+ return
+}
+
+// CHECK-LABEL: @reductions
+func.func @reductions(%f: f32, %h: f16, %i: i32, %s: i16) {
+ // CHECK: agx.reduce add %{{.*}} : f32
+ %0 = agx.reduce add %f : f32
+ // CHECK: agx.reduce maxnumf %{{.*}} scope quad : f16
+ %1 = agx.reduce maxnumf %h scope quad : f16
+ // CHECK: agx.reduce add %{{.*}} prefix : i32
+ %2 = agx.reduce add %i prefix : i32
+ // CHECK: agx.reduce minsi %{{.*}} : i16
+ %3 = agx.reduce minsi %s : i16
+ return
+}
+
+// CHECK-LABEL: @misc
+func.func @misc(%p: i1) -> i32 {
+ // CHECK: agx.threadgroup_barrier
+ agx.threadgroup_barrier
+ // CHECK: agx.ballot %{{.*}} scope quad : i32
+ %0 = agx.ballot %p scope quad : i32
+ return %0 : i32
+}
diff --git a/mlir/test/Dialect/GPU/agxir-attach-target.mlir b/mlir/test/Dialect/GPU/agxir-attach-target.mlir
new file mode 100644
index 0000000000000..4faf588adab38
--- /dev/null
+++ b/mlir/test/Dialect/GPU/agxir-attach-target.mlir
@@ -0,0 +1,10 @@
+// RUN: mlir-opt %s --agxir-attach-target='module=agx.* chip=agx-g13c O=3' | FileCheck %s
+
+module attributes {gpu.container_module} {
+ // CHECK: @agx_module [#agxir.target<O = 3, chip = "agx-g13c">]
+ gpu.module @agx_module {
+ }
+ // CHECK: @other_module {
+ gpu.module @other_module {
+ }
+}
diff --git a/mlir/test/Dialect/GPU/module-to-binary-agxir.mlir b/mlir/test/Dialect/GPU/module-to-binary-agxir.mlir
new file mode 100644
index 0000000000000..d04e730fae2b0
--- /dev/null
+++ b/mlir/test/Dialect/GPU/module-to-binary-agxir.mlir
@@ -0,0 +1,19 @@
+// REQUIRES: host-supports-agx
+// RUN: mlir-opt %s --gpu-module-to-binary="format=llvm" | FileCheck %s
+// RUN: mlir-opt %s --gpu-module-to-binary="format=isa" | FileCheck %s -check-prefix=CHECK-ISA
+// RUN: mlir-opt %s --gpu-module-to-binary="format=bin" | FileCheck %s -check-prefix=CHECK-BIN
+
+module attributes {gpu.container_module} {
+ // CHECK-LABEL: gpu.binary @kernel_module
+ // CHECK: [#gpu.object<#agxir.target<chip = "agx-g13g">, offload = "{{.*}}">]
+ // CHECK-ISA: [#gpu.object<#agxir.target<chip = "agx-g13g">, assembly = "{{.*}}device_store{{.*}}stop{{.*}}">]
+ // CHECK-BIN: [#gpu.object<#agxir.target<chip = "agx-g13g">, bin = "{{.*}}ELF{{.*}}">]
+ gpu.module @kernel_module [#agxir.target<chip = "agx-g13g">] {
+ llvm.func agx_kernelcc @kernel(%out: !llvm.ptr<1>, %v: f32) attributes {agxir.kernel} {
+ %id = agxir.thread.position.in.grid.x : i32
+ %p = llvm.getelementptr %out[%id] : (!llvm.ptr<1>, i32) -> !llvm.ptr<1>, f32
+ llvm.store %v, %p : f32, !llvm.ptr<1>
+ llvm.return
+ }
+ }
+}
diff --git a/mlir/test/Dialect/LLVMIR/agxir.mlir b/mlir/test/Dialect/LLVMIR/agxir.mlir
new file mode 100644
index 0000000000000..64d93a9d3b247
--- /dev/null
+++ b/mlir/test/Dialect/LLVMIR/agxir.mlir
@@ -0,0 +1,72 @@
+// RUN: mlir-opt %s -split-input-file -verify-diagnostics | mlir-opt | FileCheck %s
+
+// CHECK-LABEL: @special_registers
+func.func @special_registers() -> i32 {
+ // CHECK: agxir.thread.position.in.grid.x : i32
+ %0 = agxir.thread.position.in.grid.x : i32
+ // CHECK: agxir.thread.position.in.threadgroup.y range <i32, 0, 1024> : i32
+ %1 = agxir.thread.position.in.threadgroup.y range <i32, 0, 1024> : i32
+ // CHECK: agxir.threadgroup.position.in.grid.z : i32
+ %2 = agxir.threadgroup.position.in.grid.z : i32
+ // CHECK: agxir.threads.per.threadgroup.x : i32
+ %3 = agxir.threads.per.threadgroup.x : i32
+ // CHECK: agxir.threadgroups.per.grid.y : i32
+ %4 = agxir.threadgroups.per.grid.y : i32
+ // CHECK: agxir.thread.index.in.simdgroup : i32
+ %5 = agxir.thread.index.in.simdgroup : i32
+ // CHECK: agxir.simdgroup.index.in.threadgroup : i32
+ %6 = agxir.simdgroup.index.in.threadgroup : i32
+ // CHECK: agxir.total.active.threads.in.simdgroup : i32
+ %7 = agxir.total.active.threads.in.simdgroup : i32
+ llvm.return %0 : i32
+}
+
+// CHECK-LABEL: @sync
+func.func @sync() {
+ // CHECK: agxir.threadgroup.barrier
+ agxir.threadgroup.barrier
+ // CHECK: agxir.memory.barrier 1, 2, 9
+ agxir.memory.barrier 1, 2, 9
+ // CHECK: agxir.wait 0
+ agxir.wait 0
+ llvm.return
+}
+
+// CHECK-LABEL: @simdgroup
+func.func @simdgroup(%v: f32, %h: f16, %i: i32, %p: i1) -> f32 {
+ // CHECK: agxir.simd.ballot %{{.*}} : i32
+ %0 = agxir.simd.ballot %p : i32
+ // CHECK: agxir.quad.ballot %{{.*}} : i32
+ %1 = agxir.quad.ballot %p : i32
+ // CHECK: agxir.simd.shuffle %{{.*}}, %{{.*}} : f32
+ %2 = agxir.simd.shuffle %v, %i : f32
+ // CHECK: agxir.simd.shuffle.xor %{{.*}}, %{{.*}} : f16
+ %3 = agxir.simd.shuffle.xor %h, %i : f16
+ // CHECK: agxir.quad.shuffle.down %{{.*}}, %{{.*}} : i32
+ %4 = agxir.quad.shuffle.down %i, %i : i32
+ // CHECK: agxir.simd.reduce fadd %{{.*}} : f32
+ %5 = agxir.simd.reduce fadd %v : f32
+ // CHECK: agxir.quad.prefix iadd %{{.*}} : i32
+ %6 = agxir.quad.prefix iadd %i : i32
+ // CHECK: agxir.simd.prefix umax %{{.*}} : i32
+ %7 = agxir.simd.prefix umax %i : i32
+ // CHECK: agxir.rcp %{{.*}} : f32
+ %8 = agxir.rcp %v : f32
+ // CHECK: agxir.rsqrt %{{.*}} : f16
+ %9 = agxir.rsqrt %h : f16
+ // CHECK: agxir.dfdx %{{.*}} : f32
+ %10 = agxir.dfdx %v : f32
+ llvm.return %2 : f32
+}
+
+// CHECK: gpu.module @targets [#agxir.target, #agxir.target<O = 3, chip = "agx-g13c">]
+gpu.module @targets [#agxir.target, #agxir.target<O = 3, chip = "agx-g13c">] {
+}
+
+// -----
+
+func.func @bad_reduce(%v: f32) -> f32 {
+ // expected-error @below {{reduction 'iadd' does not apply to 'f32'}}
+ %0 = agxir.simd.reduce iadd %v : f32
+ llvm.return %0 : f32
+}
diff --git a/mlir/test/Dialect/LLVMIR/func.mlir b/mlir/test/Dialect/LLVMIR/func.mlir
index f243200e7ca13..493af127cea7f 100644
--- a/mlir/test/Dialect/LLVMIR/func.mlir
+++ b/mlir/test/Dialect/LLVMIR/func.mlir
@@ -536,7 +536,7 @@ module {
module {
"llvm.func"() ({
- // expected-error @below {{expected one of [ccc, fastcc, coldcc, cc_10, cc_11, anyregcc, preserve_mostcc, preserve_allcc, swiftcc, cxx_fast_tlscc, tailcc, cfguard_checkcc, swifttailcc, x86_stdcallcc, x86_fastcallcc, arm_apcscc, arm_aapcscc, arm_aapcs_vfpcc, msp430_intrcc, x86_thiscallcc, ptx_kernelcc, ptx_devicecc, spir_funccc, spir_kernelcc, intel_ocl_bicc, x86_64_sysvcc, win64cc, x86_vectorcallcc, hhvmcc, hhvm_ccc, x86_intrcc, avr_intrcc, avr_builtincc, amdgpu_vscc, amdgpu_gscc, amdgpu_cscc, amdgpu_kernelcc, x86_regcallcc, amdgpu_hscc, msp430_builtincc, amdgpu_lscc, amdgpu_escc, aarch64_vectorcallcc, aarch64_sve_vectorcallcc, wasm_emscripten_invokecc, amdgpu_gfxcc, m68k_intrcc] for Calling Conventions, got: cc_12}}
+ // expected-error @below {{expected one of [ccc, fastcc, coldcc, cc_10, cc_11, anyregcc, preserve_mostcc, preserve_allcc, swiftcc, cxx_fast_tlscc, tailcc, cfguard_checkcc, swifttailcc, x86_stdcallcc, x86_fastcallcc, arm_apcscc, arm_aapcscc, arm_aapcs_vfpcc, msp430_intrcc, x86_thiscallcc, ptx_kernelcc, ptx_devicecc, spir_funccc, spir_kernelcc, intel_ocl_bicc, x86_64_sysvcc, win64cc, x86_vectorcallcc, hhvmcc, hhvm_ccc, x86_intrcc, avr_intrcc, avr_builtincc, amdgpu_vscc, amdgpu_gscc, amdgpu_cscc, amdgpu_kernelcc, x86_regcallcc, amdgpu_hscc, msp430_builtincc, amdgpu_lscc, amdgpu_escc, aarch64_vectorcallcc, aarch64_sve_vectorcallcc, wasm_emscripten_invokecc, amdgpu_gfxcc, m68k_intrcc, agx_kernelcc] for Calling Conventions, got: cc_12}}
// expected-error @below {{failed to parse CConvAttr parameter 'CallingConv' which is to be a `CConv`}}
}) {sym_name = "generic_unknown_calling_convention", CConv = #llvm.cconv<cc_12>, function_type = !llvm.func<i64 (i64, i64)>} : () -> ()
}
diff --git a/mlir/test/Target/LLVMIR/agxir.mlir b/mlir/test/Target/LLVMIR/agxir.mlir
new file mode 100644
index 0000000000000..232f855fc16cd
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/agxir.mlir
@@ -0,0 +1,37 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// CHECK: define agx_kernel void @kernel(ptr addrspace(1) %0)
+llvm.func @kernel(%out: !llvm.ptr<1>) attributes {agxir.kernel} {
+ // CHECK: call i32 @llvm.agx.thread.position.in.grid.x()
+ %0 = agxir.thread.position.in.grid.x : i32
+ // CHECK: call range(i32 0, 64) i32 @llvm.agx.thread.position.in.threadgroup.y()
+ %1 = agxir.thread.position.in.threadgroup.y range <i32, 0, 64> : i32
+ // CHECK: call i32 @llvm.agx.threadgroups.per.grid.z()
+ %2 = agxir.threadgroups.per.grid.z : i32
+ // CHECK: call i32 @llvm.agx.thread.index.in.simdgroup()
+ %3 = agxir.thread.index.in.simdgroup : i32
+ // CHECK: call void @llvm.agx.threadgroup.barrier()
+ agxir.threadgroup.barrier
+ // CHECK: call void @llvm.agx.memory.barrier(i32 1, i32 2, i32 9)
+ agxir.memory.barrier 1, 2, 9
+ %f = llvm.bitcast %0 : i32 to f32
+ // CHECK: call float @llvm.agx.simd.shuffle.xor.f32(float %{{.*}}, i32 %{{.*}})
+ %4 = agxir.simd.shuffle.xor %f, %3 : f32
+ // CHECK: call float @llvm.agx.simd.reduce.f32(float %{{.*}}, i32 1)
+ %5 = agxir.simd.reduce fadd %4 : f32
+ // CHECK: call i32 @llvm.agx.quad.prefix.i32(i32 %{{.*}}, i32 30)
+ %6 = agxir.quad.prefix umax %1 : i32
+ %p = llvm.icmp "ugt" %6, %2 : i32
+ // CHECK: call i32 @llvm.agx.simd.ballot(i1 %{{.*}})
+ %7 = agxir.simd.ballot %p : i32
+ // CHECK: call float @llvm.agx.rsqrt.f32(float %{{.*}})
+ %8 = agxir.rsqrt %5 : f32
+ llvm.store %8, %out : f32, !llvm.ptr<1>
+ llvm.return
+}
+
+// Functions without the attribute keep their calling convention.
+// CHECK: define void @helper()
+llvm.func @helper() {
+ llvm.return
+}
diff --git a/mlir/test/lit.cfg.py b/mlir/test/lit.cfg.py
index dbd18f8e8b0f2..5239278887337 100644
--- a/mlir/test/lit.cfg.py
+++ b/mlir/test/lit.cfg.py
@@ -398,6 +398,9 @@ def have_host_jit_feature_support(feature_name):
if config.run_rocm_tests:
config.available_features.add("host-supports-amdgpu")
+if config.run_agx_tests:
+ config.available_features.add("host-supports-agx")
+
if config.arm_emulator_executable:
config.available_features.add("arm-emulator")
diff --git a/mlir/test/lit.site.cfg.py.in b/mlir/test/lit.site.cfg.py.in
index 76dab98e2b51e..1e50ef90012ab 100644
--- a/mlir/test/lit.site.cfg.py.in
+++ b/mlir/test/lit.site.cfg.py.in
@@ -36,6 +36,8 @@ config.build_examples = @LLVM_BUILD_EXAMPLES@
config.run_nvptx_tests = @LLVM_HAS_NVPTX_TARGET@
config.enable_cuda_runner = @MLIR_ENABLE_CUDA_RUNNER@
config.run_rocm_tests = @LLVM_HAS_AMDGPU_TARGET@
+# AGX is an experimental target: the variable is unset unless it is built.
+config.run_agx_tests = "@LLVM_HAS_AGX_TARGET@" == "1"
config.enable_rocm_runner = @MLIR_ENABLE_ROCM_RUNNER@
config.gpu_compilation_format = "@MLIR_GPU_COMPILATION_TEST_FORMAT@"
config.rocm_test_chipset = "@ROCM_TEST_CHIPSET@"
>From 9a317577bdc71d545acce7613943df2178220560 Mon Sep 17 00:00:00 2001
From: Claude <noreply at anthropic.com>
Date: Sun, 18 Oct 2026 21:34:12 +0000
Subject: [PATCH 08/22] [AGX][docs] Document the MLIR agx and agxir dialects
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
Claude-Session: https://claude.ai/code/session_01E56KEecg6edNaS23myRWER
---
llvm/docs/AGXUsage.rst | 21 +++++++++++++++++++++
1 file changed, 21 insertions(+)
diff --git a/llvm/docs/AGXUsage.rst b/llvm/docs/AGXUsage.rst
index c470981f929ad..c1f03831df644 100644
--- a/llvm/docs/AGXUsage.rst
+++ b/llvm/docs/AGXUsage.rst
@@ -279,3 +279,24 @@ number, and ``llvm.agx.simd.reduce``/``prefix`` and
5 fmin
7 fmax
===== ========= ===== =========
+
+MLIR
+====
+
+MLIR targets AGX the way it targets AMDGPU:
+
+* The ``agxir`` dialect wraps the ``llvm.agx.*`` intrinsics one to one, like
+ ``rocdl``. Kernels carry the ``agxir.kernel`` attribute and use the
+ ``agx_kernelcc`` calling convention.
+* The ``agx`` dialect provides higher level operations, like ``amdgpu``:
+ ``agx.shuffle`` (any type, SIMD-group or quad scope), ``agx.reduce``,
+ ``agx.ballot`` and ``agx.threadgroup_barrier``. ``-convert-agx-to-agxir``
+ lowers them.
+* ``-convert-gpu-to-agxir`` lowers the ``gpu`` dialect, and
+ ``-agxir-attach-target`` attaches ``#agxir.target`` attributes.
+ ``-gpu-module-to-binary`` then produces AGX assembly, ELF objects or
+ bitcode.
+
+.. code-block:: console
+
+ $ mlir-opt kernel.mlir --pass-pipeline='builtin.module(agxir-attach-target{chip=agx-g14g},gpu.module(convert-gpu-to-agxir),gpu-module-to-binary{format=isa})'
>From 848fe48fe7a2aded4ba796f2c6e2cbd48c7bc5f7 Mon Sep 17 00:00:00 2001
From: Claude <noreply at anthropic.com>
Date: Sun, 18 Oct 2026 22:08:26 +0000
Subject: [PATCH 09/22] [AGX] Emit typed, sized symbols for the global data
buffer
The variables in .agx.globals were plain local labels, so a runtime could
not find a variable's offset by name (for example for cuModuleGetGlobal).
Emit them with their linkage, visibility, @object type and size.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
Claude-Session: https://claude.ai/code/session_01E56KEecg6edNaS23myRWER
---
llvm/lib/Target/AGX/AGXAsmPrinter.cpp | 10 +++++++++-
llvm/test/CodeGen/AGX/global-data.ll | 6 ++++++
2 files changed, 15 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AGX/AGXAsmPrinter.cpp b/llvm/lib/Target/AGX/AGXAsmPrinter.cpp
index bbf183924bbb8..12a9d156ee817 100644
--- a/llvm/lib/Target/AGX/AGXAsmPrinter.cpp
+++ b/llvm/lib/Target/AGX/AGXAsmPrinter.cpp
@@ -29,6 +29,7 @@
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/Module.h"
#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCExpr.h"
#include "llvm/MC/MCInst.h"
#include "llvm/MC/MCSectionELF.h"
#include "llvm/MC/MCStreamer.h"
@@ -124,8 +125,15 @@ void AGXAsmPrinter::emitEndOfAsmFile(Module &M) {
uint64_t End = 0;
for (const auto &[GV, Offset] : Layout.Offsets) {
OutStreamer->emitZeros(Offset - End);
- OutStreamer->emitLabel(getSymbol(GV));
+ // Typed, sized symbols with the variable's linkage, so that a runtime
+ // can find a variable's offset in the buffer by name.
+ MCSymbol *Sym = getSymbol(GV);
uint64_t Size = DL.getTypeAllocSize(GV->getValueType());
+ emitLinkage(GV, Sym);
+ emitVisibility(Sym, GV->getVisibility(), !GV->isDeclaration());
+ OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeObject);
+ OutStreamer->emitELFSize(Sym, MCConstantExpr::create(Size, OutContext));
+ OutStreamer->emitLabel(Sym);
const Constant *Init = GV->hasInitializer() ? GV->getInitializer() : nullptr;
if (Init && !isa<UndefValue>(Init))
emitGlobalConstant(DL, Init);
diff --git a/llvm/test/CodeGen/AGX/global-data.ll b/llvm/test/CodeGen/AGX/global-data.ll
index bf0d0e430bd06..f8d27a2b600b4 100644
--- a/llvm/test/CodeGen/AGX/global-data.ll
+++ b/llvm/test/CodeGen/AGX/global-data.ll
@@ -50,9 +50,15 @@ define agx_kernel void @kernel(ptr addrspace(1) %out, i32 %i) {
; CHECK-LABEL: .section .agx.globals,"aw", at progbits
; CHECK-NEXT: .p2align 4, 0x0
+; CHECK-NEXT: .globl counter
+; CHECK-NEXT: .type counter, at object
+; CHECK-NEXT: .size counter, 4
; CHECK-NEXT: counter:
; CHECK-NEXT: .long 7
; CHECK-NEXT: .zero 12
+; CHECK-NEXT: .globl table
+; CHECK-NEXT: .type table, at object
+; CHECK-NEXT: .size table, 16
; CHECK-NEXT: table:
; CHECK-NEXT: .long 0x3f800000
; CHECK-NOT: tg
>From 51a831fd638b881e44ebbefa4fd16a3fb6921574 Mon Sep 17 00:00:00 2001
From: Claude <noreply at anthropic.com>
Date: Sun, 18 Oct 2026 22:08:26 +0000
Subject: [PATCH 10/22] [AGX] Lower divergent shuffles, flush half denormals,
diagnose doubles
Found by running the PTX-formalization test programs through ZLUDA and the
functional AGX simulator:
- simd_shuffle and the quad shuffles OR the low two index bits of the
threads of each quad, including inactive threads (applegpu documents
this, Mesa works around it in agx_nir_lower_subgroups and
agx_lower_divergent_shuffle). With quad-shuffle-index-merge, shuffles
with divergent indices are now split into the four quad positions, and
register indices are passed in r0h, which is zero whenever no shuffle
executes. Immediate indices are encoded directly.
- The hardware keeps 16-bit denormals. Functions whose denormal_fpenv
flushes half denormals get explicit flushes of the operands and results
of half arithmetic, comparisons and conversions.
- Double precision arithmetic is diagnosed as unsupported instead of
failing with "no libcall available".
- fcopysign is lowered to bit operations, and selects of f32, f16 and i16
values on a comparison use a single icmpsel/fcmpsel.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
Claude-Session: https://claude.ai/code/session_01E56KEecg6edNaS23myRWER
---
llvm/docs/AGXUsage.rst | 39 ++-
llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp | 289 +++++++++++++++++++-
llvm/lib/Target/AGX/AGXFeatures.td | 4 +-
llvm/lib/Target/AGX/AGXISelLowering.cpp | 27 +-
llvm/lib/Target/AGX/AGXISelLowering.h | 1 +
llvm/lib/Target/AGX/AGXInstructions.td | 44 +++
llvm/lib/Target/AGX/AGXLowerControlFlow.cpp | 57 +++-
llvm/test/CodeGen/AGX/errors-f64.ll | 12 +
llvm/test/CodeGen/AGX/fp.ll | 10 +-
llvm/test/CodeGen/AGX/half-denormals.ll | 32 +++
llvm/test/CodeGen/AGX/shuffle-divergent.ll | 75 +++++
llvm/test/CodeGen/AGX/simd.ll | 11 +-
12 files changed, 578 insertions(+), 23 deletions(-)
create mode 100644 llvm/test/CodeGen/AGX/errors-f64.ll
create mode 100644 llvm/test/CodeGen/AGX/half-denormals.ll
create mode 100644 llvm/test/CodeGen/AGX/shuffle-divergent.ll
diff --git a/llvm/docs/AGXUsage.rst b/llvm/docs/AGXUsage.rst
index c1f03831df644..18fefcdf57605 100644
--- a/llvm/docs/AGXUsage.rst
+++ b/llvm/docs/AGXUsage.rst
@@ -97,8 +97,10 @@ another.
``simd-reduce`` SIMD-group and quad reductions and prefixes.
``simd-ballot`` Ballots.
``simd-matrix`` 8x8 SIMD-group matrix multiply-accumulate.
- ``quad-shuffle-index-merge`` ``simd_shuffle`` merges the low index bits of
- each quad (informational).
+ ``quad-shuffle-index-merge`` Shuffles merge the low index bits of each
+ quad. Divergent shuffle indices are expanded
+ into four shuffles (see `SIMD-group
+ shuffles`_).
``multi-die-coherency`` Device atomics set the coherency bit (multi-die
G13 parts).
``threadgroup-atomics`` Threadgroup memory atomics.
@@ -239,10 +241,14 @@ Supported:
There is no floating point environment: constrained floating point
intrinsics are compiled as the ordinary operations, with round to nearest
-even.
+even. 32-bit operations always flush denormals. 16-bit operations keep them;
+in functions whose ``denormal_fpenv`` flushes ``half`` denormals, the
+operands and results of ``half`` arithmetic, comparisons and conversions are
+flushed explicitly.
-Not supported: ``double``, vector memory operations (vectors are
-scalarized), textures and graphics shaders, calls and recursion.
+Not supported: ``double`` (diagnosed as unsupported), vector memory
+operations (vectors are scalarized), textures and graphics shaders, calls
+and recursion.
.. _agx-apple9:
@@ -280,6 +286,29 @@ number, and ``llvm.agx.simd.reduce``/``prefix`` and
7 fmax
===== ========= ===== =========
+SIMD-group shuffles
+-------------------
+
+``llvm.agx.simd.shuffle(v, i)`` returns ``v`` of thread ``i`` of the
+SIMD-group and ``llvm.agx.quad.shuffle(v, i)`` of thread ``i`` of the quad.
+The ``xor``, ``up`` and ``down`` variants read the thread at the current
+thread index xor, minus or plus ``i``; with ``up`` and ``down``, threads whose
+source is outside the SIMD-group (or quad) keep their own value.
+
+On G13 to G15 (``quad-shuffle-index-merge``) the shuffle instructions read
+the index of all four threads of a quad, including inactive threads, and use
+the OR of their low two bits for the whole quad. The backend therefore:
+
+* expands shuffles whose index is divergent (according to the uniformity
+ analysis) into four shuffles, one per quad position, whose low index bits
+ are the same in every thread, and selects the result of the thread's
+ position; relative shuffles with divergent offsets become shuffles of the
+ computed thread;
+* passes register indices in ``r0h``, which is zeroed at kernel entry and
+ after each shuffle, so that inactive threads contribute zero bits.
+
+Immediate and uniform register indices are used directly.
+
MLIR
====
diff --git a/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp b/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp
index aa29c93cf2a5c..cbc6d6ffd00ae 100644
--- a/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp
@@ -17,20 +17,36 @@
// floating point environment, so constrained floating point intrinsics
// become their ordinary operations. It also unifies the function exits,
// which the control flow structurizer expects.
+// * On subtargets where simd_shuffle merges the low index bits of each quad
+// (quad-shuffle-index-merge), shuffles with divergent indices are
+// expanded into four shuffles whose low index bits are uniform, like
+// Mesa's agx_nir_lower_subgroups.
+// * The hardware flushes 32-bit denormals but keeps 16-bit ones. Functions
+// whose denormal mode for half asks for flushing get explicit flushes of
+// the operands and results of their half operations.
+// * There is no double precision hardware: double precision arithmetic is
+// diagnosed as unsupported.
//
//===----------------------------------------------------------------------===//
#include "AGX.h"
+#include "AGXSubtarget.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/Analysis/UniformityAnalysis.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/Constants.h"
+#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstIterator.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IntrinsicsAGX.h"
#include "llvm/IR/Module.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
+#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/Utils/IntegerDivision.h"
#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
@@ -113,6 +129,9 @@ class AGXCodeGenPrepare : public FunctionPass {
bool expandMemIntrinsics(Function &F, const TargetTransformInfo &TTI);
bool unifyExits(Function &F);
bool lowerConstrainedFP(Function &F);
+ bool lowerDivergentShuffles(Function &F, const UniformityInfo &UI);
+ bool flushHalfDenormals(Function &F);
+ bool diagnoseDoubles(Function &F);
public:
static char ID;
@@ -122,6 +141,8 @@ class AGXCodeGenPrepare : public FunctionPass {
bool runOnFunction(Function &F) override;
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.addRequired<TargetTransformInfoWrapperPass>();
+ AU.addRequired<TargetPassConfig>();
+ AU.addRequired<UniformityInfoWrapperPass>();
}
};
@@ -132,6 +153,8 @@ char AGXCodeGenPrepare::ID = 0;
INITIALIZE_PASS_BEGIN(AGXCodeGenPrepare, DEBUG_TYPE, "AGX CodeGen Prepare",
false, false)
INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
+INITIALIZE_PASS_DEPENDENCY(UniformityInfoWrapperPass)
INITIALIZE_PASS_END(AGXCodeGenPrepare, DEBUG_TYPE, "AGX CodeGen Prepare",
false, false)
@@ -290,12 +313,276 @@ bool AGXCodeGenPrepare::unifyExits(Function &F) {
return true;
}
+/// simd_shuffle reads from each quad the thread at the OR of the low two
+/// index bits of the quad's threads (applegpu documents this, Mesa works
+/// around it). A shuffle whose index is the same in a quad (an immediate, or
+/// low bits that are uniform) reads the thread it names. Divergent indices
+/// are split into the four quad positions:
+///
+/// r = shuffle(v, idx & ~3); r = idx & 3 == k ? shuffle(v, idx & ~3 | k) : r
+///
+/// Quad shuffles take the four immediate positions instead. Relative
+/// shuffles with divergent deltas become shuffles of the computed thread;
+/// threads whose source is outside the SIMD-group (or quad) keep their own
+/// value, as Metal documents for simd_shuffle_up and simd_shuffle_down.
+bool AGXCodeGenPrepare::lowerDivergentShuffles(Function &F,
+ const UniformityInfo &UI) {
+ SmallVector<IntrinsicInst *, 8> Worklist;
+ for (Instruction &I : instructions(F)) {
+ auto *II = dyn_cast<IntrinsicInst>(&I);
+ if (!II)
+ continue;
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::agx_simd_shuffle:
+ case Intrinsic::agx_simd_shuffle_xor:
+ case Intrinsic::agx_simd_shuffle_up:
+ case Intrinsic::agx_simd_shuffle_down:
+ case Intrinsic::agx_quad_shuffle:
+ case Intrinsic::agx_quad_shuffle_xor:
+ case Intrinsic::agx_quad_shuffle_up:
+ case Intrinsic::agx_quad_shuffle_down:
+ if (!isa<Constant>(II->getArgOperand(1)) &&
+ UI.isDivergentAtUse(II->getArgOperandUse(1)))
+ Worklist.push_back(II);
+ break;
+ default:
+ break;
+ }
+ }
+
+ for (IntrinsicInst *II : Worklist) {
+ IRBuilder<> B(II);
+ Value *V = II->getArgOperand(0);
+ Value *Idx = II->getArgOperand(1);
+ Type *Ty = V->getType();
+ Intrinsic::ID ID = II->getIntrinsicID();
+ bool Quad = ID == Intrinsic::agx_quad_shuffle ||
+ ID == Intrinsic::agx_quad_shuffle_xor ||
+ ID == Intrinsic::agx_quad_shuffle_up ||
+ ID == Intrinsic::agx_quad_shuffle_down;
+ unsigned Size = Quad ? 4 : 32;
+
+ // The thread to read, relative to the start of the quad for quad
+ // shuffles.
+ Value *Lane = nullptr;
+ if (ID != Intrinsic::agx_simd_shuffle && ID != Intrinsic::agx_quad_shuffle) {
+ Lane = B.CreateIntrinsic(Intrinsic::agx_thread_index_in_simdgroup, {});
+ if (Quad)
+ Lane = B.CreateAnd(Lane, 3);
+ }
+ switch (ID) {
+ case Intrinsic::agx_simd_shuffle_xor:
+ case Intrinsic::agx_quad_shuffle_xor:
+ Idx = B.CreateAnd(B.CreateXor(Lane, Idx), Size - 1);
+ break;
+ case Intrinsic::agx_simd_shuffle_up:
+ case Intrinsic::agx_quad_shuffle_up:
+ Idx = B.CreateSelect(B.CreateICmpUGE(Lane, Idx), B.CreateSub(Lane, Idx),
+ Lane);
+ break;
+ case Intrinsic::agx_simd_shuffle_down:
+ case Intrinsic::agx_quad_shuffle_down:
+ Idx = B.CreateSelect(
+ B.CreateICmpULT(Idx, B.CreateSub(B.getInt32(Size), Lane)),
+ B.CreateAdd(Lane, Idx), Lane);
+ break;
+ default:
+ break;
+ }
+
+ Intrinsic::ID Shuffle =
+ Quad ? Intrinsic::agx_quad_shuffle : Intrinsic::agx_simd_shuffle;
+ Value *Low = B.CreateAnd(Idx, 3);
+ Value *Base = Quad ? nullptr : B.CreateAnd(Idx, 0x1c);
+ auto Position = [&](unsigned K) -> Value * {
+ if (Quad)
+ return B.getInt32(K);
+ return K ? B.CreateOr(Base, K) : Base;
+ };
+ Value *Result = B.CreateIntrinsic(Shuffle, {Ty}, {V, Position(0)});
+ for (unsigned K = 1; K < 4; ++K) {
+ Value *R = B.CreateIntrinsic(Shuffle, {Ty}, {V, Position(K)});
+ Result = B.CreateSelect(B.CreateICmpEQ(Low, B.getInt32(K)), R, Result);
+ }
+ Result->takeName(II);
+ II->replaceAllUsesWith(Result);
+ II->eraseFromParent();
+ }
+ return !Worklist.empty();
+}
+
+/// Whether I computes a floating point value from its operands (and so
+/// reads and produces denormals according to the denormal mode).
+static bool isFPArithmetic(const Instruction &I) {
+ switch (I.getOpcode()) {
+ case Instruction::FAdd:
+ case Instruction::FSub:
+ case Instruction::FMul:
+ case Instruction::FDiv:
+ case Instruction::FRem:
+ return true;
+ default:
+ break;
+ }
+ const auto *II = dyn_cast<IntrinsicInst>(&I);
+ if (!II)
+ return false;
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::fma:
+ case Intrinsic::fmuladd:
+ case Intrinsic::sqrt:
+ case Intrinsic::minnum:
+ case Intrinsic::maxnum:
+ case Intrinsic::minimum:
+ case Intrinsic::maximum:
+ case Intrinsic::minimumnum:
+ case Intrinsic::maximumnum:
+ case Intrinsic::exp2:
+ case Intrinsic::log2:
+ case Intrinsic::sin:
+ case Intrinsic::cos:
+ case Intrinsic::floor:
+ case Intrinsic::ceil:
+ case Intrinsic::trunc:
+ case Intrinsic::rint:
+ case Intrinsic::nearbyint:
+ case Intrinsic::roundeven:
+ case Intrinsic::canonicalize:
+ case Intrinsic::agx_rcp:
+ case Intrinsic::agx_rsqrt:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static bool isHalf(const Type *Ty) { return Ty->getScalarType()->isHalfTy(); }
+
+bool AGXCodeGenPrepare::flushHalfDenormals(Function &F) {
+ DenormalMode Mode = F.getDenormalMode(APFloat::IEEEhalf());
+ bool FlushInputs = Mode.Input != DenormalMode::IEEE &&
+ Mode.Input != DenormalMode::Dynamic;
+ bool FlushOutputs = Mode.Output != DenormalMode::IEEE &&
+ Mode.Output != DenormalMode::Dynamic;
+ if (!FlushInputs && !FlushOutputs)
+ return false;
+ bool PositiveZero = Mode.Output == DenormalMode::PositiveZero;
+
+ // flush(x) = |x| < 2^-14 ? +-0 : x. NaNs compare false and pass through.
+ SmallPtrSet<Value *, 16> Flushed;
+ auto Flush = [&](IRBuilder<> &B, Value *X, bool Positive) -> Value * {
+ if (Flushed.contains(X))
+ return X;
+ if (auto *C = dyn_cast<ConstantFP>(X))
+ if (!C->getValueAPF().isDenormal())
+ return X;
+ Type *Ty = X->getType();
+ Value *Abs = B.CreateUnaryIntrinsic(Intrinsic::fabs, X);
+ Value *Small = B.CreateFCmpOLT(
+ Abs, ConstantFP::get(Ty, APFloat::getSmallestNormalized(
+ APFloat::IEEEhalf())));
+ Value *Zero = Positive ? ConstantFP::get(Ty, 0.0)
+ : B.CreateBinaryIntrinsic(Intrinsic::copysign,
+ ConstantFP::get(Ty, 0.0),
+ X);
+ Value *R = B.CreateSelect(Small, Zero, X);
+ Flushed.insert(R);
+ return R;
+ };
+
+ SmallVector<Instruction *, 32> Worklist;
+ for (Instruction &I : instructions(F)) {
+ bool Arith = isFPArithmetic(I) && isHalf(I.getType());
+ bool Reads = Arith || (isa<FCmpInst>(I) &&
+ isHalf(I.getOperand(0)->getType())) ||
+ (isa<FPExtInst>(I) && isHalf(I.getOperand(0)->getType())) ||
+ ((isa<FPToSIInst>(I) || isa<FPToUIInst>(I)) &&
+ isHalf(I.getOperand(0)->getType()));
+ bool Writes = Arith || (isa<FPTruncInst>(I) && isHalf(I.getType()));
+ if (Reads || Writes)
+ Worklist.push_back(&I);
+ }
+
+ for (Instruction *I : Worklist) {
+ IRBuilder<> B(I);
+ bool Arith = isFPArithmetic(*I) && isHalf(I->getType());
+ bool Reads = !isa<FPTruncInst>(I);
+ if (FlushInputs && Reads) {
+ unsigned NumOps = isa<CallBase>(I) ? cast<CallBase>(I)->arg_size()
+ : I->getNumOperands();
+ for (unsigned Op = 0; Op < NumOps; ++Op)
+ if (I->getOperand(Op)->getType()->isFPOrFPVectorTy() &&
+ isHalf(I->getOperand(Op)->getType()))
+ I->setOperand(Op, Flush(B, I->getOperand(Op), false));
+ }
+ if (auto *II = dyn_cast<IntrinsicInst>(I);
+ II && II->getIntrinsicID() == Intrinsic::canonicalize) {
+ // The operand is flushed already; nothing else to canonicalize.
+ Value *Op = II->getArgOperand(0);
+ if (FlushOutputs && !Flushed.contains(Op))
+ Op = Flush(B, Op, PositiveZero);
+ II->replaceAllUsesWith(Op);
+ II->eraseFromParent();
+ continue;
+ }
+ if (FlushOutputs && (Arith || isa<FPTruncInst>(I))) {
+ B.SetInsertPoint(I->getNextNode());
+ SmallVector<User *, 4> OldUsers(I->users());
+ Value *R = Flush(B, I, PositiveZero);
+ I->replaceUsesWithIf(R, [&](Use &U) {
+ return is_contained(OldUsers, U.getUser());
+ });
+ }
+ }
+ // The flushes are explicit now. Later combines would otherwise assume that
+ // the hardware flushes and fold them away.
+ AttrBuilder AB(F.getContext());
+ AB.addDenormalFPEnvAttr(DenormalFPEnv(
+ DenormalMode::getIEEE(), F.getDenormalMode(APFloat::IEEEsingle())));
+ F.addFnAttrs(AB);
+ return true;
+}
+
+bool AGXCodeGenPrepare::diagnoseDoubles(Function &F) {
+ SmallVector<Instruction *, 8> Unsupported;
+ for (Instruction &I : instructions(F)) {
+ bool Double = (isFPArithmetic(I) || isa<FCmpInst>(I) ||
+ isa<FPToSIInst>(I) || isa<FPToUIInst>(I) ||
+ isa<FPTruncInst>(I)) &&
+ I.getOperand(0)->getType()->getScalarType()->isDoubleTy();
+ Double |= (isa<SIToFPInst>(I) || isa<UIToFPInst>(I) ||
+ isa<FPExtInst>(I) || isFPArithmetic(I)) &&
+ I.getType()->getScalarType()->isDoubleTy();
+ if (Double)
+ Unsupported.push_back(&I);
+ }
+ if (Unsupported.empty())
+ return false;
+ F.getContext().diagnose(DiagnosticInfoUnsupported(
+ F, "double precision floating point arithmetic (AGX has no double "
+ "precision hardware)",
+ Unsupported.front()->getDebugLoc()));
+ for (Instruction *I : Unsupported) {
+ I->replaceAllUsesWith(PoisonValue::get(I->getType()));
+ I->eraseFromParent();
+ }
+ return true;
+}
+
bool AGXCodeGenPrepare::runOnFunction(Function &F) {
if (skipFunction(F))
return false;
const TargetTransformInfo &TTI =
getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
- bool Changed = lowerConstrainedFP(F);
+ const TargetMachine &TM =
+ getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
+ const auto &ST = TM.getSubtarget<AGXSubtarget>(F);
+ bool Changed = diagnoseDoubles(F);
+ if (ST.hasQuadShuffleIndexMerge())
+ Changed |= lowerDivergentShuffles(
+ F, getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo());
+ Changed |= lowerConstrainedFP(F);
+ Changed |= flushHalfDenormals(F);
Changed |= expandDivisions(F);
Changed |= expandMemIntrinsics(F, TTI);
Changed |= unifyExits(F);
diff --git a/llvm/lib/Target/AGX/AGXFeatures.td b/llvm/lib/Target/AGX/AGXFeatures.td
index d692d160b143a..fcb3729b01de4 100644
--- a/llvm/lib/Target/AGX/AGXFeatures.td
+++ b/llvm/lib/Target/AGX/AGXFeatures.td
@@ -71,8 +71,8 @@ def FeatureSIMDMatrix : SubtargetFeature<"simd-matrix",
def FeatureQuadShuffleIndexMerge : SubtargetFeature<"quad-shuffle-index-merge",
"HasQuadShuffleIndexMerge", "true",
- "simd_shuffle ORs the low two index bits across each quad, so divergent "
- "shuffle indices must be lowered with a loop"
+ "Shuffles OR the low two index bits of the threads of each quad; "
+ "divergent indices are expanded into four shuffles"
>;
def FeatureMultiDieCoherency : SubtargetFeature<"multi-die-coherency",
diff --git a/llvm/lib/Target/AGX/AGXISelLowering.cpp b/llvm/lib/Target/AGX/AGXISelLowering.cpp
index 7a5779c4e618c..85346d4260ae6 100644
--- a/llvm/lib/Target/AGX/AGXISelLowering.cpp
+++ b/llvm/lib/Target/AGX/AGXISelLowering.cpp
@@ -120,7 +120,8 @@ AGXTargetLowering::AGXTargetLowering(const TargetMachine &TM,
ISD::FNEARBYINT, ISD::FROUNDEVEN, ISD::FMINNUM,
ISD::FMAXNUM, ISD::FEXP2, ISD::FLOG2},
VT, Legal);
- setOperationAction({ISD::FSUB, ISD::SELECT_CC, ISD::FCOPYSIGN,
+ setOperationAction(ISD::FCOPYSIGN, VT, Custom);
+ setOperationAction({ISD::FSUB, ISD::SELECT_CC,
ISD::FMINIMUM, ISD::FMAXIMUM, ISD::FMINNUM_IEEE,
ISD::FMAXNUM_IEEE, ISD::FSINCOS, ISD::FROUND},
VT, Expand);
@@ -376,6 +377,8 @@ SDValue AGXTargetLowering::LowerOperation(SDValue Op,
return lowerFDIV(Op, DAG);
case ISD::FREM:
return lowerFREM(Op, DAG);
+ case ISD::FCOPYSIGN:
+ return lowerFCOPYSIGN(Op, DAG);
case ISD::FSIN:
case ISD::FCOS:
return lowerTrig(Op, DAG);
@@ -740,6 +743,28 @@ SDValue AGXTargetLowering::lowerFDIV(SDValue Op, SelectionDAG &DAG) const {
return DAG.getNode(ISD::FMA, DL, VT, E, R, Q);
}
+/// copysign(m, s) = (m & ~sign) | (s & sign), on the bits.
+SDValue AGXTargetLowering::lowerFCOPYSIGN(SDValue Op,
+ SelectionDAG &DAG) const {
+ SDLoc DL(Op);
+ EVT VT = Op.getValueType();
+ EVT IntVT = VT == MVT::f16 ? MVT::i16 : MVT::i32;
+ SDValue Mag = Op.getOperand(0);
+ SDValue Sign = Op.getOperand(1);
+ if (Sign.getValueType() != VT)
+ Sign = Sign.getValueType().bitsLT(VT)
+ ? DAG.getNode(ISD::FP_EXTEND, DL, VT, Sign)
+ : DAG.getNode(ISD::FP_ROUND, DL, VT, Sign,
+ DAG.getTargetConstant(0, DL, MVT::i32));
+ APInt SignMask = APInt::getSignMask(IntVT.getSizeInBits());
+ SDValue M = DAG.getNode(ISD::AND, DL, IntVT,
+ DAG.getBitcast(IntVT, Mag),
+ DAG.getConstant(~SignMask, DL, IntVT));
+ SDValue S = DAG.getNode(ISD::AND, DL, IntVT, DAG.getBitcast(IntVT, Sign),
+ DAG.getConstant(SignMask, DL, IntVT));
+ return DAG.getBitcast(VT, DAG.getNode(ISD::OR, DL, IntVT, M, S));
+}
+
SDValue AGXTargetLowering::lowerFREM(SDValue Op, SelectionDAG &DAG) const {
// a - trunc(a / b) * b
SDLoc DL(Op);
diff --git a/llvm/lib/Target/AGX/AGXISelLowering.h b/llvm/lib/Target/AGX/AGXISelLowering.h
index 5dff124131a95..35f3f8143f23b 100644
--- a/llvm/lib/Target/AGX/AGXISelLowering.h
+++ b/llvm/lib/Target/AGX/AGXISelLowering.h
@@ -34,6 +34,7 @@ class AGXTargetLowering final : public TargetLowering {
SDValue lowerCTLZ_CTTZ64(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFDIV(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFREM(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerTrig(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerExpLog(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/AGX/AGXInstructions.td b/llvm/lib/Target/AGX/AGXInstructions.td
index 98e94d5b08680..e97387bbe28f6 100644
--- a/llvm/lib/Target/AGX/AGXInstructions.td
+++ b/llvm/lib/Target/AGX/AGXInstructions.td
@@ -495,6 +495,35 @@ defm : SetCCPats<FCMPSEL_F32, f32, i32, AGXFPSrc32, AGXIntSrc32,
defm : SetCCPats<FCMPSEL_F16_I32, f16, i32, AGXFPSrc16, AGXIntSrc32,
AGXFPConds, 1>;
+// Selects of other value types on a comparison.
+multiclass SelectCCPats<Instruction inst, ValueType cvt, ValueType rvt,
+ ComplexPattern cpat, list<AGXCond> conds, bit fp> {
+ foreach C = conds in {
+ if fp then
+ def : Pat<(rvt (select (i32 (setcc (cpat cvt:$a, i32:$am),
+ (cpat cvt:$b, i32:$bm), C.CC)),
+ rvt:$x, rvt:$y)),
+ !if(C.Swap, (inst C.HW, $am, $a, $bm, $b, 0, $y, 0, $x),
+ (inst C.HW, $am, $a, $bm, $b, 0, $x, 0, $y))>;
+ else
+ def : Pat<(rvt (select (i32 (setcc cpat:$a, cpat:$b, C.CC)),
+ rvt:$x, rvt:$y)),
+ !if(C.Swap, (inst C.HW, 0, $a, 0, $b, 0, $y, 0, $x),
+ (inst C.HW, 0, $a, 0, $b, 0, $x, 0, $y))>;
+ }
+}
+
+defm : SelectCCPats<ICMPSEL_I32, i32, f32, AGXIntSrc32, AGXIntConds, 0>;
+defm : SelectCCPats<ICMPSEL_I16_I32, i16, f32, AGXIntSrc16, AGXIntConds, 0>;
+defm : SelectCCPats<FCMPSEL_F32, f32, f32, AGXFPSrc32, AGXFPConds, 1>;
+defm : SelectCCPats<FCMPSEL_F16_I32, f16, f32, AGXFPSrc16, AGXFPConds, 1>;
+foreach VT = [i16, f16, bf16] in {
+defm : SelectCCPats<ICMPSEL_I32_I16, i32, VT, AGXIntSrc32, AGXIntConds, 0>;
+defm : SelectCCPats<ICMPSEL_I16, i16, VT, AGXIntSrc16, AGXIntConds, 0>;
+defm : SelectCCPats<FCMPSEL_F32_I16, f32, VT, AGXFPSrc32, AGXFPConds, 1>;
+defm : SelectCCPats<FCMPSEL_F16, f16, VT, AGXFPSrc16, AGXFPConds, 1>;
+}
+
// Select on a boolean (0/1 in a 32-bit register).
def : Pat<(i32 (select i32:$c, AGXIntSrc32:$x, AGXIntSrc32:$y)),
(ICMPSEL_I32 IC_UEQ, 0, $c, 0, (i32 0), 0, $y, 0, $x)>;
@@ -1082,6 +1111,12 @@ def WAIT : AGXPseudo<(outs), (ins i32imm:$slot), [(int_agx_wait timm:$slot)]> {
// SIMD-group operations
//===----------------------------------------------------------------------===//
+def shuffle_imm_xform : SDNodeXForm<imm, [{
+ return CurDAG->getTargetConstant(N->getZExtValue(), SDLoc(N), MVT::i16);
+}]>;
+def shuffle_imm : ImmLeaf<i32, [{ return Imm >= 0 && Imm < 256; }],
+ shuffle_imm_xform>;
+
multiclass ShufflePseudo<Instruction real, SDPatternOperator intr> {
let isConvergent = 1 in {
def _32 : AGXALUPseudo<real, (outs GPR32:$dst),
@@ -1101,6 +1136,15 @@ multiclass ShufflePseudo<Instruction real, SDPatternOperator intr> {
def : Pat<(VT (intr VT:$v, i32:$idx)),
(!cast<Instruction>(NAME#"_16") 0, $v, 0,
(EXTRACT_SUBREG $idx, lo16))>;
+ // Immediate indices are the same in every thread.
+ let AddedComplexity = 10 in {
+ foreach VT = [i32, f32] in
+ def : Pat<(VT (intr VT:$v, shuffle_imm:$idx)),
+ (!cast<Instruction>(NAME#"_32") 0, $v, 0, (i16 (shuffle_imm_xform $idx)))>;
+ foreach VT = [i16, f16] in
+ def : Pat<(VT (intr VT:$v, shuffle_imm:$idx)),
+ (!cast<Instruction>(NAME#"_16") 0, $v, 0, (i16 (shuffle_imm_xform $idx)))>;
+ }
}
defm SIMD_SHUFFLE : ShufflePseudo<SIMD_SHUFFLE_g13, int_agx_simd_shuffle>;
diff --git a/llvm/lib/Target/AGX/AGXLowerControlFlow.cpp b/llvm/lib/Target/AGX/AGXLowerControlFlow.cpp
index 07f9165f126b0..147a60cbcfa37 100644
--- a/llvm/lib/Target/AGX/AGXLowerControlFlow.cpp
+++ b/llvm/lib/Target/AGX/AGXLowerControlFlow.cpp
@@ -20,6 +20,12 @@
// instructions alone implement the semantics. The nesting counter is zeroed
// at kernel entry.
//
+// Shuffles read the index of every thread of a quad, including inactive
+// ones, whose register may hold anything. Register indices are therefore
+// passed in r0h (like Mesa's agx_lower_divergent_shuffle), which is zero at
+// entry and set back to zero after each shuffle, so the threads that are
+// inactive at a shuffle read zero.
+//
//===----------------------------------------------------------------------===//
#include "AGX.h"
@@ -73,9 +79,52 @@ static MachineInstrBuilder buildExecOp(MachineBasicBlock &MBB,
return BuildMI(MBB, I, DL, TII.get(Opc)).addImm(0).addImm(CC);
}
+static bool isShuffle(unsigned Opc) {
+ switch (Opc) {
+ case AGX::SIMD_SHUFFLE_32:
+ case AGX::SIMD_SHUFFLE_16:
+ case AGX::SIMD_SHUFFLE_XOR_32:
+ case AGX::SIMD_SHUFFLE_XOR_16:
+ case AGX::SIMD_SHUFFLE_UP_32:
+ case AGX::SIMD_SHUFFLE_UP_16:
+ case AGX::SIMD_SHUFFLE_DOWN_32:
+ case AGX::SIMD_SHUFFLE_DOWN_16:
+ case AGX::QUAD_SHUFFLE_32:
+ case AGX::QUAD_SHUFFLE_16:
+ case AGX::QUAD_SHUFFLE_XOR_32:
+ case AGX::QUAD_SHUFFLE_XOR_16:
+ case AGX::QUAD_SHUFFLE_UP_32:
+ case AGX::QUAD_SHUFFLE_UP_16:
+ case AGX::QUAD_SHUFFLE_DOWN_32:
+ case AGX::QUAD_SHUFFLE_DOWN_16:
+ return true;
+ default:
+ return false;
+ }
+}
+
+/// Pass the register index of a shuffle in r0h. Returns true if MI was
+/// changed.
+static bool lowerShuffleIndex(MachineInstr &MI, const AGXInstrInfo &TII) {
+ // (dst, src0_mods, src0, src1_mods, src1)
+ MachineOperand &Index = MI.getOperand(4);
+ if (!Index.isReg() || !AGX::GPR16RegClass.contains(Index.getReg()))
+ return false;
+ MachineBasicBlock &MBB = *MI.getParent();
+ const DebugLoc &DL = MI.getDebugLoc();
+ TII.copyPhysReg(MBB, MI, DL, AGX::R0H, Index.getReg(), Index.isKill());
+ Index.setReg(AGX::R0H);
+ Index.setIsKill();
+ BuildMI(MBB, std::next(MI.getIterator()), DL, TII.get(AGX::MOV_IMM16),
+ AGX::R0H)
+ .addImm(0);
+ return true;
+}
+
bool AGXLowerControlFlow::runOnMachineFunction(MachineFunction &MF) {
const AGXInstrInfo &TII = *MF.getSubtarget<AGXSubtarget>().getInstrInfo();
bool HasControlFlow = false;
+ bool HasShuffleIndex = false;
for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : make_early_inc_range(MBB)) {
@@ -143,6 +192,8 @@ bool AGXLowerControlFlow::runOnMachineFunction(MachineFunction &MF) {
continue;
}
default:
+ if (isShuffle(MI.getOpcode()))
+ HasShuffleIndex |= lowerShuffleIndex(MI, TII);
continue;
}
HasControlFlow = true;
@@ -150,8 +201,12 @@ bool AGXLowerControlFlow::runOnMachineFunction(MachineFunction &MF) {
}
}
+ MachineBasicBlock &Entry = MF.front();
+ if (HasShuffleIndex)
+ BuildMI(Entry, Entry.begin(), DebugLoc(), TII.get(AGX::MOV_IMM16),
+ AGX::R0H)
+ .addImm(0);
if (HasControlFlow) {
- MachineBasicBlock &Entry = MF.front();
BuildMI(Entry, Entry.begin(), DebugLoc(), TII.get(AGX::SET_NEST))
.addImm(0);
}
diff --git a/llvm/test/CodeGen/AGX/errors-f64.ll b/llvm/test/CodeGen/AGX/errors-f64.ll
new file mode 100644
index 0000000000000..310e1a887ac9c
--- /dev/null
+++ b/llvm/test/CodeGen/AGX/errors-f64.ll
@@ -0,0 +1,12 @@
+; RUN: not llc -mtriple=agx -mcpu=agx-g13g < %s 2>&1 | FileCheck %s
+
+; CHECK: error: {{.*}}in function doubles {{.*}}: double precision floating point arithmetic (AGX has no double precision hardware)
+; CHECK-NOT: error
+
+define agx_kernel void @doubles(ptr addrspace(1) %out, double %a, float %b) {
+ %e = fpext float %b to double
+ %s = fadd double %a, %e
+ %m = fmul double %s, %a
+ store double %m, ptr addrspace(1) %out
+ ret void
+}
diff --git a/llvm/test/CodeGen/AGX/fp.ll b/llvm/test/CodeGen/AGX/fp.ll
index 2cbb402d0668b..c87d98ccc02b6 100644
--- a/llvm/test/CodeGen/AGX/fp.ll
+++ b/llvm/test/CodeGen/AGX/fp.ll
@@ -151,15 +151,13 @@ define agx_kernel void @rounding(ptr addrspace(1) %out, float %a, float %b) {
define agx_kernel void @compare(ptr addrspace(1) %out, float %a, float %b) {
; CHECK-LABEL: compare:
; CHECK: // uniform registers: 14 x 16-bit (threadgroups_per_grid at u4)
-; CHECK-NEXT: // general purpose registers: 3 x 32-bit
+; CHECK-NEXT: // general purpose registers: 2 x 32-bit
; CHECK-NEXT: // threadgroup memory: 0 bytes
; CHECK-NEXT: // stack size: 0 bytes
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: fcmpsel lt, r1, u2, u3, 1, 0
-; CHECK-NEXT: mov_imm r2, 1077936128
-; CHECK-NEXT: icmpsel ueq, r1, r1, 0, r2, u2
-; CHECK-NEXT: fcmpsel lt, r2, r1, u3, 0, 1
-; CHECK-NEXT: icmpsel ueq, r1, r2, 0, u3, r1
+; CHECK-NEXT: mov_imm r1, 1077936128
+; CHECK-NEXT: fcmpsel lt, r1, u2, u3, u2, r1
+; CHECK-NEXT: fcmpsel lt, r1, r1, u3, u3, r1
; CHECK-NEXT: device_store 0, i32, x, r1, u0_u1, 0, signed
; CHECK-NEXT: stop
; CHECK-NEXT: trap
diff --git a/llvm/test/CodeGen/AGX/half-denormals.ll b/llvm/test/CodeGen/AGX/half-denormals.ll
new file mode 100644
index 0000000000000..2b3c30879cede
--- /dev/null
+++ b/llvm/test/CodeGen/AGX/half-denormals.ll
@@ -0,0 +1,32 @@
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -verify-machineinstrs < %s | FileCheck %s
+
+; The hardware keeps 16-bit denormals. Functions whose denormal mode for half
+; flushes get flush(x) = |x| < 2^-14 ? copysign(0, x) : x on the operands and
+; the results of half arithmetic.
+
+; CHECK-LABEL: flush:
+; CHECK: mov_imm [[SIGN:r[0-9]+[lh]]], 32768
+; CHECK: mov_imm [[MIN:r[0-9]+[lh]]], 1024
+; CHECK: fcmpsel lt, {{r[0-9]+[lh]}}, u2l.abs, [[MIN]],
+; CHECK: fcmpsel lt, {{r[0-9]+[lh]}}, u2h.abs, [[MIN]],
+; CHECK: fadd16 [[SUM:r[0-9]+[lh]]],
+; CHECK: and {{r[0-9]+[lh]}}, [[SUM]], [[SIGN]]
+; CHECK: fcmpsel lt, {{r[0-9]+[lh]}}, [[SUM]].abs, [[MIN]],
+; CHECK: device_store
+define agx_kernel void @flush(ptr addrspace(1) %out, half %a, half %b) #0 {
+ %s = fadd half %a, %b
+ store half %s, ptr addrspace(1) %out
+ ret void
+}
+
+; CHECK-LABEL: ieee:
+; CHECK-NOT: fcmpsel
+; CHECK: fadd16
+; CHECK-NEXT: device_store
+define agx_kernel void @ieee(ptr addrspace(1) %out, half %a, half %b) {
+ %s = fadd half %a, %b
+ store half %s, ptr addrspace(1) %out
+ ret void
+}
+
+attributes #0 = { denormal_fpenv(preservesign) }
diff --git a/llvm/test/CodeGen/AGX/shuffle-divergent.ll b/llvm/test/CodeGen/AGX/shuffle-divergent.ll
new file mode 100644
index 0000000000000..9c18e8b5a57cf
--- /dev/null
+++ b/llvm/test/CodeGen/AGX/shuffle-divergent.ll
@@ -0,0 +1,75 @@
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -verify-machineinstrs < %s | FileCheck %s
+
+; simd_shuffle ORs the low two index bits of the threads of each quad, and
+; reads the index of inactive threads too. Divergent indices are split into
+; the four quad positions, and register indices are passed in r0h, which is
+; zero whenever no shuffle is executing.
+
+declare i32 @llvm.agx.thread.index.in.simdgroup()
+declare float @llvm.agx.simd.shuffle.f32(float, i32)
+declare float @llvm.agx.simd.shuffle.down.f32(float, i32)
+declare float @llvm.agx.quad.shuffle.f32(float, i32)
+
+; CHECK-LABEL: divergent:
+; CHECK: mov_imm r0h, 0
+; CHECK: simd_shuffle {{r[0-9]+}}, {{[ru][0-9]+}}, r0h
+; CHECK-NEXT: mov_imm r0h, 0
+; CHECK: simd_shuffle {{r[0-9]+}}, {{[ru][0-9]+}}, r0h
+; CHECK-NEXT: mov_imm r0h, 0
+; CHECK: simd_shuffle {{r[0-9]+}}, {{[ru][0-9]+}}, r0h
+; CHECK-NEXT: mov_imm r0h, 0
+; CHECK: simd_shuffle {{r[0-9]+}}, {{[ru][0-9]+}}, r0h
+; CHECK-NEXT: mov_imm r0h, 0
+; CHECK-NOT: simd_shuffle
+; CHECK: stop
+define agx_kernel void @divergent(ptr addrspace(1) %out, float %x) {
+ %lane = call i32 @llvm.agx.thread.index.in.simdgroup()
+ %idx = mul i32 %lane, 7
+ %r = call float @llvm.agx.simd.shuffle.f32(float %x, i32 %idx)
+ %p = getelementptr float, ptr addrspace(1) %out, i32 %lane
+ store float %r, ptr addrspace(1) %p
+ ret void
+}
+
+; A uniform index is used directly.
+; CHECK-LABEL: uniform:
+; CHECK-NOT: r0h
+; CHECK: simd_shuffle {{r[0-9]+}}, {{[ru][0-9]+}}, u3l
+; CHECK-NOT: simd_shuffle
+; CHECK: stop
+define agx_kernel void @uniform(ptr addrspace(1) %out, float %x, i32 %i) {
+ %lane = call i32 @llvm.agx.thread.index.in.simdgroup()
+ %v = uitofp i32 %lane to float
+ %r = call float @llvm.agx.simd.shuffle.f32(float %v, i32 %i)
+ %p = getelementptr float, ptr addrspace(1) %out, i32 %lane
+ store float %r, ptr addrspace(1) %p
+ ret void
+}
+
+; A divergent delta reads the computed thread; quad shuffles select between
+; the four immediate positions.
+; CHECK-LABEL: relative:
+; CHECK-COUNT-4: simd_shuffle {{r[0-9]+}}, {{[ru][0-9]+}}, r0h
+; CHECK-NOT: simd_shuffle_down
+; CHECK: quad_shuffle {{r[0-9]+}}, {{r[0-9]+}}, 0
+; CHECK: quad_shuffle {{r[0-9]+}}, {{r[0-9]+}}, 1
+; CHECK: quad_shuffle {{r[0-9]+}}, {{r[0-9]+}}, 2
+; CHECK: quad_shuffle {{r[0-9]+}}, {{r[0-9]+}}, 3
+; CHECK: stop
+define agx_kernel void @relative(ptr addrspace(1) %out, float %x) {
+ %lane = call i32 @llvm.agx.thread.index.in.simdgroup()
+ %d = and i32 %lane, 3
+ %v = uitofp i32 %lane to float
+ %a = call float @llvm.agx.simd.shuffle.down.f32(float %v, i32 %d)
+ %b = call float @llvm.agx.quad.shuffle.f32(float %a, i32 %d)
+ %p = getelementptr float, ptr addrspace(1) %out, i32 %lane
+ store float %b, ptr addrspace(1) %p
+ ret void
+}
+
+; Without the merge, divergent indices are used directly.
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -mattr=-quad-shuffle-index-merge < %s | FileCheck %s --check-prefix=NOMERGE
+; NOMERGE-LABEL: divergent:
+; NOMERGE-COUNT-1: simd_shuffle
+; NOMERGE-NOT: simd_shuffle
+; NOMERGE: stop
diff --git a/llvm/test/CodeGen/AGX/simd.ll b/llvm/test/CodeGen/AGX/simd.ll
index c328c5be15d30..8b6ae5946425e 100644
--- a/llvm/test/CodeGen/AGX/simd.ll
+++ b/llvm/test/CodeGen/AGX/simd.ll
@@ -4,17 +4,14 @@
define agx_kernel void @shuffles(ptr addrspace(1) %out, float %x, i32 %i) {
; CHECK-LABEL: shuffles:
; CHECK: // uniform registers: 14 x 16-bit (threadgroups_per_grid at u4)
-; CHECK-NEXT: // general purpose registers: 3 x 32-bit
+; CHECK-NEXT: // general purpose registers: 2 x 32-bit
; CHECK-NEXT: // threadgroup memory: 0 bytes
; CHECK-NEXT: // stack size: 0 bytes
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: simd_shuffle r1, u2, u3l
-; CHECK-NEXT: mov_imm r2, 1
-; CHECK-NEXT: simd_shuffle_xor r1, r1, r2l
-; CHECK-NEXT: mov_imm r2, 2
-; CHECK-NEXT: simd_shuffle_down r1, r1, r2l
-; CHECK-NEXT: mov_imm r2, 3
-; CHECK-NEXT: quad_shuffle r1, r1, r2l
+; CHECK-NEXT: simd_shuffle_xor r1, r1, 1
+; CHECK-NEXT: simd_shuffle_down r1, r1, 2
+; CHECK-NEXT: quad_shuffle r1, r1, 3
; CHECK-NEXT: device_store 0, i32, x, r1, u0_u1, 0, signed
; CHECK-NEXT: stop
; CHECK-NEXT: trap
>From 1ece7823221e15694583a2a38b2de267c5fc759c Mon Sep 17 00:00:00 2001
From: Claude <noreply at anthropic.com>
Date: Sun, 18 Oct 2026 22:38:23 +0000
Subject: [PATCH 11/22] [AGX] Add opt-in software IEEE binary32 and canonical
NaN modes
The float ALU flushes denormals, division and square root are not
correctly rounded and NaN results have hardware bit patterns. Two new
subtarget features, off by default, implement the IEEE 754 / PTX
behavior in software (AGXStrictFP):
- ieee-fp32 (functions whose float denormal mode is IEEE):
* fadd/fsub/fmul/fma keep the hardware instruction and recompute the
result with exact integer arithmetic in a conditional block only
where the hardware can be wrong (both addends below 2^-101, a
denormal operand, or a product/result of at most 2^-125);
* fdiv and sqrt are correctly rounded: refined hardware estimates of
the mantissa quotient or root are corrected with exact FMA remainders
(Tuckerman's test for sqrt); operands or results outside the normal
range use integer arithmetic;
* comparisons and min/max compare zeros and denormals as integers;
rounding functions, rcp, rsqrt, log2 and exp2 handle denormals.
- canonical-nan: NaN results become 0x7fffffff (0x7fff for half/bfloat).
Validated bit-exactly on the functional AGX simulator against an exact
rational IEEE reference for add, sub, mul, fma, div, sqrt, comparisons,
the min/max variants and rounding functions, over denormals, ties,
cancellations, near-midpoint quotients and roots, and specials.
Also select i16 to i64 extensions and i64 to i16 truncations, which the
new code needs.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
Claude-Session: https://claude.ai/code/session_01E56KEecg6edNaS23myRWER
---
llvm/docs/AGXUsage.rst | 42 ++
llvm/lib/Target/AGX/AGX.h | 2 +
llvm/lib/Target/AGX/AGXFeatures.td | 12 +
llvm/lib/Target/AGX/AGXInstructions.td | 12 +-
llvm/lib/Target/AGX/AGXStrictFP.cpp | 877 +++++++++++++++++++++++
llvm/lib/Target/AGX/AGXTargetMachine.cpp | 3 +
llvm/lib/Target/AGX/CMakeLists.txt | 1 +
llvm/test/CodeGen/AGX/strict-fp.ll | 60 ++
8 files changed, 1008 insertions(+), 1 deletion(-)
create mode 100644 llvm/lib/Target/AGX/AGXStrictFP.cpp
create mode 100644 llvm/test/CodeGen/AGX/strict-fp.ll
diff --git a/llvm/docs/AGXUsage.rst b/llvm/docs/AGXUsage.rst
index 18fefcdf57605..5c3b7d83c523f 100644
--- a/llvm/docs/AGXUsage.rst
+++ b/llvm/docs/AGXUsage.rst
@@ -115,6 +115,11 @@ another.
``source-lifetime`` ALU sources carry register release flags
(Apple9).
``max-gprs-128``, ``-64`` Limit the general purpose registers used.
+ ``ieee-fp32`` IEEE 754 ``float`` arithmetic in software
+ where the hardware differs (off by default,
+ see `Strict floating point`_).
+ ``canonical-nan`` NaN results are the canonical NaN of PTX
+ (off by default).
============================== ==============================================
Address spaces
@@ -286,6 +291,43 @@ number, and ``llvm.agx.simd.reduce``/``prefix`` and
7 fmax
===== ========= ===== =========
+Strict floating point
+---------------------
+
+The 32-bit floating point unit flushes denormal operands and results to zero,
+division and square root are not correctly rounded, and NaN results have the
+hardware's bit patterns. Two features, off by default, implement IEEE 754 and
+PTX behavior in software (``AGXStrictFP.cpp``):
+
+``ieee-fp32``
+ In functions whose ``denormal_fpenv`` for ``float`` is IEEE (the default):
+
+ * ``fadd``, ``fsub``, ``fmul`` and ``llvm.fma``/``llvm.fmuladd`` keep the
+ hardware instruction, and recompute the result exactly with integer
+ arithmetic in a conditional block where it can be wrong: an addition
+ whose operands are both below 2\ :sup:`-101`, a multiplication or fused
+ multiply-add with a denormal operand or a result of at most
+ 2\ :sup:`-125`.
+ * ``fdiv`` (without ``arcp`` or ``afn``) and ``llvm.sqrt`` (without ``afn``)
+ are correctly rounded: the mantissas are divided (or the root taken) with
+ refined hardware estimates, and corrected until the exact FMA remainder
+ shows that the result is correctly rounded. Operands or results outside
+ the normal range use integer arithmetic.
+ * Comparisons, ``minnum``/``maxnum``/``minimum``/``maximum`` (and the
+ ``num`` variants) compare zeros and denormals as integers, and rounding
+ functions, ``llvm.agx.rcp``, ``llvm.agx.rsqrt``, ``llvm.log2`` and
+ ``llvm.exp2`` handle denormal operands or results explicitly.
+
+``canonical-nan``
+ NaN results of floating point arithmetic and conversions are ``0x7fffffff``
+ (``0x7fff`` for ``half`` and ``bfloat``); minimum and maximum keep the bits
+ of a non-NaN operand. This costs one instruction per operation.
+
+Measured on the functional simulator, per SIMD-group and operation with
+normal operands, ``ieee-fp32`` adds about 8 instructions to an addition, 12
+to a multiplication, 22 to a fused multiply-add, 13 to a comparison, 65 to a
+division and 39 to a square root.
+
SIMD-group shuffles
-------------------
diff --git a/llvm/lib/Target/AGX/AGX.h b/llvm/lib/Target/AGX/AGX.h
index 8bea7ed34e48d..80b9005ac124b 100644
--- a/llvm/lib/Target/AGX/AGX.h
+++ b/llvm/lib/Target/AGX/AGX.h
@@ -25,6 +25,7 @@ class PassRegistry;
// IR passes.
ModulePass *createAGXAlwaysInlinePass();
FunctionPass *createAGXCodeGenPreparePass();
+FunctionPass *createAGXStrictFPPass();
FunctionPass *createAGXAnnotateControlFlowPass();
// Instruction selection.
@@ -37,6 +38,7 @@ FunctionPass *createAGXInsertWaitsPass();
void initializeAGXAlwaysInlinePass(PassRegistry &);
void initializeAGXCodeGenPreparePass(PassRegistry &);
+void initializeAGXStrictFPPass(PassRegistry &);
void initializeAGXAnnotateControlFlowPass(PassRegistry &);
void initializeAGXDAGToDAGISelLegacyPass(PassRegistry &);
void initializeAGXFoldOperandsPass(PassRegistry &);
diff --git a/llvm/lib/Target/AGX/AGXFeatures.td b/llvm/lib/Target/AGX/AGXFeatures.td
index fcb3729b01de4..343545c09ca18 100644
--- a/llvm/lib/Target/AGX/AGXFeatures.td
+++ b/llvm/lib/Target/AGX/AGXFeatures.td
@@ -75,6 +75,18 @@ def FeatureQuadShuffleIndexMerge : SubtargetFeature<"quad-shuffle-index-merge",
"divergent indices are expanded into four shuffles"
>;
+def FeatureIEEEFP32 : SubtargetFeature<"ieee-fp32", "HasIEEEFP32", "true",
+ "Implement IEEE 754 binary32 arithmetic in software where the hardware "
+ "differs: denormals (in functions whose denormal mode for float is IEEE), "
+ "and correctly rounded division and square root"
+>;
+
+def FeatureCanonicalNaN : SubtargetFeature<"canonical-nan", "HasCanonicalNaN",
+ "true",
+ "NaN results of floating point arithmetic are the canonical NaN of PTX "
+ "(0x7fffffff, 0x7fff for half and bfloat)"
+>;
+
def FeatureMultiDieCoherency : SubtargetFeature<"multi-die-coherency",
"HasMultiDieCoherency", "true",
"Device atomics must set the coherency bit (multi-die G13 parts)"
diff --git a/llvm/lib/Target/AGX/AGXInstructions.td b/llvm/lib/Target/AGX/AGXInstructions.td
index e97387bbe28f6..d5ad46a41e9a0 100644
--- a/llvm/lib/Target/AGX/AGXInstructions.td
+++ b/llvm/lib/Target/AGX/AGXInstructions.td
@@ -278,6 +278,15 @@ def : Pat<(i32 (sext i16:$a)), (IADD_I32_FROM16 MOD_SX, $a, 0, (i16 0), 0)>;
def : Pat<(i64 (sext i32:$a)), (IADD_I64_FROM32 MOD_SX, $a, 0, (i32 0), 0)>;
def : Pat<(i64 (zext i32:$a)), (IADD_I64_FROM32 0, $a, 0, (i32 0), 0)>;
def : Pat<(i64 (anyext i32:$a)), (IADD_I64_FROM32 0, $a, 0, (i32 0), 0)>;
+def : Pat<(i64 (zext i16:$a)),
+ (IADD_I64_FROM32 0, (BITOP_I32_FROM16 TT_MOV, 0, $a, 0, (i16 0)), 0,
+ (i32 0), 0)>;
+def : Pat<(i64 (anyext i16:$a)),
+ (IADD_I64_FROM32 0, (BITOP_I32_FROM16 TT_MOV, 0, $a, 0, (i16 0)), 0,
+ (i32 0), 0)>;
+def : Pat<(i64 (sext i16:$a)),
+ (IADD_I64_FROM32 MOD_SX, (IADD_I32_FROM16 MOD_SX, $a, 0, (i16 0), 0),
+ 0, (i32 0), 0)>;
def : Pat<(i32 (sext_inreg i32:$a, i16)),
(IADD_I32_FROM16 MOD_SX, (EXTRACT_SUBREG $a, lo16), 0, (i16 0), 0)>;
def : Pat<(i64 (sext_inreg i64:$a, i32)),
@@ -285,7 +294,8 @@ def : Pat<(i64 (sext_inreg i64:$a, i32)),
def : Pat<(i16 (trunc i32:$a)), (EXTRACT_SUBREG $a, lo16)>;
def : Pat<(i32 (trunc i64:$a)), (EXTRACT_SUBREG $a, sub0)>;
-def : Pat<(i16 (trunc i64:$a)), (EXTRACT_SUBREG (EXTRACT_SUBREG $a, sub0), lo16)>;
+def : Pat<(i16 (trunc i64:$a)),
+ (i16 (EXTRACT_SUBREG (i32 (EXTRACT_SUBREG $a, sub0)), lo16))>;
// imadd: D = A * B + C.
multiclass IMAddPseudos<RegisterClass rc, RegisterOperand src,
diff --git a/llvm/lib/Target/AGX/AGXStrictFP.cpp b/llvm/lib/Target/AGX/AGXStrictFP.cpp
new file mode 100644
index 0000000000000..78d5b9e8dad11
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXStrictFP.cpp
@@ -0,0 +1,877 @@
+//===-- AGXStrictFP.cpp - IEEE binary32 and canonical NaNs in software ----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The 32-bit floating point ALU flushes denormal operands and results to zero,
+// division and square root are not correctly rounded, and NaN results have the
+// hardware's bit patterns. Two opt-in subtarget features implement the
+// missing behavior in software:
+//
+// * ieee-fp32: IEEE 754 binary32 arithmetic in functions whose denormal mode
+// for float is IEEE. Additions, multiplications and fused multiply-adds
+// keep the hardware instruction and recompute the result with integer
+// arithmetic only where the hardware result can be wrong (denormal
+// operands, or a result close to the denormal range). Comparisons,
+// minimum and maximum compare denormals as integers, and rounding
+// functions and the approximate functions (rcp, rsqrt, log2, exp2) treat
+// denormal operands or results explicitly. Division and square root are
+// correctly rounded: refined hardware estimates are corrected with exact
+// remainders for normal operands and results, and computed with integer
+// arithmetic otherwise.
+//
+// * canonical-nan: NaN results of floating point arithmetic are the
+// canonical NaN of PTX, 0x7fffffff (0x7fff for half and bfloat).
+//
+// Exact results are represented as sign * M * 2^E with a 64-bit integer M.
+// convert() rounds such a value to nearest even, producing denormals,
+// normals or infinities as IEEE 754 requires; values that are not exact keep
+// a sticky bit in the low bit of M.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AGX.h"
+#include "AGXSubtarget.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstIterator.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IntrinsicsAGX.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "agx-strict-fp"
+
+namespace {
+
+class AGXStrictFP : public FunctionPass {
+public:
+ static char ID;
+ AGXStrictFP() : FunctionPass(ID) {}
+
+ StringRef getPassName() const override { return "AGX Strict FP"; }
+ bool runOnFunction(Function &F) override;
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<TargetPassConfig>();
+ }
+};
+
+/// sign * M * 2^E: Sign is i1, M is i64 and E is i32.
+struct Term {
+ Value *Sign;
+ Value *M;
+ Value *E;
+};
+
+/// Builds the integer implementations at the builder's insertion point.
+class SoftFloat {
+ IRBuilder<> &B;
+ Type *I1, *I32, *I64, *F32;
+
+public:
+ explicit SoftFloat(IRBuilder<> &B)
+ : B(B), I1(B.getInt1Ty()), I32(B.getInt32Ty()), I64(B.getInt64Ty()),
+ F32(B.getFloatTy()) {}
+
+ Value *c32(uint32_t V) { return B.getInt32(V); }
+ Value *c64(uint64_t V) { return B.getInt64(V); }
+ Value *bits(Value *F) { return B.CreateBitCast(F, I32); }
+ Value *flt(Value *I) { return B.CreateBitCast(I, F32); }
+ Value *fconst(float V) { return ConstantFP::get(F32, V); }
+
+ Value *umin(Value *A, Value *X) {
+ return B.CreateBinaryIntrinsic(Intrinsic::umin, A, X);
+ }
+ Value *smax(Value *A, Value *X) {
+ return B.CreateBinaryIntrinsic(Intrinsic::smax, A, X);
+ }
+ /// Bit length of a 64-bit value, as i32 (0 for 0).
+ Value *bitLength(Value *M) {
+ Value *Lz = B.CreateBinaryIntrinsic(Intrinsic::ctlz, M, B.getFalse());
+ return B.CreateTrunc(B.CreateSub(c64(64), Lz), I32);
+ }
+ /// M << S and M >> S for 0 <= S; amounts of 64 or more give 0.
+ Value *shl(Value *M, Value *S) {
+ Value *R = B.CreateShl(M, B.CreateZExt(umin(S, c32(63)), I64));
+ return B.CreateSelect(B.CreateICmpUGT(S, c32(63)), c64(0), R);
+ }
+ Value *lshr(Value *M, Value *S) {
+ Value *R = B.CreateLShr(M, B.CreateZExt(umin(S, c32(63)), I64));
+ return B.CreateSelect(B.CreateICmpUGT(S, c32(63)), c64(0), R);
+ }
+ /// M >> S with the bits shifted out ORed into the low bit, for 0 <= S.
+ Value *lshrSticky(Value *M, Value *S) {
+ Value *Q = lshr(M, S);
+ Value *Lost = B.CreateICmpNE(B.CreateShl(Q, B.CreateZExt(umin(S, c32(63)), I64)), M);
+ Lost = B.CreateOr(Lost, B.CreateAnd(B.CreateICmpUGT(S, c32(63)),
+ B.CreateICmpNE(M, c64(0))));
+ return B.CreateOr(Q, B.CreateZExt(Lost, I64));
+ }
+
+ /// The exponent field is 0: zero or denormal.
+ Value *expZero(Value *Bits) {
+ return B.CreateICmpEQ(B.CreateAnd(Bits, c32(0x7f800000)), c32(0));
+ }
+ Value *isDenormal(Value *Bits) {
+ Value *Abs = B.CreateAnd(Bits, c32(0x7fffffff));
+ return B.CreateICmpULT(B.CreateSub(Abs, c32(1)), c32(0x7fffff));
+ }
+ Value *isZero(Value *Bits) {
+ return B.CreateICmpEQ(B.CreateAnd(Bits, c32(0x7fffffff)), c32(0));
+ }
+ Value *isInfOrNaN(Value *Bits) {
+ return B.CreateICmpEQ(B.CreateAnd(Bits, c32(0x7f800000)), c32(0x7f800000));
+ }
+ Value *signBit(Value *Bits) { return B.CreateICmpSLT(Bits, c32(0)); }
+
+ /// A finite value as sign * M * 2^E with M < 2^24. With Normalize, the
+ /// most significant bit of a nonzero M is bit 23.
+ Term unpack(Value *Bits, bool Normalize) {
+ Value *Exp = B.CreateAnd(B.CreateLShr(Bits, 23), c32(0xff));
+ Value *Frac = B.CreateAnd(Bits, c32(0x7fffff));
+ Value *Den = B.CreateICmpEQ(Exp, c32(0));
+ Value *M = B.CreateSelect(Den, Frac, B.CreateOr(Frac, c32(0x800000)));
+ Value *E = B.CreateSub(B.CreateSelect(Den, c32(1), Exp), c32(150));
+ if (Normalize) {
+ Value *Lz = B.CreateBinaryIntrinsic(Intrinsic::ctlz, M, B.getFalse());
+ Value *Sh = B.CreateSelect(B.CreateICmpEQ(M, c32(0)), c32(0),
+ B.CreateSub(Lz, c32(8)));
+ M = B.CreateShl(M, Sh);
+ E = B.CreateSub(E, Sh);
+ }
+ return {signBit(Bits), B.CreateZExt(M, I64), E};
+ }
+
+ /// Round sign * M * 2^E to nearest even: the bits of the binary32 result.
+ Value *convert(const Term &T) {
+ Value *L = bitLength(T.M);
+ // Shift so that 24 significant bits remain, or fewer for denormals.
+ Value *S = smax(B.CreateSub(L, c32(24)), B.CreateSub(c32(-149), T.E));
+ Value *Right = B.CreateICmpSGT(S, c32(0));
+ Value *K = B.CreateSelect(Right, umin(S, c32(64)), c32(0));
+ Value *Q = lshr(T.M, K);
+ Value *Rem = B.CreateSub(T.M, shl(Q, K));
+ Value *Half = shl(c64(1), B.CreateSub(smax(K, c32(1)), c32(1)));
+ Value *Up = B.CreateOr(
+ B.CreateICmpUGT(Rem, Half),
+ B.CreateAnd(B.CreateICmpEQ(Rem, Half),
+ B.CreateICmpNE(B.CreateAnd(Q, c64(1)), c64(0))));
+ Up = B.CreateAnd(Up, Right);
+ Value *Rounded = B.CreateAdd(Q, B.CreateZExt(Up, I64));
+ Value *Left = shl(T.M, B.CreateSelect(Right, c32(0), B.CreateNeg(S)));
+ Value *Mant = B.CreateTrunc(B.CreateSelect(Right, Rounded, Left), I32);
+ // The exponent field minus one: the implicit bit of Mant adds the one,
+ // and a rounding carry into bit 24 increments the exponent.
+ Value *Biased = B.CreateAdd(B.CreateAdd(T.E, S), c32(149));
+ Value *Bits = B.CreateAdd(B.CreateShl(Biased, 23), Mant);
+ Value *Over = B.CreateOr(B.CreateICmpSGT(Biased, c32(253)),
+ B.CreateICmpUGE(Bits, c32(0x7f800000)));
+ Bits = B.CreateSelect(Over, c32(0x7f800000), Bits);
+ Bits = B.CreateSelect(B.CreateICmpEQ(T.M, c64(0)), c32(0), Bits);
+ return B.CreateOr(Bits, B.CreateShl(B.CreateZExt(T.Sign, I32), 31));
+ }
+
+ Term mul(const Term &X, const Term &Y) {
+ return {B.CreateXor(X.Sign, Y.Sign), B.CreateMul(X.M, Y.M),
+ B.CreateAdd(X.E, Y.E)};
+ }
+
+ /// X + Y, exact or with a sticky low bit.
+ Term add(const Term &X, const Term &Y) {
+ Value *TopX = B.CreateAdd(bitLength(X.M), X.E);
+ Value *TopY = B.CreateAdd(bitLength(Y.M), Y.E);
+ Value *Swap = B.CreateICmpSGT(TopY, TopX);
+ Value *SB = B.CreateSelect(Swap, Y.Sign, X.Sign);
+ Value *MB = B.CreateSelect(Swap, Y.M, X.M);
+ Value *EB = B.CreateSelect(Swap, Y.E, X.E);
+ Value *SS = B.CreateSelect(Swap, X.Sign, Y.Sign);
+ Value *MS = B.CreateSelect(Swap, X.M, Y.M);
+ Value *ES = B.CreateSelect(Swap, X.E, Y.E);
+ // The larger term with its most significant bit at bit 61.
+ Value *Up = B.CreateSub(c32(62), bitLength(MB));
+ MB = shl(MB, Up);
+ EB = B.CreateSub(EB, Up);
+ Value *D = B.CreateSub(EB, ES);
+ Value *Aligned = B.CreateSelect(B.CreateICmpSGT(D, c32(0)),
+ lshrSticky(MS, D),
+ shl(MS, B.CreateNeg(D)));
+ Value *Same = B.CreateICmpEQ(SB, SS);
+ Value *Sum = B.CreateSelect(Same, B.CreateAdd(MB, Aligned),
+ B.CreateSub(MB, Aligned));
+ Value *Neg = B.CreateICmpSLT(Sum, c64(0));
+ Value *M = B.CreateSelect(Neg, B.CreateNeg(Sum), Sum);
+ // An exact cancellation is +0 when rounding to nearest.
+ Value *Sign = B.CreateAnd(B.CreateXor(SB, Neg), B.CreateICmpNE(M, c64(0)));
+ // Zero terms: the other term, and -0 + -0 = -0.
+ Value *XZero = B.CreateICmpEQ(X.M, c64(0));
+ Value *YZero = B.CreateICmpEQ(Y.M, c64(0));
+ Value *BothZero = B.CreateAnd(XZero, YZero);
+ Sign = B.CreateSelect(BothZero, B.CreateAnd(X.Sign, Y.Sign),
+ B.CreateSelect(XZero, Y.Sign,
+ B.CreateSelect(YZero, X.Sign, Sign)));
+ M = B.CreateSelect(XZero, Y.M, B.CreateSelect(YZero, X.M, M));
+ Value *E = B.CreateSelect(XZero, Y.E, B.CreateSelect(YZero, X.E, EB));
+ return {Sign, M, E};
+ }
+
+ /// A denormal operand replaced by a normal value of the same sign, for the
+ /// hardware operations on infinities and NaNs.
+ Value *nonzeroNormal(Value *X) {
+ Value *Bits = bits(X);
+ Value *One = B.CreateOr(B.CreateAnd(Bits, c32(0x80000000)), c32(0x3f800000));
+ return flt(B.CreateSelect(isDenormal(Bits), One, Bits));
+ }
+
+ /// X * 2^64 for |X| < 2^-62, exact (denormals included).
+ Value *scaleUp64(Value *X) {
+ Value *Bits = bits(X);
+ Value *Frac = B.CreateAnd(Bits, c32(0x7fffff));
+ // A denormal is Frac * 2^-149, so X * 2^64 = Frac * 2^-85.
+ Value *Den = B.CreateFMul(B.CreateUIToFP(Frac, F32), fconst(0x1p-85f));
+ Den = flt(B.CreateOr(bits(Den), B.CreateAnd(Bits, c32(0x80000000))));
+ return B.CreateSelect(isDenormal(Bits), Den,
+ B.CreateFMul(X, fconst(0x1p64f)));
+ }
+
+ Value *fma(Value *X, Value *Y, Value *Z) {
+ return B.CreateIntrinsic(Intrinsic::fma, {F32}, {X, Y, Z});
+ }
+ /// The neighbors of a positive normal value.
+ Value *nextUp(Value *X) { return flt(B.CreateAdd(bits(X), c32(1))); }
+ Value *nextDown(Value *X) { return flt(B.CreateSub(bits(X), c32(1))); }
+
+ /// Division of normal operands with a normal result, using the hardware:
+ /// the mantissas (in [1, 2)) are divided with a refined reciprocal, and
+ /// the quotient is corrected until the exact remainder shows that it is
+ /// the correctly rounded one. Returns the result and whether it is valid.
+ std::pair<Value *, Value *> fastDiv(Value *A, Value *D) {
+ Value *AB = bits(A), *DB = bits(D);
+ Value *EA = B.CreateAnd(B.CreateLShr(AB, 23), c32(0xff));
+ Value *ED = B.CreateAnd(B.CreateLShr(DB, 23), c32(0xff));
+ auto Normal = [&](Value *E) {
+ return B.CreateICmpULT(B.CreateSub(E, c32(1)), c32(254));
+ };
+ Value *Mant = c32(0x7fffff), *One = c32(0x3f800000);
+ Value *A1 = flt(B.CreateOr(B.CreateAnd(AB, Mant), One));
+ Value *D1 = flt(B.CreateOr(B.CreateAnd(DB, Mant), One));
+ Value *Y0 = B.CreateIntrinsic(Intrinsic::agx_rcp, {F32}, {D1});
+ Value *NegD = B.CreateFNeg(D1);
+ Value *Y = fma(fma(NegD, Y0, fconst(1.0f)), Y0, Y0);
+ Value *Q0 = B.CreateFMul(A1, Y);
+ Value *Q = fma(fma(NegD, Q0, A1), Y, Q0);
+ for (int I = 0; I < 2; ++I) {
+ // a/b lies in (q - gap below / 2, q + gap above / 2] around the
+ // correctly rounded q; the remainder r = a - b * q is exact.
+ Value *Up = nextUp(Q), *Down = nextDown(Q);
+ Value *R = fma(NegD, Q, A1);
+ Value *HalfUp = B.CreateFMul(D1, B.CreateFMul(B.CreateFSub(Up, Q), fconst(0.5f)));
+ Value *HalfDown = B.CreateFMul(D1, B.CreateFMul(B.CreateFSub(Q, Down), fconst(0.5f)));
+ Value *Odd = B.CreateTrunc(bits(Q), I1);
+ Value *GoUp = B.CreateOr(B.CreateFCmpOGT(R, HalfUp),
+ B.CreateAnd(B.CreateFCmpOEQ(R, HalfUp), Odd));
+ Value *NegHalfDown = B.CreateFNeg(HalfDown);
+ Value *GoDown = B.CreateOr(B.CreateFCmpOLT(R, NegHalfDown),
+ B.CreateAnd(B.CreateFCmpOEQ(R, NegHalfDown), Odd));
+ Q = B.CreateSelect(GoUp, Up, B.CreateSelect(GoDown, Down, Q));
+ }
+ Value *QB = bits(Q);
+ Value *Shift = B.CreateSub(EA, ED);
+ Value *Exp = B.CreateAdd(B.CreateAnd(B.CreateLShr(QB, 23), c32(0xff)), Shift);
+ Value *Valid = B.CreateAnd(B.CreateAnd(Normal(EA), Normal(ED)), Normal(Exp));
+ Value *Sign = B.CreateAnd(B.CreateXor(AB, DB), c32(0x80000000));
+ Value *R = B.CreateOr(B.CreateAdd(QB, B.CreateShl(Shift, 23)), Sign);
+ return {flt(R), Valid};
+ }
+
+ /// Square root of a positive normal value, using the hardware: the root of
+ /// the mantissa (in [1, 4)) is refined from rsqrt and corrected until the
+ /// midpoint test of Tuckerman shows that it is correctly rounded.
+ std::pair<Value *, Value *> fastSqrt(Value *X) {
+ Value *XB = bits(X);
+ Value *Valid = B.CreateICmpULT(B.CreateSub(XB, c32(0x00800000)),
+ c32(0x7f000000));
+ Value *K = B.CreateSub(B.CreateAnd(B.CreateLShr(XB, 23), c32(0xff)), c32(127));
+ Value *Odd = B.CreateAnd(K, c32(1));
+ Value *Exp = B.CreateSelect(B.CreateICmpNE(Odd, c32(0)), c32(0x40000000),
+ c32(0x3f800000));
+ Value *M = flt(B.CreateOr(B.CreateAnd(XB, c32(0x7fffff)), Exp));
+ Value *NegM = B.CreateFNeg(M);
+ Value *Y = B.CreateIntrinsic(Intrinsic::agx_rsqrt, {F32}, {M});
+ Value *S0 = B.CreateFMul(M, Y);
+ Value *S = fma(fma(B.CreateFNeg(S0), S0, M), B.CreateFMul(Y, fconst(0.5f)), S0);
+ for (int I = 0; I < 2; ++I) {
+ // s is correctly rounded iff s * down(s) < m <= s * up(s) (there are no
+ // ties for square roots).
+ Value *Up = nextUp(S), *Down = nextDown(S);
+ Value *Low = B.CreateFCmpOGE(fma(S, Down, NegM), fconst(0.0f));
+ Value *High = B.CreateFCmpOLT(fma(S, Up, NegM), fconst(0.0f));
+ S = B.CreateSelect(High, Up, B.CreateSelect(Low, Down, S));
+ }
+ Value *Half = B.CreateAShr(B.CreateSub(K, Odd), 1);
+ Value *R = B.CreateAdd(bits(S), B.CreateShl(Half, 23));
+ return {flt(R), Valid};
+ }
+
+ /// IEEE division of finite or special operands, correctly rounded.
+ Value *div(Value *A, Value *D) {
+ Value *AB = bits(A), *DB = bits(D);
+ Term TA = unpack(AB, true), TD = unpack(DB, true);
+ // Q = floor(MA * 2^26 / MD), with MA, MD in [2^23, 2^24): 26 or 27 bits.
+ // A floating point estimate is exact to a few units; two integer
+ // corrections make it exact.
+ Value *X = B.CreateShl(TA.M, 26);
+ Value *FD = B.CreateUIToFP(TD.M, F32);
+ Value *Inv = B.CreateIntrinsic(Intrinsic::agx_rcp, {F32}, {FD});
+ Value *Est = B.CreateFMul(B.CreateFMul(B.CreateUIToFP(TA.M, F32), Inv),
+ fconst(0x1p26f));
+ Value *Q = B.CreateZExt(B.CreateFPToUI(Est, I32), I64);
+ Value *R = B.CreateSub(X, B.CreateMul(Q, TD.M));
+ Value *C = B.CreateFPToSI(
+ B.CreateFMul(B.CreateSIToFP(R, F32), Inv), I32);
+ Q = B.CreateAdd(Q, B.CreateSExt(C, I64));
+ R = B.CreateSub(R, B.CreateMul(B.CreateSExt(C, I64), TD.M));
+ for (int I = 0; I < 2; ++I) {
+ Value *Low = B.CreateICmpSLT(R, c64(0));
+ Q = B.CreateSelect(Low, B.CreateSub(Q, c64(1)), Q);
+ R = B.CreateSelect(Low, B.CreateAdd(R, TD.M), R);
+ Value *High = B.CreateICmpSGE(R, TD.M);
+ Q = B.CreateSelect(High, B.CreateAdd(Q, c64(1)), Q);
+ R = B.CreateSelect(High, B.CreateSub(R, TD.M), R);
+ }
+ Value *M = B.CreateOr(B.CreateShl(Q, 1),
+ B.CreateZExt(B.CreateICmpNE(R, c64(0)), I64));
+ Value *E = B.CreateSub(B.CreateSub(TA.E, TD.E), c32(27));
+ Value *Sign = B.CreateXor(TA.Sign, TD.Sign);
+ Value *Bits = convert({Sign, M, E});
+
+ // Zeros, infinities and NaNs.
+ Value *SignBits = B.CreateShl(B.CreateZExt(Sign, I32), 31);
+ Value *AZero = isZero(AB), *DZero = isZero(DB);
+ Value *AInf = B.CreateICmpEQ(B.CreateAnd(AB, c32(0x7fffffff)), c32(0x7f800000));
+ Value *DInf = B.CreateICmpEQ(B.CreateAnd(DB, c32(0x7fffffff)), c32(0x7f800000));
+ Value *NaN = B.CreateOr(B.CreateFCmpUNO(A, D),
+ B.CreateOr(B.CreateAnd(AZero, DZero),
+ B.CreateAnd(AInf, DInf)));
+ Value *Inf = B.CreateOr(AInf, DZero);
+ Value *Zero = B.CreateOr(AZero, DInf);
+ Bits = B.CreateSelect(Zero, SignBits, Bits);
+ Bits = B.CreateSelect(Inf, B.CreateOr(SignBits, c32(0x7f800000)), Bits);
+ Bits = B.CreateSelect(NaN, c32(0x7fc00000), Bits);
+ return flt(Bits);
+ }
+
+ /// IEEE square root, correctly rounded.
+ Value *sqrt(Value *X) {
+ Value *XB = bits(X);
+ Term T = unpack(XB, true);
+ // An even exponent: M in [2^23, 2^25).
+ Value *Odd = B.CreateTrunc(T.E, I1);
+ Value *M = B.CreateSelect(Odd, B.CreateShl(T.M, 1), T.M);
+ Value *E = B.CreateSelect(Odd, B.CreateSub(T.E, c32(1)), T.E);
+ // S = floor(sqrt(M * 2^28)), 26 or 27 bits.
+ Value *W = B.CreateShl(M, 28);
+ Value *FM = B.CreateUIToFP(M, F32);
+ Value *Root = B.CreateFMul(
+ FM, B.CreateIntrinsic(Intrinsic::agx_rsqrt, {F32}, {FM}));
+ Value *S = B.CreateZExt(
+ B.CreateFPToUI(B.CreateFMul(Root, fconst(0x1p14f)), I32), I64);
+ Value *R = B.CreateSub(W, B.CreateMul(S, S));
+ Value *Half = B.CreateIntrinsic(
+ Intrinsic::agx_rcp, {F32},
+ {B.CreateFMul(B.CreateUIToFP(S, F32), fconst(2.0f))});
+ Value *C = B.CreateFPToSI(B.CreateFMul(B.CreateSIToFP(R, F32), Half), I32);
+ S = B.CreateAdd(S, B.CreateSExt(C, I64));
+ R = B.CreateSub(W, B.CreateMul(S, S));
+ for (int I = 0; I < 2; ++I) {
+ Value *Low = B.CreateICmpSLT(R, c64(0));
+ R = B.CreateSelect(Low, B.CreateAdd(R, B.CreateSub(B.CreateShl(S, 1),
+ c64(1))),
+ R);
+ S = B.CreateSelect(Low, B.CreateSub(S, c64(1)), S);
+ Value *High = B.CreateICmpSGT(R, B.CreateShl(S, 1));
+ R = B.CreateSelect(High, B.CreateSub(R, B.CreateAdd(B.CreateShl(S, 1),
+ c64(1))),
+ R);
+ S = B.CreateSelect(High, B.CreateAdd(S, c64(1)), S);
+ }
+ Value *Root2 = B.CreateOr(B.CreateShl(S, 1),
+ B.CreateZExt(B.CreateICmpNE(R, c64(0)), I64));
+ Value *RE = B.CreateSub(B.CreateAShr(B.CreateSub(E, c32(28)), 1), c32(1));
+ Value *Bits = convert({B.getFalse(), Root2, RE});
+ // sqrt(+-0) = +-0, sqrt(+inf) = +inf, NaN for negative values and NaNs.
+ Bits = B.CreateSelect(B.CreateICmpEQ(XB, c32(0x7f800000)), XB, Bits);
+ Bits = B.CreateSelect(
+ B.CreateOr(B.CreateFCmpUNO(X, X),
+ B.CreateAnd(signBit(XB), B.CreateNot(isZero(XB)))),
+ c32(0x7fc00000), Bits);
+ Bits = B.CreateSelect(isZero(XB), XB, Bits);
+ return flt(Bits);
+ }
+
+ /// A comparison operand as a signed integer, for zeros and denormals.
+ Value *denormalOrder(Value *Bits) {
+ Value *Mag = B.CreateAnd(Bits, c32(0x7fffff));
+ return B.CreateSelect(signBit(Bits), B.CreateNeg(Mag), Mag);
+ }
+
+ /// fcmp that orders denormals.
+ Value *compare(CmpInst::Predicate P, Value *X, Value *Y, Value *HW) {
+ Value *XB = bits(X), *YB = bits(Y);
+ Value *Both = B.CreateAnd(expZero(XB), expZero(YB));
+ Value *VX = denormalOrder(XB), *VY = denormalOrder(YB);
+ Value *Int;
+ switch (P) {
+ case CmpInst::FCMP_OEQ:
+ case CmpInst::FCMP_UEQ:
+ Int = B.CreateICmpEQ(VX, VY);
+ break;
+ case CmpInst::FCMP_ONE:
+ case CmpInst::FCMP_UNE:
+ Int = B.CreateICmpNE(VX, VY);
+ break;
+ case CmpInst::FCMP_OLT:
+ case CmpInst::FCMP_ULT:
+ Int = B.CreateICmpSLT(VX, VY);
+ break;
+ case CmpInst::FCMP_OLE:
+ case CmpInst::FCMP_ULE:
+ Int = B.CreateICmpSLE(VX, VY);
+ break;
+ case CmpInst::FCMP_OGT:
+ case CmpInst::FCMP_UGT:
+ Int = B.CreateICmpSGT(VX, VY);
+ break;
+ case CmpInst::FCMP_OGE:
+ case CmpInst::FCMP_UGE:
+ Int = B.CreateICmpSGE(VX, VY);
+ break;
+ default:
+ return HW; // ord, uno, true, false: denormals change nothing.
+ }
+ return B.CreateSelect(Both, Int, HW);
+ }
+
+ /// X < Y, with denormals.
+ Value *less(Value *X, Value *Y) {
+ return compare(CmpInst::FCMP_OLT, X, Y, B.CreateFCmpOLT(X, Y));
+ }
+
+ /// minimumnum / maximumnum (and minnum / maxnum): a NaN operand is ignored,
+ /// -0 < +0. With Propagate (minimum / maximum), NaN operands give NaN.
+ Value *minMax(Value *X, Value *Y, bool Min, bool Propagate) {
+ Value *XB = bits(X), *YB = bits(Y);
+ Value *Lt = less(X, Y), *Gt = less(Y, X);
+ Value *Pick = Min ? B.CreateSelect(Lt, XB, B.CreateSelect(Gt, YB, B.CreateOr(XB, YB)))
+ : B.CreateSelect(Gt, XB, B.CreateSelect(Lt, YB, B.CreateAnd(XB, YB)));
+ Value *XNaN = B.CreateFCmpUNO(X, X), *YNaN = B.CreateFCmpUNO(Y, Y);
+ if (Propagate) {
+ Pick = B.CreateSelect(YNaN, YB, Pick);
+ Pick = B.CreateSelect(XNaN, XB, Pick);
+ } else {
+ Pick = B.CreateSelect(YNaN, XB, Pick);
+ Pick = B.CreateSelect(XNaN, YB, Pick);
+ }
+ return flt(Pick);
+ }
+};
+
+} // namespace
+
+char AGXStrictFP::ID = 0;
+
+INITIALIZE_PASS_BEGIN(AGXStrictFP, DEBUG_TYPE, "AGX Strict FP", false, false)
+INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
+INITIALIZE_PASS_END(AGXStrictFP, DEBUG_TYPE, "AGX Strict FP", false, false)
+
+FunctionPass *llvm::createAGXStrictFPPass() { return new AGXStrictFP(); }
+
+static bool isF32(const Value *V) { return V->getType()->isFloatTy(); }
+
+static Intrinsic::ID getIntrinsicID(const Instruction &I) {
+ if (const auto *II = dyn_cast<IntrinsicInst>(&I))
+ return II->getIntrinsicID();
+ return Intrinsic::not_intrinsic;
+}
+
+/// Floating point operations whose NaN results are canonicalized.
+static bool producesArithmeticNaN(const Instruction &I) {
+ if (!I.getType()->isFPOrFPVectorTy())
+ return false;
+ switch (I.getOpcode()) {
+ case Instruction::FAdd:
+ case Instruction::FSub:
+ case Instruction::FMul:
+ case Instruction::FDiv:
+ case Instruction::FRem:
+ case Instruction::FPExt:
+ case Instruction::FPTrunc:
+ return true;
+ default:
+ break;
+ }
+ switch (getIntrinsicID(I)) {
+ case Intrinsic::fma:
+ case Intrinsic::fmuladd:
+ case Intrinsic::sqrt:
+ case Intrinsic::exp2:
+ case Intrinsic::log2:
+ case Intrinsic::sin:
+ case Intrinsic::cos:
+ case Intrinsic::floor:
+ case Intrinsic::ceil:
+ case Intrinsic::trunc:
+ case Intrinsic::rint:
+ case Intrinsic::nearbyint:
+ case Intrinsic::roundeven:
+ case Intrinsic::round:
+ case Intrinsic::minnum:
+ case Intrinsic::maxnum:
+ case Intrinsic::minimum:
+ case Intrinsic::maximum:
+ case Intrinsic::minimumnum:
+ case Intrinsic::maximumnum:
+ case Intrinsic::canonicalize:
+ case Intrinsic::agx_rcp:
+ case Intrinsic::agx_rsqrt:
+ return true;
+ default:
+ return false;
+ }
+}
+
+/// Replace the NaN results of I by the canonical NaN.
+static void canonicalizeNaN(Instruction &I) {
+ Type *Ty = I.getType();
+ const fltSemantics &Sem = Ty->getScalarType()->getFltSemantics();
+ unsigned Width = APFloat::getSizeInBits(Sem);
+ APFloat NaN(Sem, APInt::getSignedMaxValue(Width));
+ IRBuilder<> B(I.getNextNode());
+ SmallVector<Use *, 4> Uses;
+ for (Use &U : I.uses())
+ Uses.push_back(&U);
+ Value *R = B.CreateSelect(B.CreateFCmpUNO(&I, &I), ConstantFP::get(Ty, NaN),
+ &I);
+ for (Use *U : Uses)
+ U->set(R);
+}
+
+/// Replace I by Fast where Slow is false, and by the value computed by Build
+/// in a conditional block otherwise. Cond is built right after I.
+template <typename CondFn, typename SlowFn>
+static void lowerWithSlowPath(Instruction &I, CondFn BuildCond,
+ SlowFn BuildSlow) {
+ SmallVector<Use *, 4> Uses;
+ for (Use &U : I.uses())
+ Uses.push_back(&U);
+ Instruction *Next = I.getNextNode();
+ IRBuilder<> B(Next);
+ SoftFloat SF(B);
+ Value *Cond = BuildCond(SF, B);
+ Instruction *ThenTerm = SplitBlockAndInsertIfThen(Cond, Next, false);
+ B.SetInsertPoint(ThenTerm);
+ Value *Slow = BuildSlow(SF, B);
+ BasicBlock *Then = ThenTerm->getParent();
+ BasicBlock *Head = Then->getSinglePredecessor();
+ B.SetInsertPoint(Next->getParent(), Next->getParent()->begin());
+ PHINode *Phi = B.CreatePHI(I.getType(), 2);
+ Phi->addIncoming(&I, Head);
+ Phi->addIncoming(Slow, Then);
+ for (Use *U : Uses)
+ U->set(Phi);
+}
+
+/// Replace I by the result of BuildFast, or of BuildSlow (in a conditional
+/// block) where BuildFast says that its result is not valid.
+template <typename FastFn, typename SlowFn>
+static void replaceWithSlowPath(Instruction &I, FastFn BuildFast,
+ SlowFn BuildSlow) {
+ Instruction *Next = I.getNextNode();
+ IRBuilder<> B(Next);
+ SoftFloat SF(B);
+ auto [Fast, Valid] = BuildFast(SF, B);
+ Instruction *ThenTerm =
+ SplitBlockAndInsertIfThen(B.CreateNot(Valid), Next, false);
+ B.SetInsertPoint(ThenTerm);
+ Value *Slow = BuildSlow(SF, B);
+ BasicBlock *Then = ThenTerm->getParent();
+ BasicBlock *Head = Then->getSinglePredecessor();
+ B.SetInsertPoint(Next->getParent(), Next->getParent()->begin());
+ PHINode *Phi = B.CreatePHI(I.getType(), 2);
+ Phi->addIncoming(Fast, Head);
+ Phi->addIncoming(Slow, Then);
+ Phi->takeName(&I);
+ I.replaceAllUsesWith(Phi);
+ I.eraseFromParent();
+}
+
+/// Replace I by the value computed by Build right after it.
+template <typename BuildFn> static void replace(Instruction &I, BuildFn Build) {
+ IRBuilder<> B(I.getNextNode());
+ SoftFloat SF(B);
+ Value *R = Build(SF, B);
+ R->takeName(&I);
+ I.replaceAllUsesWith(R);
+ I.eraseFromParent();
+}
+
+static bool lowerIEEE(Instruction &I) {
+ Intrinsic::ID ID = getIntrinsicID(I);
+ bool Approx = isa<FPMathOperator>(I) &&
+ (I.hasApproxFunc() || I.hasAllowReciprocal());
+
+ switch (I.getOpcode()) {
+ case Instruction::FAdd:
+ case Instruction::FSub: {
+ if (!isF32(&I))
+ return false;
+ Value *X = I.getOperand(0), *Y = I.getOperand(1);
+ bool Sub = I.getOpcode() == Instruction::FSub;
+ // The hardware result is right unless both operands are below 2^-101:
+ // otherwise the result is above 2^-102, so a denormal operand is below
+ // half an ulp of it, and a cancellation of normal operands cannot reach
+ // the denormal range.
+ lowerWithSlowPath(
+ I,
+ [&](SoftFloat &SF, IRBuilder<> &B) {
+ Value *Limit = SF.fconst(0x1p-101f);
+ auto Small = [&](Value *V) {
+ return B.CreateFCmpOLT(
+ B.CreateUnaryIntrinsic(Intrinsic::fabs, V), Limit);
+ };
+ return B.CreateAnd(Small(X), Small(Y));
+ },
+ [&](SoftFloat &SF, IRBuilder<> &B) {
+ Term TX = SF.unpack(SF.bits(X), false);
+ Term TY = SF.unpack(SF.bits(Y), false);
+ if (Sub)
+ TY.Sign = B.CreateNot(TY.Sign);
+ return SF.flt(SF.convert(SF.add(TX, TY)));
+ });
+ return true;
+ }
+ case Instruction::FMul: {
+ if (!isF32(&I))
+ return false;
+ Value *X = I.getOperand(0), *Y = I.getOperand(1);
+ lowerWithSlowPath(
+ I,
+ [&](SoftFloat &SF, IRBuilder<> &B) {
+ // A product at most 2^-125 (or NaN) of nonzero operands.
+ Value *Tiny = B.CreateFCmpULE(
+ B.CreateUnaryIntrinsic(Intrinsic::fabs, &I),
+ SF.fconst(0x1p-125f));
+ return B.CreateAnd(Tiny, B.CreateNot(B.CreateOr(
+ SF.isZero(SF.bits(X)),
+ SF.isZero(SF.bits(Y)))));
+ },
+ [&](SoftFloat &SF, IRBuilder<> &B) {
+ Value *XB = SF.bits(X), *YB = SF.bits(Y);
+ Value *Exact = SF.flt(SF.convert(
+ SF.mul(SF.unpack(XB, false), SF.unpack(YB, false))));
+ Value *Special = B.CreateFMul(SF.nonzeroNormal(X), SF.nonzeroNormal(Y));
+ return B.CreateSelect(B.CreateOr(SF.isInfOrNaN(XB), SF.isInfOrNaN(YB)),
+ Special, Exact);
+ });
+ return true;
+ }
+ case Instruction::FDiv: {
+ if (!isF32(&I) || Approx)
+ return false;
+ Value *X = I.getOperand(0), *Y = I.getOperand(1);
+ replaceWithSlowPath(
+ I, [&](SoftFloat &SF, IRBuilder<> &) { return SF.fastDiv(X, Y); },
+ [&](SoftFloat &SF, IRBuilder<> &) { return SF.div(X, Y); });
+ return true;
+ }
+ case Instruction::FCmp: {
+ if (!isF32(I.getOperand(0)))
+ return false;
+ auto *Cmp = cast<FCmpInst>(&I);
+ Value *X = Cmp->getOperand(0), *Y = Cmp->getOperand(1);
+ CmpInst::Predicate P = Cmp->getPredicate();
+ SmallVector<Use *, 4> Uses;
+ for (Use &U : I.uses())
+ Uses.push_back(&U);
+ IRBuilder<> B(I.getNextNode());
+ SoftFloat SF(B);
+ Value *R = SF.compare(P, X, Y, &I);
+ if (R == &I)
+ return false;
+ for (Use *U : Uses)
+ U->set(R);
+ return true;
+ }
+ default:
+ break;
+ }
+
+ if (!isF32(&I))
+ return false;
+ auto *II = dyn_cast<IntrinsicInst>(&I);
+ switch (ID) {
+ case Intrinsic::fma:
+ case Intrinsic::fmuladd: {
+ Value *X = II->getArgOperand(0), *Y = II->getArgOperand(1),
+ *Z = II->getArgOperand(2);
+ lowerWithSlowPath(
+ I,
+ [&](SoftFloat &SF, IRBuilder<> &B) {
+ Value *XB = SF.bits(X), *YB = SF.bits(Y), *ZB = SF.bits(Z);
+ Value *Den = B.CreateOr(SF.isDenormal(XB),
+ B.CreateOr(SF.isDenormal(YB), SF.isDenormal(ZB)));
+ Value *Tiny = B.CreateFCmpULE(
+ B.CreateUnaryIntrinsic(Intrinsic::fabs, &I), SF.fconst(0x1p-125f));
+ Tiny = B.CreateAnd(Tiny, B.CreateNot(B.CreateOr(SF.isZero(XB),
+ SF.isZero(YB))));
+ return B.CreateOr(Den, Tiny);
+ },
+ [&](SoftFloat &SF, IRBuilder<> &B) {
+ Value *XB = SF.bits(X), *YB = SF.bits(Y), *ZB = SF.bits(Z);
+ Term P = SF.mul(SF.unpack(XB, false), SF.unpack(YB, false));
+ Value *Exact = SF.flt(SF.convert(SF.add(P, SF.unpack(ZB, false))));
+ Value *Special = B.CreateIntrinsic(
+ Intrinsic::fma, {B.getFloatTy()},
+ {SF.nonzeroNormal(X), SF.nonzeroNormal(Y), SF.nonzeroNormal(Z)});
+ Value *AnySpecial = B.CreateOr(
+ SF.isInfOrNaN(XB), B.CreateOr(SF.isInfOrNaN(YB), SF.isInfOrNaN(ZB)));
+ return B.CreateSelect(AnySpecial, Special, Exact);
+ });
+ return true;
+ }
+ case Intrinsic::sqrt: {
+ if (Approx)
+ return false;
+ Value *X = II->getArgOperand(0);
+ replaceWithSlowPath(
+ I, [&](SoftFloat &SF, IRBuilder<> &) { return SF.fastSqrt(X); },
+ [&](SoftFloat &SF, IRBuilder<> &) { return SF.sqrt(X); });
+ return true;
+ }
+ case Intrinsic::minnum:
+ case Intrinsic::maxnum:
+ case Intrinsic::minimumnum:
+ case Intrinsic::maximumnum:
+ case Intrinsic::minimum:
+ case Intrinsic::maximum: {
+ Value *X = II->getArgOperand(0), *Y = II->getArgOperand(1);
+ bool Min = ID == Intrinsic::minnum || ID == Intrinsic::minimumnum ||
+ ID == Intrinsic::minimum;
+ bool Propagate = ID == Intrinsic::minimum || ID == Intrinsic::maximum;
+ replace(I, [&](SoftFloat &SF, IRBuilder<> &) {
+ return SF.minMax(X, Y, Min, Propagate);
+ });
+ return true;
+ }
+ case Intrinsic::floor:
+ case Intrinsic::ceil:
+ case Intrinsic::trunc:
+ case Intrinsic::rint:
+ case Intrinsic::nearbyint:
+ case Intrinsic::roundeven:
+ case Intrinsic::round: {
+ // Denormals round to zero, except away from zero for floor and ceil.
+ Value *X = II->getArgOperand(0);
+ SmallVector<Use *, 4> Uses;
+ for (Use &U : I.uses())
+ Uses.push_back(&U);
+ IRBuilder<> B(I.getNextNode());
+ SoftFloat SF(B);
+ Value *XB = SF.bits(X);
+ Value *Sign = B.CreateAnd(XB, SF.c32(0x80000000));
+ Value *Fix = Sign;
+ if (ID == Intrinsic::floor)
+ Fix = B.CreateSelect(SF.signBit(XB), SF.c32(0xbf800000), SF.c32(0));
+ if (ID == Intrinsic::ceil)
+ Fix = B.CreateSelect(SF.signBit(XB), SF.c32(0x80000000), SF.c32(0x3f800000));
+ Value *R = B.CreateSelect(SF.isDenormal(XB), SF.flt(Fix), &I);
+ for (Use *U : Uses)
+ U->set(R);
+ return true;
+ }
+ case Intrinsic::agx_rcp:
+ case Intrinsic::agx_rsqrt:
+ case Intrinsic::log2: {
+ // Denormal operands are scaled by 2^64 first.
+ Value *X = II->getArgOperand(0);
+ SmallVector<Use *, 4> Uses;
+ for (Use &U : I.uses())
+ Uses.push_back(&U);
+ IRBuilder<> B(I.getNextNode());
+ SoftFloat SF(B);
+ Value *Scaled = B.CreateIntrinsic(ID, {B.getFloatTy()}, {SF.scaleUp64(X)});
+ Value *Fixed;
+ if (ID == Intrinsic::agx_rcp)
+ Fixed = B.CreateFMul(Scaled, SF.fconst(0x1p64f));
+ else if (ID == Intrinsic::agx_rsqrt)
+ Fixed = B.CreateFMul(Scaled, SF.fconst(0x1p32f));
+ else
+ Fixed = B.CreateFSub(Scaled, SF.fconst(64.0f));
+ Value *R = B.CreateSelect(SF.isDenormal(SF.bits(X)), Fixed, &I);
+ for (Use *U : Uses)
+ U->set(R);
+ return true;
+ }
+ case Intrinsic::exp2: {
+ // Results below 2^-126 are denormals: 2^(x + 64) * 2^-64.
+ Value *X = II->getArgOperand(0);
+ SmallVector<Use *, 4> Uses;
+ for (Use &U : I.uses())
+ Uses.push_back(&U);
+ IRBuilder<> B(I.getNextNode());
+ SoftFloat SF(B);
+ Value *Scaled = B.CreateIntrinsic(Intrinsic::exp2, {B.getFloatTy()},
+ {B.CreateFAdd(X, SF.fconst(64.0f))});
+ Value *N = B.CreateUnaryIntrinsic(
+ Intrinsic::rint, B.CreateFMul(Scaled, SF.fconst(0x1p85f)));
+ Value *Den = SF.flt(B.CreateFPToUI(N, B.getInt32Ty()));
+ Value *InRange = B.CreateAnd(B.CreateFCmpOLT(X, SF.fconst(-126.0f)),
+ B.CreateFCmpOGT(X, SF.fconst(-151.0f)));
+ Value *R = B.CreateSelect(InRange, Den, &I);
+ for (Use *U : Uses)
+ U->set(R);
+ return true;
+ }
+ default:
+ return false;
+ }
+}
+
+bool AGXStrictFP::runOnFunction(Function &F) {
+ if (skipFunction(F))
+ return false;
+ const TargetMachine &TM =
+ getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
+ const auto &ST = TM.getSubtarget<AGXSubtarget>(F);
+ bool IEEE = ST.hasIEEEFP32() &&
+ F.getDenormalMode(APFloat::IEEEsingle()).Output ==
+ DenormalMode::IEEE;
+ bool CanonicalNaN = ST.hasCanonicalNaN();
+ if (!IEEE && !CanonicalNaN)
+ return false;
+
+ SmallVector<Instruction *, 32> Worklist;
+ for (Instruction &I : instructions(F))
+ Worklist.push_back(&I);
+
+ bool Changed = false;
+ // NaN canonicalization first: the IEEE lowering then replaces the
+ // canonicalized value.
+ if (CanonicalNaN)
+ for (Instruction *I : Worklist)
+ if (producesArithmeticNaN(*I)) {
+ canonicalizeNaN(*I);
+ Changed = true;
+ }
+ if (IEEE)
+ for (Instruction *I : Worklist)
+ Changed |= lowerIEEE(*I);
+ return Changed;
+}
diff --git a/llvm/lib/Target/AGX/AGXTargetMachine.cpp b/llvm/lib/Target/AGX/AGXTargetMachine.cpp
index e80d4d26f5484..59869105122b0 100644
--- a/llvm/lib/Target/AGX/AGXTargetMachine.cpp
+++ b/llvm/lib/Target/AGX/AGXTargetMachine.cpp
@@ -36,6 +36,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAGXTarget() {
PassRegistry &PR = *PassRegistry::getPassRegistry();
initializeAGXAlwaysInlinePass(PR);
initializeAGXCodeGenPreparePass(PR);
+ initializeAGXStrictFPPass(PR);
initializeAGXAnnotateControlFlowPass(PR);
initializeAGXDAGToDAGISelLegacyPass(PR);
initializeAGXFoldOperandsPass(PR);
@@ -149,6 +150,8 @@ void AGXPassConfig::addIRPasses() {
addPass(createSROAPass());
}
addPass(createAGXCodeGenPreparePass());
+ // Software IEEE binary32 arithmetic and canonical NaNs, when enabled.
+ addPass(createAGXStrictFPPass());
TargetPassConfig::addIRPasses();
}
diff --git a/llvm/lib/Target/AGX/CMakeLists.txt b/llvm/lib/Target/AGX/CMakeLists.txt
index 4761716dc456b..0c5515668c967 100644
--- a/llvm/lib/Target/AGX/CMakeLists.txt
+++ b/llvm/lib/Target/AGX/CMakeLists.txt
@@ -31,6 +31,7 @@ add_llvm_target(AGXCodeGen
AGXMCInstLower.cpp
AGXRegisterInfo.cpp
AGXSelectionDAGInfo.cpp
+ AGXStrictFP.cpp
AGXSubtarget.cpp
AGXTargetMachine.cpp
AGXTargetTransformInfo.cpp
diff --git a/llvm/test/CodeGen/AGX/strict-fp.ll b/llvm/test/CodeGen/AGX/strict-fp.ll
new file mode 100644
index 0000000000000..86430ad803242
--- /dev/null
+++ b/llvm/test/CodeGen/AGX/strict-fp.ll
@@ -0,0 +1,60 @@
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -mattr=+ieee-fp32,+canonical-nan -stop-after=agx-strict-fp < %s | FileCheck %s
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -mattr=+ieee-fp32,+canonical-nan -verify-machineinstrs < %s -o /dev/null
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -stop-after=agx-strict-fp < %s | FileCheck %s --check-prefix=DEFAULT
+
+; ieee-fp32: additions keep the hardware instruction and recompute the result
+; in a conditional block when both operands are below 2^-101; division is
+; computed with integers, comparisons order denormals.
+; canonical-nan: NaN results become 0x7fffffff.
+
+; CHECK-LABEL: define agx_kernel void @ops(
+; CHECK: %add = fadd float %a, %b
+; CHECK: fcmp olt float {{.*}}, f0x0D000000
+; CHECK: br i1 {{.*}}, label %[[SLOW:[0-9]+]], label %[[JOIN:[0-9]+]]
+; CHECK: [[JOIN]]:
+; CHECK-NEXT: %[[SUM:[0-9]+]] = phi float [ %add, %0 ], [ %{{[0-9]+}}, %[[SLOW]] ]
+; CHECK-NEXT: %[[NAN:[0-9]+]] = fcmp uno float %[[SUM]], %[[SUM]]
+; CHECK-NEXT: select i1 %[[NAN]], float +nan(0x3FFFFF), float %[[SUM]]
+; CHECK-NOT: fdiv
+; CHECK: call float @llvm.agx.rcp.f32(
+; CHECK: select i1 {{.*}}, float +nan(0x3FFFFF), float %div
+; CHECK: %cmp = fcmp olt float
+; CHECK: icmp slt i32
+; CHECK: select i1 {{.*}}, i1 {{.*}}, i1 %cmp
+
+; DEFAULT-LABEL: define agx_kernel void @ops(
+; DEFAULT-NEXT: %add = fadd float %a, %b
+; DEFAULT-NEXT: %div = fdiv float %add, %c
+; DEFAULT-NEXT: %cmp = fcmp olt float %div, %a
+define agx_kernel void @ops(ptr addrspace(1) %out, float %a, float %b, float %c) {
+ %add = fadd float %a, %b
+ %div = fdiv float %add, %c
+ %cmp = fcmp olt float %div, %a
+ %sel = select i1 %cmp, float %div, float %b
+ store float %sel, ptr addrspace(1) %out
+ ret void
+}
+
+; Functions that flush float denormals use the hardware operations; NaNs are
+; still canonicalized.
+; CHECK-LABEL: define agx_kernel void @ftz(
+; CHECK-NEXT: %add = fadd float %a, %b
+; CHECK-NEXT: %[[N:[0-9]+]] = fcmp uno float %add, %add
+; CHECK-NEXT: %[[R:[0-9]+]] = select i1 %[[N]], float +nan(0x3FFFFF), float %add
+; CHECK-NEXT: store float %[[R]]
+define agx_kernel void @ftz(ptr addrspace(1) %out, float %a, float %b) #0 {
+ %add = fadd float %a, %b
+ store float %add, ptr addrspace(1) %out
+ ret void
+}
+
+; Half NaNs become 0x7fff.
+; CHECK-LABEL: define agx_kernel void @half(
+; CHECK: select i1 {{.*}}, half +nan(0x1FF), half %m
+define agx_kernel void @half(ptr addrspace(1) %out, half %a, half %b) {
+ %m = fmul half %a, %b
+ store half %m, ptr addrspace(1) %out
+ ret void
+}
+
+attributes #0 = { denormal_fpenv(float: preservesign) }
>From fad8724b0324533e9222390d48ff9207928aff13 Mon Sep 17 00:00:00 2001
From: Claude <noreply at anthropic.com>
Date: Mon, 19 Oct 2026 00:20:41 +0000
Subject: [PATCH 12/22] [AGX] Emulate directed rounding modes and a floating
point mode register
llvm.agx.set.fpmode/llvm.agx.get.fpmode access an emulated mode register
(f32 and f16 rounding modes, f32 and f16 flush bits). The new
agx-lower-fp-env pass keeps it in SSA values and lowers the constrained
floating point intrinsics: round to nearest becomes an ordinary operation,
the directed modes are computed exactly with the SoftFloat integer core
(add, sub, mul, div, fma, sqrt, fptrunc to half, int to float, rounding
functions), and an operation whose denormal mode differs from its
function's flushes its operands and result explicitly.
With ieee-fp32, fdiv and sqrt in flushing functions are correctly rounded
too. The default fdiv lowering no longer turns infinite and zero quotients
into NaN and +0 in its Newton-Raphson step.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
Claude-Session: https://claude.ai/code/session_01E56KEecg6edNaS23myRWER
---
llvm/docs/AGXUsage.rst | 33 ++
llvm/include/llvm/IR/IntrinsicsAGX.td | 23 +
llvm/lib/Target/AGX/AGX.h | 2 +
llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp | 3 +
llvm/lib/Target/AGX/AGXISelLowering.cpp | 14 +-
llvm/lib/Target/AGX/AGXStrictFP.cpp | 603 ++++++++++++++++++++--
llvm/lib/Target/AGX/AGXTargetMachine.cpp | 4 +
llvm/test/CodeGen/AGX/fp-env.ll | 57 ++
llvm/test/CodeGen/AGX/fp.ll | 9 +-
9 files changed, 696 insertions(+), 52 deletions(-)
create mode 100644 llvm/test/CodeGen/AGX/fp-env.ll
diff --git a/llvm/docs/AGXUsage.rst b/llvm/docs/AGXUsage.rst
index 5c3b7d83c523f..6d13e0dd43961 100644
--- a/llvm/docs/AGXUsage.rst
+++ b/llvm/docs/AGXUsage.rst
@@ -318,6 +318,10 @@ PTX behavior in software (``AGXStrictFP.cpp``):
functions, ``llvm.agx.rcp``, ``llvm.agx.rsqrt``, ``llvm.log2`` and
``llvm.exp2`` handle denormal operands or results explicitly.
+ In functions that flush ``float`` denormals, ``fdiv`` and ``llvm.sqrt`` are
+ correctly rounded in the same way, on flushed operands and with a flushed
+ result.
+
``canonical-nan``
NaN results of floating point arithmetic and conversions are ``0x7fffffff``
(``0x7fff`` for ``half`` and ``bfloat``); minimum and maximum keep the bits
@@ -328,6 +332,35 @@ normal operands, ``ieee-fp32`` adds about 8 instructions to an addition, 12
to a multiplication, 22 to a fused multiply-add, 13 to a comparison, 65 to a
division and 39 to a square root.
+Rounding modes
+--------------
+
+The hardware rounds to nearest even only. ``llvm.agx.set.fpmode(i32)`` and
+``llvm.agx.get.fpmode()`` access an emulated floating point mode register:
+
+ ===== ======================================================
+ Bits Meaning
+ ===== ======================================================
+ 0-1 ``float`` rounding: 0 toward zero, 1 to nearest even,
+ 2 upward, 3 downward (the ``FLT_ROUNDS`` numbering)
+ 2-3 ``half`` (and ``double``) rounding, same encoding
+ 4 ``float`` denormals are flushed
+ 5 ``half`` (and ``double``) denormals are flushed
+ ===== ======================================================
+
+It holds round to nearest even and the function's ``denormal_fpenv`` at
+entry, and is kept in SSA values by the ``agx-lower-fp-env`` pass, which also
+lowers the constrained floating point intrinsics
+(``llvm.experimental.constrained.*``). Those that round to nearest even,
+statically or through the register, become ordinary operations. The others
+(``fadd``, ``fsub``, ``fmul``, ``fdiv``, ``fma``, ``sqrt``, ``fptrunc`` to
+``half``, ``sitofp``/``uitofp`` and the rounding functions) are computed in
+software and correctly rounded in their direction, with the overflow result
+of that direction (the largest finite value or infinity) and the sign of an
+exact zero sum that it requires. An operation whose denormal mode differs
+from its function's flushes its operands and result explicitly. The software
+paths do not depend on the ``ieee-fp32`` feature.
+
SIMD-group shuffles
-------------------
diff --git a/llvm/include/llvm/IR/IntrinsicsAGX.td b/llvm/include/llvm/IR/IntrinsicsAGX.td
index c55b1915e39c8..77d603ab9bfd0 100644
--- a/llvm/include/llvm/IR/IntrinsicsAGX.td
+++ b/llvm/include/llvm/IR/IntrinsicsAGX.td
@@ -82,6 +82,29 @@ def int_agx_wait
[IntrHasSideEffects, IntrWillReturn, IntrNoCallback,
IntrNoFree, ImmArg<ArgIndex<0>>]>;
+//===----------------------------------------------------------------------===//
+// Floating point environment.
+//===----------------------------------------------------------------------===//
+
+// There is no floating point mode register: these model one, and the AGX
+// backend turns it into SSA values before instruction selection. The mode is
+// bits 0-1 float rounding (0 toward zero, 1 nearest even, 2 up, 3 down,
+// like llvm.get.rounding),
+// bits 2-3 half and double rounding,
+// bit 4 float denormals flushed to zero,
+// bit 5 half and double denormals flushed to zero.
+// Constrained floating point intrinsics with dynamic rounding use it. At
+// function entry, rounding is to nearest even and the denormal bits follow
+// the function's denormal_fpenv.
+def int_agx_set_fpmode
+ : Intrinsic<[], [llvm_i32_ty],
+ [IntrInaccessibleMemOnly, IntrWillReturn, IntrNoCallback,
+ IntrNoFree]>;
+def int_agx_get_fpmode
+ : Intrinsic<[llvm_i32_ty], [],
+ [IntrInaccessibleMemOnly, IntrReadMem, IntrWillReturn,
+ IntrNoCallback, IntrNoFree]>;
+
//===----------------------------------------------------------------------===//
// SIMD-group and quad-group operations.
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AGX/AGX.h b/llvm/lib/Target/AGX/AGX.h
index 80b9005ac124b..5dc429c4f61c0 100644
--- a/llvm/lib/Target/AGX/AGX.h
+++ b/llvm/lib/Target/AGX/AGX.h
@@ -26,6 +26,7 @@ class PassRegistry;
ModulePass *createAGXAlwaysInlinePass();
FunctionPass *createAGXCodeGenPreparePass();
FunctionPass *createAGXStrictFPPass();
+FunctionPass *createAGXLowerFPEnvPass();
FunctionPass *createAGXAnnotateControlFlowPass();
// Instruction selection.
@@ -39,6 +40,7 @@ FunctionPass *createAGXInsertWaitsPass();
void initializeAGXAlwaysInlinePass(PassRegistry &);
void initializeAGXCodeGenPreparePass(PassRegistry &);
void initializeAGXStrictFPPass(PassRegistry &);
+void initializeAGXLowerFPEnvPass(PassRegistry &);
void initializeAGXAnnotateControlFlowPass(PassRegistry &);
void initializeAGXDAGToDAGISelLegacyPass(PassRegistry &);
void initializeAGXFoldOperandsPass(PassRegistry &);
diff --git a/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp b/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp
index cbc6d6ffd00ae..cf09418d0da87 100644
--- a/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp
@@ -492,6 +492,9 @@ bool AGXCodeGenPrepare::flushHalfDenormals(Function &F) {
SmallVector<Instruction *, 32> Worklist;
for (Instruction &I : instructions(F)) {
+ // Operations with their own denormal handling (from AGXLowerFPEnv).
+ if (I.hasMetadata("agx.ieee") || I.hasMetadata("agx.exact"))
+ continue;
bool Arith = isFPArithmetic(I) && isHalf(I.getType());
bool Reads = Arith || (isa<FCmpInst>(I) &&
isHalf(I.getOperand(0)->getType())) ||
diff --git a/llvm/lib/Target/AGX/AGXISelLowering.cpp b/llvm/lib/Target/AGX/AGXISelLowering.cpp
index 85346d4260ae6..67d77a2eb4fa8 100644
--- a/llvm/lib/Target/AGX/AGXISelLowering.cpp
+++ b/llvm/lib/Target/AGX/AGXISelLowering.cpp
@@ -740,7 +740,19 @@ SDValue AGXTargetLowering::lowerFDIV(SDValue Op, SelectionDAG &DAG) const {
// One Newton-Raphson step on the quotient: q += (a - b * q) * rcp(b).
SDValue E = DAG.getNode(ISD::FMA, DL, VT, DAG.getNode(ISD::FNEG, DL, VT, B),
Q, A);
- return DAG.getNode(ISD::FMA, DL, VT, E, R, Q);
+ SDValue Refined = DAG.getNode(ISD::FMA, DL, VT, E, R, Q);
+ // The step turns an infinite quotient into NaN (0 * inf) and a zero one
+ // into +0; those quotients are already exact.
+ EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
+ SDValue AbsQ = DAG.getNode(ISD::FABS, DL, VT, Q);
+ SDValue Zero = DAG.getConstantFP(0.0, DL, VT);
+ SDValue Inf = DAG.getConstantFP(APFloat::getInf(VT.getFltSemantics()), DL,
+ VT);
+ SDValue Exact =
+ DAG.getNode(ISD::OR, DL, CCVT,
+ DAG.getSetCC(DL, CCVT, AbsQ, Zero, ISD::SETOEQ),
+ DAG.getSetCC(DL, CCVT, AbsQ, Inf, ISD::SETOEQ));
+ return DAG.getSelect(DL, VT, Exact, Q, Refined);
}
/// copysign(m, s) = (m & ~sign) | (s & sign), on the bits.
diff --git a/llvm/lib/Target/AGX/AGXStrictFP.cpp b/llvm/lib/Target/AGX/AGXStrictFP.cpp
index 78d5b9e8dad11..0cd6b310dd4e0 100644
--- a/llvm/lib/Target/AGX/AGXStrictFP.cpp
+++ b/llvm/lib/Target/AGX/AGXStrictFP.cpp
@@ -44,7 +44,9 @@
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Target/TargetMachine.h"
+#include "llvm/IR/Dominators.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/PromoteMemToReg.h"
using namespace llvm;
@@ -64,6 +66,28 @@ class AGXStrictFP : public FunctionPass {
}
};
+/// A binary floating point format.
+struct Format {
+ unsigned Precision; // significand bits, including the implicit one
+ int Bias;
+ unsigned Width;
+ int minLSB() const { return 2 - Bias - int(Precision); }
+ uint32_t fracMask() const { return (1u << (Precision - 1)) - 1; }
+ uint32_t expMask() const { return uint32_t(2 * Bias + 1); }
+ uint32_t infBits() const { return expMask() << (Precision - 1); }
+ uint32_t signBit() const { return 1u << (Width - 1); }
+};
+constexpr Format Binary32 = {24, 127, 32};
+constexpr Format Binary16 = {11, 15, 16};
+
+/// Rounding directions, numbered like FLT_ROUNDS and llvm.get.rounding.
+enum RoundingCode : uint32_t {
+ RoundTowardZero = 0,
+ RoundNearestEven = 1,
+ RoundUp = 2,
+ RoundDown = 3,
+};
+
/// sign * M * 2^E: Sign is i1, M is i64 and E is i32.
struct Term {
Value *Sign;
@@ -73,13 +97,21 @@ struct Term {
/// Builds the integer implementations at the builder's insertion point.
class SoftFloat {
- IRBuilder<> &B;
+ IRBuilderBase &B;
Type *I1, *I32, *I64, *F32;
public:
- explicit SoftFloat(IRBuilder<> &B)
+ /// The rounding direction (a RoundingCode, constant or not) of the
+ /// results built by convert().
+ Value *Rounding;
+
+ explicit SoftFloat(IRBuilderBase &B)
: B(B), I1(B.getInt1Ty()), I32(B.getInt32Ty()), I64(B.getInt64Ty()),
- F32(B.getFloatTy()) {}
+ F32(B.getFloatTy()), Rounding(B.getInt32(RoundNearestEven)) {}
+
+ Value *roundingIs(RoundingCode C) {
+ return B.CreateICmpEQ(Rounding, B.getInt32(C));
+ }
Value *c32(uint32_t V) { return B.getInt32(V); }
Value *c64(uint64_t V) { return B.getInt64(V); }
@@ -132,51 +164,71 @@ class SoftFloat {
}
Value *signBit(Value *Bits) { return B.CreateICmpSLT(Bits, c32(0)); }
- /// A finite value as sign * M * 2^E with M < 2^24. With Normalize, the
- /// most significant bit of a nonzero M is bit 23.
- Term unpack(Value *Bits, bool Normalize) {
- Value *Exp = B.CreateAnd(B.CreateLShr(Bits, 23), c32(0xff));
- Value *Frac = B.CreateAnd(Bits, c32(0x7fffff));
+ /// A finite value (bits in an i32) as sign * M * 2^E with M <
+ /// 2^Precision. With Normalize, the most significant bit of a nonzero M is
+ /// bit Precision - 1.
+ Term unpack(Value *Bits, bool Normalize, Format Fmt = Binary32) {
+ unsigned P = Fmt.Precision;
+ Value *Exp = B.CreateAnd(B.CreateLShr(Bits, P - 1), c32(Fmt.expMask()));
+ Value *Frac = B.CreateAnd(Bits, c32(Fmt.fracMask()));
Value *Den = B.CreateICmpEQ(Exp, c32(0));
- Value *M = B.CreateSelect(Den, Frac, B.CreateOr(Frac, c32(0x800000)));
- Value *E = B.CreateSub(B.CreateSelect(Den, c32(1), Exp), c32(150));
+ Value *M = B.CreateSelect(Den, Frac, B.CreateOr(Frac, c32(1u << (P - 1))));
+ Value *E = B.CreateSub(B.CreateSelect(Den, c32(1), Exp),
+ c32(Fmt.Bias + P - 1));
if (Normalize) {
Value *Lz = B.CreateBinaryIntrinsic(Intrinsic::ctlz, M, B.getFalse());
Value *Sh = B.CreateSelect(B.CreateICmpEQ(M, c32(0)), c32(0),
- B.CreateSub(Lz, c32(8)));
+ B.CreateSub(Lz, c32(32 - P)));
M = B.CreateShl(M, Sh);
E = B.CreateSub(E, Sh);
}
- return {signBit(Bits), B.CreateZExt(M, I64), E};
+ Value *Sign = B.CreateICmpNE(B.CreateAnd(Bits, c32(Fmt.signBit())), c32(0));
+ return {Sign, B.CreateZExt(M, I64), E};
}
- /// Round sign * M * 2^E to nearest even: the bits of the binary32 result.
- Value *convert(const Term &T) {
+ /// Round sign * M * 2^E in the direction Rounding: the bits of the result
+ /// in Fmt (in an i32).
+ Value *convert(const Term &T, Format Fmt = Binary32) {
+ unsigned P = Fmt.Precision;
Value *L = bitLength(T.M);
- // Shift so that 24 significant bits remain, or fewer for denormals.
- Value *S = smax(B.CreateSub(L, c32(24)), B.CreateSub(c32(-149), T.E));
+ // Shift so that P significant bits remain, or fewer for denormals.
+ Value *S = smax(B.CreateSub(L, c32(P)), B.CreateSub(c32(Fmt.minLSB()), T.E));
Value *Right = B.CreateICmpSGT(S, c32(0));
Value *K = B.CreateSelect(Right, umin(S, c32(64)), c32(0));
Value *Q = lshr(T.M, K);
Value *Rem = B.CreateSub(T.M, shl(Q, K));
Value *Half = shl(c64(1), B.CreateSub(smax(K, c32(1)), c32(1)));
- Value *Up = B.CreateOr(
+ Value *Inexact = B.CreateICmpNE(Rem, c64(0));
+ Value *Nearest = B.CreateOr(
B.CreateICmpUGT(Rem, Half),
B.CreateAnd(B.CreateICmpEQ(Rem, Half),
B.CreateICmpNE(B.CreateAnd(Q, c64(1)), c64(0))));
+ Value *Up = B.CreateSelect(
+ roundingIs(RoundNearestEven), Nearest,
+ B.CreateSelect(roundingIs(RoundUp), B.CreateAnd(Inexact, B.CreateNot(T.Sign)),
+ B.CreateSelect(roundingIs(RoundDown),
+ B.CreateAnd(Inexact, T.Sign), B.getFalse())));
Up = B.CreateAnd(Up, Right);
Value *Rounded = B.CreateAdd(Q, B.CreateZExt(Up, I64));
Value *Left = shl(T.M, B.CreateSelect(Right, c32(0), B.CreateNeg(S)));
Value *Mant = B.CreateTrunc(B.CreateSelect(Right, Rounded, Left), I32);
// The exponent field minus one: the implicit bit of Mant adds the one,
- // and a rounding carry into bit 24 increments the exponent.
- Value *Biased = B.CreateAdd(B.CreateAdd(T.E, S), c32(149));
- Value *Bits = B.CreateAdd(B.CreateShl(Biased, 23), Mant);
- Value *Over = B.CreateOr(B.CreateICmpSGT(Biased, c32(253)),
- B.CreateICmpUGE(Bits, c32(0x7f800000)));
- Bits = B.CreateSelect(Over, c32(0x7f800000), Bits);
+ // and a rounding carry into bit P increments the exponent.
+ Value *Biased = B.CreateAdd(B.CreateAdd(T.E, S), c32(Fmt.Bias + P - 2));
+ Value *Bits = B.CreateAdd(B.CreateShl(Biased, P - 1), Mant);
+ Value *Over = B.CreateOr(B.CreateICmpSGT(Biased, c32(2 * Fmt.Bias - 1)),
+ B.CreateICmpUGE(Bits, c32(Fmt.infBits())));
+ // Overflows give infinity, or the largest finite value when rounding
+ // toward zero or away from the sign's infinity.
+ Value *Max = c32(Fmt.infBits() - 1), *Inf = c32(Fmt.infBits());
+ Value *OverBits = B.CreateSelect(
+ roundingIs(RoundTowardZero), Max,
+ B.CreateSelect(roundingIs(RoundUp), B.CreateSelect(T.Sign, Max, Inf),
+ B.CreateSelect(roundingIs(RoundDown),
+ B.CreateSelect(T.Sign, Inf, Max), Inf)));
+ Bits = B.CreateSelect(Over, OverBits, Bits);
Bits = B.CreateSelect(B.CreateICmpEQ(T.M, c64(0)), c32(0), Bits);
- return B.CreateOr(Bits, B.CreateShl(B.CreateZExt(T.Sign, I32), 31));
+ return B.CreateOr(Bits, B.CreateSelect(T.Sign, c32(Fmt.signBit()), c32(0)));
}
Term mul(const Term &X, const Term &Y) {
@@ -208,13 +260,18 @@ class SoftFloat {
B.CreateSub(MB, Aligned));
Value *Neg = B.CreateICmpSLT(Sum, c64(0));
Value *M = B.CreateSelect(Neg, B.CreateNeg(Sum), Sum);
- // An exact cancellation is +0 when rounding to nearest.
- Value *Sign = B.CreateAnd(B.CreateXor(SB, Neg), B.CreateICmpNE(M, c64(0)));
- // Zero terms: the other term, and -0 + -0 = -0.
+ // An exact cancellation is +0, or -0 when rounding down.
+ Value *Sign = B.CreateSelect(B.CreateICmpNE(M, c64(0)), B.CreateXor(SB, Neg),
+ roundingIs(RoundDown));
+ // Zero terms: the other term; the sum of two zeros is negative if both
+ // are, or (rounding down) if either is.
Value *XZero = B.CreateICmpEQ(X.M, c64(0));
Value *YZero = B.CreateICmpEQ(Y.M, c64(0));
Value *BothZero = B.CreateAnd(XZero, YZero);
- Sign = B.CreateSelect(BothZero, B.CreateAnd(X.Sign, Y.Sign),
+ Value *ZeroSign = B.CreateSelect(roundingIs(RoundDown),
+ B.CreateOr(X.Sign, Y.Sign),
+ B.CreateAnd(X.Sign, Y.Sign));
+ Sign = B.CreateSelect(BothZero, ZeroSign,
B.CreateSelect(XZero, Y.Sign,
B.CreateSelect(YZero, X.Sign, Sign)));
M = B.CreateSelect(XZero, Y.M, B.CreateSelect(YZero, X.M, M));
@@ -248,6 +305,25 @@ class SoftFloat {
Value *nextUp(Value *X) { return flt(B.CreateAdd(bits(X), c32(1))); }
Value *nextDown(Value *X) { return flt(B.CreateSub(bits(X), c32(1))); }
+ /// Q, the correctly rounded positive magnitude of a result with sign Neg,
+ /// rounded in the direction Rounding instead: Err has the sign of the exact
+ /// magnitude minus Q.
+ Value *adjustMagnitude(Value *Q, Value *Err, Value *Neg) {
+ Value *Above = B.CreateFCmpOGT(Err, fconst(0.0f));
+ Value *Below = B.CreateFCmpOLT(Err, fconst(0.0f));
+ // Rounding up a positive or down a negative result increases the
+ // magnitude; toward zero, down a positive or up a negative one decreases
+ // it.
+ Value *Away = B.CreateOr(B.CreateAnd(roundingIs(RoundUp), B.CreateNot(Neg)),
+ B.CreateAnd(roundingIs(RoundDown), Neg));
+ Value *Toward = B.CreateOr(
+ roundingIs(RoundTowardZero),
+ B.CreateOr(B.CreateAnd(roundingIs(RoundUp), Neg),
+ B.CreateAnd(roundingIs(RoundDown), B.CreateNot(Neg))));
+ Value *R = B.CreateSelect(B.CreateAnd(Away, Above), nextUp(Q), Q);
+ return B.CreateSelect(B.CreateAnd(Toward, Below), nextDown(Q), R);
+ }
+
/// Division of normal operands with a normal result, using the hardware:
/// the mantissas (in [1, 2)) are divided with a refined reciprocal, and
/// the quotient is corrected until the exact remainder shows that it is
@@ -282,6 +358,10 @@ class SoftFloat {
B.CreateAnd(B.CreateFCmpOEQ(R, NegHalfDown), Odd));
Q = B.CreateSelect(GoUp, Up, B.CreateSelect(GoDown, Down, Q));
}
+ // Directed rounding: the remainder gives the direction of the error of
+ // the magnitude.
+ Value *Neg = B.CreateICmpSLT(B.CreateXor(AB, DB), c32(0));
+ Q = adjustMagnitude(Q, fma(NegD, Q, A1), Neg);
Value *QB = bits(Q);
Value *Shift = B.CreateSub(EA, ED);
Value *Exp = B.CreateAdd(B.CreateAnd(B.CreateLShr(QB, 23), c32(0xff)), Shift);
@@ -315,6 +395,7 @@ class SoftFloat {
Value *High = B.CreateFCmpOLT(fma(S, Up, NegM), fconst(0.0f));
S = B.CreateSelect(High, Up, B.CreateSelect(Low, Down, S));
}
+ S = adjustMagnitude(S, fma(B.CreateFNeg(S), S, M), B.getFalse());
Value *Half = B.CreateAShr(B.CreateSub(K, Odd), 1);
Value *R = B.CreateAdd(bits(S), B.CreateShl(Half, 23));
return {flt(R), Valid};
@@ -416,6 +497,83 @@ class SoftFloat {
return flt(Bits);
}
+ /// Binary32 X + Y (or X - Y), rounded in the direction Rounding.
+ Value *roundedAdd(Value *X, Value *Y, bool Sub) {
+ Value *XB = bits(X), *YB = bits(Y);
+ Term TX = unpack(XB, false), TY = unpack(YB, false);
+ if (Sub)
+ TY.Sign = B.CreateNot(TY.Sign);
+ Value *Exact = flt(convert(add(TX, TY)));
+ // Infinities and NaNs: denormal operands do not change the result.
+ Value *HW = Sub ? B.CreateFSub(X, Y) : B.CreateFAdd(X, Y);
+ return B.CreateSelect(B.CreateOr(isInfOrNaN(XB), isInfOrNaN(YB)), HW, Exact);
+ }
+
+ /// Binary32 X * Y, rounded in the direction Rounding.
+ Value *roundedMul(Value *X, Value *Y) {
+ Value *XB = bits(X), *YB = bits(Y);
+ Value *Exact = flt(convert(mul(unpack(XB, false), unpack(YB, false))));
+ Value *HW = B.CreateFMul(nonzeroNormal(X), nonzeroNormal(Y));
+ return B.CreateSelect(B.CreateOr(isInfOrNaN(XB), isInfOrNaN(YB)), HW, Exact);
+ }
+
+ /// Binary32 X * Y + Z with one rounding in the direction Rounding.
+ Value *roundedFma(Value *X, Value *Y, Value *Z) {
+ Value *XB = bits(X), *YB = bits(Y), *ZB = bits(Z);
+ Term P = mul(unpack(XB, false), unpack(YB, false));
+ Value *Exact = flt(convert(add(P, unpack(ZB, false))));
+ Value *HW = fma(nonzeroNormal(X), nonzeroNormal(Y), nonzeroNormal(Z));
+ Value *Special = B.CreateOr(isInfOrNaN(XB),
+ B.CreateOr(isInfOrNaN(YB), isInfOrNaN(ZB)));
+ return B.CreateSelect(Special, HW, Exact);
+ }
+
+ /// An integer converted to Fmt (float or half), rounded in the direction
+ /// Rounding.
+ Value *roundedFromInt(Value *V, bool Signed, Type *Ty) {
+ Format Fmt = Ty->isHalfTy() ? Binary16 : Binary32;
+ Value *W = Signed ? B.CreateSExtOrTrunc(V, I64) : B.CreateZExtOrTrunc(V, I64);
+ Value *Neg = Signed ? B.CreateICmpSLT(W, c64(0)) : B.getFalse();
+ Value *M = B.CreateSelect(Neg, B.CreateNeg(W), W);
+ Value *Bits = convert({Neg, M, c32(0)}, Fmt);
+ return toFloat(Bits, Ty);
+ }
+
+ /// A float converted to half, rounded in the direction Rounding.
+ Value *roundedToHalf(Value *X) {
+ Value *XB = bits(X);
+ Value *Exact = toFloat(convert(unpack(XB, false), Binary16), B.getHalfTy());
+ Value *HW = B.CreateFPTrunc(X, B.getHalfTy());
+ return B.CreateSelect(isInfOrNaN(XB), HW, Exact);
+ }
+
+ /// Bits of Fmt in an i32 as a value of type Ty.
+ Value *toFloat(Value *Bits, Type *Ty) {
+ if (Ty->isHalfTy())
+ return B.CreateBitCast(B.CreateTrunc(Bits, B.getInt16Ty()), Ty);
+ return flt(Bits);
+ }
+
+ /// Round to an integral value in the direction Rounding.
+ Value *roundedIntegral(Value *X) {
+ auto Call = [&](Intrinsic::ID ID) { return B.CreateUnaryIntrinsic(ID, X); };
+ Value *R = B.CreateSelect(roundingIs(RoundDown), Call(Intrinsic::floor),
+ Call(Intrinsic::roundeven));
+ R = B.CreateSelect(roundingIs(RoundUp), Call(Intrinsic::ceil), R);
+ return B.CreateSelect(roundingIs(RoundTowardZero), Call(Intrinsic::trunc), R);
+ }
+
+ /// flush(X) = |X| below the smallest normal ? +-0 : X, for float or half.
+ Value *flushDenormal(Value *X) {
+ Type *Ty = X->getType();
+ APFloat Min = APFloat::getSmallestNormalized(Ty->getFltSemantics());
+ Value *Small = B.CreateFCmpOLT(B.CreateUnaryIntrinsic(Intrinsic::fabs, X),
+ ConstantFP::get(Ty, Min));
+ Value *Zero = B.CreateBinaryIntrinsic(Intrinsic::copysign,
+ ConstantFP::get(Ty, 0.0), X);
+ return B.CreateSelect(Small, Zero, X);
+ }
+
/// A comparison operand as a signed integer, for zeros and denormals.
Value *denormalOrder(Value *Bits) {
Value *Mag = B.CreateAnd(Bits, c32(0x7fffff));
@@ -590,11 +748,11 @@ static void lowerWithSlowPath(Instruction &I, CondFn BuildCond,
/// Replace I by the result of BuildFast, or of BuildSlow (in a conditional
/// block) where BuildFast says that its result is not valid.
-template <typename FastFn, typename SlowFn>
+template <typename BuilderT = IRBuilder<>, typename FastFn, typename SlowFn>
static void replaceWithSlowPath(Instruction &I, FastFn BuildFast,
SlowFn BuildSlow) {
Instruction *Next = I.getNextNode();
- IRBuilder<> B(Next);
+ BuilderT B(Next);
SoftFloat SF(B);
auto [Fast, Valid] = BuildFast(SF, B);
Instruction *ThenTerm =
@@ -640,7 +798,7 @@ static bool lowerIEEE(Instruction &I) {
// the denormal range.
lowerWithSlowPath(
I,
- [&](SoftFloat &SF, IRBuilder<> &B) {
+ [&](SoftFloat &SF, IRBuilderBase &B) {
Value *Limit = SF.fconst(0x1p-101f);
auto Small = [&](Value *V) {
return B.CreateFCmpOLT(
@@ -648,7 +806,7 @@ static bool lowerIEEE(Instruction &I) {
};
return B.CreateAnd(Small(X), Small(Y));
},
- [&](SoftFloat &SF, IRBuilder<> &B) {
+ [&](SoftFloat &SF, IRBuilderBase &B) {
Term TX = SF.unpack(SF.bits(X), false);
Term TY = SF.unpack(SF.bits(Y), false);
if (Sub)
@@ -663,7 +821,7 @@ static bool lowerIEEE(Instruction &I) {
Value *X = I.getOperand(0), *Y = I.getOperand(1);
lowerWithSlowPath(
I,
- [&](SoftFloat &SF, IRBuilder<> &B) {
+ [&](SoftFloat &SF, IRBuilderBase &B) {
// A product at most 2^-125 (or NaN) of nonzero operands.
Value *Tiny = B.CreateFCmpULE(
B.CreateUnaryIntrinsic(Intrinsic::fabs, &I),
@@ -672,7 +830,7 @@ static bool lowerIEEE(Instruction &I) {
SF.isZero(SF.bits(X)),
SF.isZero(SF.bits(Y)))));
},
- [&](SoftFloat &SF, IRBuilder<> &B) {
+ [&](SoftFloat &SF, IRBuilderBase &B) {
Value *XB = SF.bits(X), *YB = SF.bits(Y);
Value *Exact = SF.flt(SF.convert(
SF.mul(SF.unpack(XB, false), SF.unpack(YB, false))));
@@ -687,8 +845,8 @@ static bool lowerIEEE(Instruction &I) {
return false;
Value *X = I.getOperand(0), *Y = I.getOperand(1);
replaceWithSlowPath(
- I, [&](SoftFloat &SF, IRBuilder<> &) { return SF.fastDiv(X, Y); },
- [&](SoftFloat &SF, IRBuilder<> &) { return SF.div(X, Y); });
+ I, [&](SoftFloat &SF, IRBuilderBase &) { return SF.fastDiv(X, Y); },
+ [&](SoftFloat &SF, IRBuilderBase &) { return SF.div(X, Y); });
return true;
}
case Instruction::FCmp: {
@@ -723,7 +881,7 @@ static bool lowerIEEE(Instruction &I) {
*Z = II->getArgOperand(2);
lowerWithSlowPath(
I,
- [&](SoftFloat &SF, IRBuilder<> &B) {
+ [&](SoftFloat &SF, IRBuilderBase &B) {
Value *XB = SF.bits(X), *YB = SF.bits(Y), *ZB = SF.bits(Z);
Value *Den = B.CreateOr(SF.isDenormal(XB),
B.CreateOr(SF.isDenormal(YB), SF.isDenormal(ZB)));
@@ -733,7 +891,7 @@ static bool lowerIEEE(Instruction &I) {
SF.isZero(YB))));
return B.CreateOr(Den, Tiny);
},
- [&](SoftFloat &SF, IRBuilder<> &B) {
+ [&](SoftFloat &SF, IRBuilderBase &B) {
Value *XB = SF.bits(X), *YB = SF.bits(Y), *ZB = SF.bits(Z);
Term P = SF.mul(SF.unpack(XB, false), SF.unpack(YB, false));
Value *Exact = SF.flt(SF.convert(SF.add(P, SF.unpack(ZB, false))));
@@ -751,8 +909,8 @@ static bool lowerIEEE(Instruction &I) {
return false;
Value *X = II->getArgOperand(0);
replaceWithSlowPath(
- I, [&](SoftFloat &SF, IRBuilder<> &) { return SF.fastSqrt(X); },
- [&](SoftFloat &SF, IRBuilder<> &) { return SF.sqrt(X); });
+ I, [&](SoftFloat &SF, IRBuilderBase &) { return SF.fastSqrt(X); },
+ [&](SoftFloat &SF, IRBuilderBase &) { return SF.sqrt(X); });
return true;
}
case Intrinsic::minnum:
@@ -765,7 +923,7 @@ static bool lowerIEEE(Instruction &I) {
bool Min = ID == Intrinsic::minnum || ID == Intrinsic::minimumnum ||
ID == Intrinsic::minimum;
bool Propagate = ID == Intrinsic::minimum || ID == Intrinsic::maximum;
- replace(I, [&](SoftFloat &SF, IRBuilder<> &) {
+ replace(I, [&](SoftFloat &SF, IRBuilderBase &) {
return SF.minMax(X, Y, Min, Propagate);
});
return true;
@@ -844,22 +1002,57 @@ static bool lowerIEEE(Instruction &I) {
}
}
+/// Instructions built by the software implementations themselves, which
+/// need no further lowering.
+static constexpr const char *ExactTag = "agx.exact";
+/// Instructions whose denormals follow IEEE although their function flushes
+/// them (their operands and results were flushed explicitly if needed).
+static constexpr const char *IEEETag = "agx.ieee";
+
+/// Division and square root, which the hardware only approximates.
+static bool isCorrectlyRounded(const Instruction &I) {
+ if (!isF32(&I) || (isa<FPMathOperator>(I) &&
+ (I.hasApproxFunc() || I.hasAllowReciprocal())))
+ return false;
+ return I.getOpcode() == Instruction::FDiv ||
+ getIntrinsicID(I) == Intrinsic::sqrt;
+}
+
+/// A correctly rounded operation in a flushing function: the IEEE lowering
+/// on flushed operands, with a flushed result.
+static bool lowerFlushing(Instruction &I) {
+ IRBuilder<> B(&I);
+ SoftFloat SF(B);
+ for (Use &Op : I.operands())
+ if (Op->getType()->isFloatTy())
+ Op.set(SF.flushDenormal(Op));
+ SmallVector<Use *, 4> Uses;
+ for (Use &U : I.uses())
+ Uses.push_back(&U);
+ B.SetInsertPoint(I.getNextNode());
+ Value *Result = SF.flushDenormal(&I);
+ for (Use *U : Uses)
+ U->set(Result);
+ lowerIEEE(I);
+ return true;
+}
+
bool AGXStrictFP::runOnFunction(Function &F) {
if (skipFunction(F))
return false;
const TargetMachine &TM =
getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
const auto &ST = TM.getSubtarget<AGXSubtarget>(F);
- bool IEEE = ST.hasIEEEFP32() &&
- F.getDenormalMode(APFloat::IEEEsingle()).Output ==
- DenormalMode::IEEE;
+ bool FunctionIEEE = F.getDenormalMode(APFloat::IEEEsingle()).Output ==
+ DenormalMode::IEEE;
bool CanonicalNaN = ST.hasCanonicalNaN();
- if (!IEEE && !CanonicalNaN)
+ if (!ST.hasIEEEFP32() && !CanonicalNaN)
return false;
SmallVector<Instruction *, 32> Worklist;
for (Instruction &I : instructions(F))
- Worklist.push_back(&I);
+ if (!I.hasMetadata(ExactTag))
+ Worklist.push_back(&I);
bool Changed = false;
// NaN canonicalization first: the IEEE lowering then replaces the
@@ -870,8 +1063,320 @@ bool AGXStrictFP::runOnFunction(Function &F) {
canonicalizeNaN(*I);
Changed = true;
}
- if (IEEE)
- for (Instruction *I : Worklist)
- Changed |= lowerIEEE(*I);
+ if (ST.hasIEEEFP32())
+ for (Instruction *I : Worklist) {
+ if (FunctionIEEE || I->hasMetadata(IEEETag))
+ Changed |= lowerIEEE(*I);
+ else if (isCorrectlyRounded(*I))
+ Changed |= lowerFlushing(*I);
+ }
return Changed;
}
+
+//===----------------------------------------------------------------------===//
+// AGXLowerFPEnv
+//===----------------------------------------------------------------------===//
+
+namespace {
+
+/// Models the floating point mode register of llvm.agx.set.fpmode and
+/// llvm.agx.get.fpmode with a value per program point, and lowers the
+/// constrained floating point intrinsics: those that round to nearest even
+/// become ordinary operations, the others are computed in software with the
+/// rounding direction of their metadata or of the mode. Operations whose
+/// denormal mode differs from their function's flush their operands and
+/// results explicitly, and are tagged agx.ieee.
+class AGXLowerFPEnv : public FunctionPass {
+public:
+ static char ID;
+ AGXLowerFPEnv() : FunctionPass(ID) {}
+
+ StringRef getPassName() const override {
+ return "AGX Lower Floating Point Environment";
+ }
+ bool runOnFunction(Function &F) override;
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<TargetPassConfig>();
+ }
+};
+
+/// A builder that tags the floating point operations it creates.
+struct TaggingBuilder : IRBuilder<ConstantFolder, IRBuilderCallbackInserter> {
+ explicit TaggingBuilder(Instruction *IP)
+ : IRBuilder(IP->getContext(), ConstantFolder(),
+ IRBuilderCallbackInserter([](Instruction *I) {
+ if (isa<FPMathOperator>(I) || isa<CallBase>(I) ||
+ isa<FPTruncInst>(I))
+ I->setMetadata(ExactTag, MDNode::get(I->getContext(), {}));
+ })) {
+ SetInsertPoint(IP);
+ }
+};
+
+} // namespace
+
+char AGXLowerFPEnv::ID = 0;
+
+INITIALIZE_PASS_BEGIN(AGXLowerFPEnv, "agx-lower-fp-env",
+ "AGX Lower Floating Point Environment", false, false)
+INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
+INITIALIZE_PASS_END(AGXLowerFPEnv, "agx-lower-fp-env",
+ "AGX Lower Floating Point Environment", false, false)
+
+FunctionPass *llvm::createAGXLowerFPEnvPass() { return new AGXLowerFPEnv(); }
+
+static constexpr uint32_t ModeHalfShift = 2;
+static constexpr uint32_t ModeFloatFlush = 0x10, ModeHalfFlush = 0x20;
+
+static bool flushes(const Function &F, const fltSemantics &Sem) {
+ DenormalMode M = F.getDenormalMode(Sem);
+ return M.Output != DenormalMode::IEEE && M.Output != DenormalMode::Dynamic;
+}
+
+static uint32_t roundingCode(RoundingMode RM) {
+ switch (RM) {
+ case RoundingMode::TowardZero:
+ return RoundTowardZero;
+ case RoundingMode::TowardPositive:
+ return RoundUp;
+ case RoundingMode::TowardNegative:
+ return RoundDown;
+ default:
+ return RoundNearestEven;
+ }
+}
+
+/// The constrained intrinsic as the ordinary operation.
+static Value *lowerToPlain(ConstrainedFPIntrinsic *CI, IRBuilderBase &B) {
+ Function &F = *CI->getFunction();
+ SmallVector<Value *, 4> Args(CI->args().begin(),
+ CI->args().begin() + CI->getNonMetadataArgCount());
+ switch (CI->getIntrinsicID()) {
+#define INSTRUCTION(NAME, NARGS, ROUND_MODE, INTRINSIC) \
+ case Intrinsic::INTRINSIC: \
+ if (Instruction::isBinaryOp(Instruction::NAME)) \
+ return B.CreateBinOp(Instruction::BinaryOps(Instruction::NAME), Args[0], \
+ Args[1]); \
+ return B.CreateCast(Instruction::CastOps(Instruction::NAME), Args[0], \
+ CI->getType());
+#define CMP_INSTRUCTION(NAME, NARGS, ROUND_MODE, INTRINSIC, DAGN) \
+ case Intrinsic::INTRINSIC: \
+ return B.CreateFCmp(cast<ConstrainedFPCmpIntrinsic>(CI)->getPredicate(), \
+ Args[0], Args[1]);
+#define FUNCTION(NAME, NARGS, ROUND_MODE, INTRINSIC) \
+ case Intrinsic::INTRINSIC: { \
+ SmallVector<Type *, 4> ArgTys; \
+ for (Value *A : Args) \
+ ArgTys.push_back(A->getType()); \
+ SmallVector<Type *, 2> OverloadTys; \
+ FunctionType *FT = FunctionType::get(CI->getType(), ArgTys, false); \
+ if (!Intrinsic::isSignatureValid(Intrinsic::NAME, FT, OverloadTys)) \
+ return nullptr; \
+ return B.CreateCall(Intrinsic::getOrInsertDeclaration( \
+ F.getParent(), Intrinsic::NAME, OverloadTys), \
+ Args); \
+ }
+#include "llvm/IR/ConstrainedOps.def"
+ default:
+ return nullptr;
+ }
+}
+
+/// The constrained intrinsic computed in software with the rounding
+/// direction Rounding (an i32 RoundingCode), or null if it does not depend on
+/// the rounding direction.
+static Value *lowerRounded(ConstrainedFPIntrinsic *CI, Value *Rounding,
+ bool CanonicalNaN) {
+ Type *Ty = CI->getType();
+ Value *R = nullptr;
+ auto Arg = [&](unsigned I) { return CI->getArgOperand(I); };
+ bool Float = Ty->isFloatTy();
+ {
+ TaggingBuilder B(CI);
+ SoftFloat SF(B);
+ SF.Rounding = Rounding;
+ switch (CI->getIntrinsicID()) {
+ case Intrinsic::experimental_constrained_fadd:
+ case Intrinsic::experimental_constrained_fsub:
+ if (Float)
+ R = SF.roundedAdd(Arg(0), Arg(1), CI->getIntrinsicID() ==
+ Intrinsic::experimental_constrained_fsub);
+ break;
+ case Intrinsic::experimental_constrained_fmul:
+ if (Float)
+ R = SF.roundedMul(Arg(0), Arg(1));
+ break;
+ case Intrinsic::experimental_constrained_fma:
+ case Intrinsic::experimental_constrained_fmuladd:
+ if (Float)
+ R = SF.roundedFma(Arg(0), Arg(1), Arg(2));
+ break;
+ case Intrinsic::experimental_constrained_sitofp:
+ case Intrinsic::experimental_constrained_uitofp:
+ if (Float || Ty->isHalfTy())
+ R = SF.roundedFromInt(Arg(0), CI->getIntrinsicID() ==
+ Intrinsic::experimental_constrained_sitofp,
+ Ty);
+ break;
+ case Intrinsic::experimental_constrained_fptrunc:
+ if (Ty->isHalfTy() && Arg(0)->getType()->isFloatTy())
+ R = SF.roundedToHalf(Arg(0));
+ break;
+ case Intrinsic::experimental_constrained_rint:
+ case Intrinsic::experimental_constrained_nearbyint:
+ R = SF.roundedIntegral(Arg(0));
+ break;
+ default:
+ break;
+ }
+ }
+ bool DivOrSqrt = Float && (CI->getIntrinsicID() ==
+ Intrinsic::experimental_constrained_fdiv ||
+ CI->getIntrinsicID() ==
+ Intrinsic::experimental_constrained_sqrt);
+ if (!R && !DivOrSqrt)
+ return nullptr;
+ if (DivOrSqrt) {
+ // Placeholder replaced below by the correctly rounded division or root.
+ Value *X = Arg(0);
+ Value *Y = CI->getIntrinsicID() == Intrinsic::experimental_constrained_fdiv
+ ? Arg(1)
+ : nullptr;
+ IRBuilder<> B(CI);
+ Instruction *Stub = cast<Instruction>(B.CreateFreeze(X));
+ replaceWithSlowPath<TaggingBuilder>(
+ *Stub,
+ [&](SoftFloat &SF, IRBuilderBase &) {
+ SF.Rounding = Rounding;
+ return Y ? SF.fastDiv(X, Y) : SF.fastSqrt(X);
+ },
+ [&](SoftFloat &SF, IRBuilderBase &) {
+ SF.Rounding = Rounding;
+ return Y ? SF.div(X, Y) : SF.sqrt(X);
+ });
+ // The phi that replaced the stub is the first instruction of the block
+ // that now holds CI.
+ R = &*CI->getParent()->begin();
+ }
+ if (CanonicalNaN) {
+ IRBuilder<> B(CI);
+ APFloat NaN(Ty->getFltSemantics(),
+ APInt::getSignedMaxValue(Ty->getPrimitiveSizeInBits()));
+ R = B.CreateSelect(B.CreateFCmpUNO(R, R), ConstantFP::get(Ty, NaN), R);
+ }
+ return R;
+}
+
+bool AGXLowerFPEnv::runOnFunction(Function &F) {
+ if (skipFunction(F))
+ return false;
+ const TargetMachine &TM =
+ getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
+ const auto &ST = TM.getSubtarget<AGXSubtarget>(F);
+
+ SmallVector<IntrinsicInst *, 8> ModeOps;
+ SmallVector<ConstrainedFPIntrinsic *, 16> Constrained;
+ for (Instruction &I : instructions(F)) {
+ if (auto *CI = dyn_cast<ConstrainedFPIntrinsic>(&I))
+ Constrained.push_back(CI);
+ else if (auto *II = dyn_cast<IntrinsicInst>(&I);
+ II && (II->getIntrinsicID() == Intrinsic::agx_set_fpmode ||
+ II->getIntrinsicID() == Intrinsic::agx_get_fpmode))
+ ModeOps.push_back(II);
+ }
+ if (ModeOps.empty() && Constrained.empty())
+ return false;
+
+ // The mode as a variable, promoted to SSA values.
+ bool FloatFlush = flushes(F, APFloat::IEEEsingle());
+ bool HalfFlush = flushes(F, APFloat::IEEEhalf());
+ uint32_t Initial = RoundNearestEven | (RoundNearestEven << ModeHalfShift) |
+ (FloatFlush ? ModeFloatFlush : 0) |
+ (HalfFlush ? ModeHalfFlush : 0);
+ IRBuilder<> Entry(&*F.getEntryBlock().getFirstInsertionPt());
+ AllocaInst *Mode = Entry.CreateAlloca(Entry.getInt32Ty());
+ Entry.CreateStore(Entry.getInt32(Initial), Mode);
+ for (IntrinsicInst *II : ModeOps) {
+ IRBuilder<> B(II);
+ if (II->getIntrinsicID() == Intrinsic::agx_set_fpmode) {
+ B.CreateStore(II->getArgOperand(0), Mode);
+ } else {
+ Value *V = B.CreateLoad(B.getInt32Ty(), Mode);
+ II->replaceAllUsesWith(V);
+ }
+ II->eraseFromParent();
+ }
+ // The mode at each constrained intrinsic, kept through the promotion by a
+ // freeze.
+ DenseMap<ConstrainedFPIntrinsic *, FreezeInst *> ModeAt;
+ for (ConstrainedFPIntrinsic *CI : Constrained) {
+ IRBuilder<> B(CI);
+ ModeAt[CI] = cast<FreezeInst>(B.CreateFreeze(B.CreateLoad(B.getInt32Ty(), Mode)));
+ }
+ DominatorTree DT(F);
+ PromoteMemToReg({Mode}, DT);
+
+ for (ConstrainedFPIntrinsic *CI : Constrained) {
+ FreezeInst *Fr = ModeAt[CI];
+ Value *Env = Fr->getOperand(0);
+ IRBuilder<> B(CI);
+ // The rounding of the result's format and the denormal mode of the
+ // operands' format.
+ Type *ResTy = CI->getType()->getScalarType();
+ Type *OpTy = CI->getArgOperand(0)->getType()->getScalarType();
+ bool ResHalf = !ResTy->isFloatTy();
+ Value *Rounding;
+ if (std::optional<RoundingMode> RM = CI->getRoundingMode();
+ RM && *RM != RoundingMode::Dynamic)
+ Rounding = B.getInt32(roundingCode(*RM));
+ else
+ Rounding = B.CreateAnd(ResHalf ? B.CreateLShr(Env, ModeHalfShift) : Env, 3);
+ bool HasFPOperand = OpTy->isFloatingPointTy();
+ Type *FlushTy = HasFPOperand ? OpTy : ResTy;
+ bool FlushFloat = FlushTy->isFloatTy();
+ Value *Flush = B.CreateICmpNE(
+ B.CreateAnd(Env, FlushFloat ? ModeFloatFlush : ModeHalfFlush),
+ B.getInt32(0));
+ bool Default = FlushFloat ? FloatFlush : HalfFlush;
+ bool KnownDefault = isa<ConstantInt>(Flush) &&
+ cast<ConstantInt>(Flush)->isOne() == Default;
+ bool Mismatch = !KnownDefault &&
+ (FlushTy->isFloatTy() || FlushTy->isHalfTy()) &&
+ !CI->getType()->isVectorTy();
+ SoftFloat SF(B);
+ // A denormal mode that differs from the function's: flush the operands
+ // explicitly (and the result below).
+ if (Mismatch)
+ for (unsigned I = 0, E = CI->getNonMetadataArgCount(); I < E; ++I)
+ if (CI->getArgOperand(I)->getType() == FlushTy)
+ CI->setArgOperand(I, B.CreateSelect(Flush,
+ SF.flushDenormal(CI->getArgOperand(I)),
+ CI->getArgOperand(I)));
+
+ bool Nearest = isa<ConstantInt>(Rounding) &&
+ cast<ConstantInt>(Rounding)->getZExtValue() == RoundNearestEven;
+ Value *R = Nearest ? nullptr : lowerRounded(CI, Rounding, ST.hasCanonicalNaN());
+ if (!R) {
+ B.SetInsertPoint(CI);
+ R = lowerToPlain(CI, B);
+ }
+ if (!R)
+ continue;
+ if (Mismatch) {
+ if (auto *RI = dyn_cast<Instruction>(R))
+ RI->setMetadata(IEEETag, MDNode::get(F.getContext(), {}));
+ if (R->getType() == FlushTy) {
+ B.SetInsertPoint(CI);
+ R = B.CreateSelect(Flush, SF.flushDenormal(R), R);
+ }
+ }
+ R->takeName(CI);
+ CI->replaceAllUsesWith(R);
+ CI->eraseFromParent();
+ }
+ for (auto &[CI, Fr] : ModeAt) {
+ Fr->replaceAllUsesWith(Fr->getOperand(0));
+ Fr->eraseFromParent();
+ }
+ return true;
+}
diff --git a/llvm/lib/Target/AGX/AGXTargetMachine.cpp b/llvm/lib/Target/AGX/AGXTargetMachine.cpp
index 59869105122b0..d79914d611b02 100644
--- a/llvm/lib/Target/AGX/AGXTargetMachine.cpp
+++ b/llvm/lib/Target/AGX/AGXTargetMachine.cpp
@@ -37,6 +37,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAGXTarget() {
initializeAGXAlwaysInlinePass(PR);
initializeAGXCodeGenPreparePass(PR);
initializeAGXStrictFPPass(PR);
+ initializeAGXLowerFPEnvPass(PR);
initializeAGXAnnotateControlFlowPass(PR);
initializeAGXDAGToDAGISelLegacyPass(PR);
initializeAGXFoldOperandsPass(PR);
@@ -149,6 +150,9 @@ void AGXPassConfig::addIRPasses() {
addPass(createInferAddressSpacesPass());
addPass(createSROAPass());
}
+ // The floating point mode register and constrained floating point
+ // intrinsics (directed rounding in software).
+ addPass(createAGXLowerFPEnvPass());
addPass(createAGXCodeGenPreparePass());
// Software IEEE binary32 arithmetic and canonical NaNs, when enabled.
addPass(createAGXStrictFPPass());
diff --git a/llvm/test/CodeGen/AGX/fp-env.ll b/llvm/test/CodeGen/AGX/fp-env.ll
new file mode 100644
index 0000000000000..ccdb0caf6de3f
--- /dev/null
+++ b/llvm/test/CodeGen/AGX/fp-env.ll
@@ -0,0 +1,57 @@
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -o /dev/null -print-after=agx-lower-fp-env < %s 2>&1 | FileCheck %s
+
+; Round-to-nearest constrained operations become ordinary ones.
+; CHECK-LABEL: @nearest(
+; CHECK: %r = fadd float %a, %b
+define void @nearest(ptr addrspace(1) %out, float %a, float %b) #0 {
+ %r = call float @llvm.experimental.constrained.fadd.f32(float %a, float %b, metadata !"round.tonearest", metadata !"fpexcept.ignore") #0
+ store float %r, ptr addrspace(1) %out
+ ret void
+}
+
+; The mode register starts at round to nearest, so a dynamic rounding with
+; no set.fpmode is an ordinary operation too.
+; CHECK-LABEL: @dynamic_default(
+; CHECK: %r = fmul float %a, %b
+define void @dynamic_default(ptr addrspace(1) %out, float %a, float %b) #0 {
+ %r = call float @llvm.experimental.constrained.fmul.f32(float %a, float %b, metadata !"round.dynamic", metadata !"fpexcept.ignore") #0
+ store float %r, ptr addrspace(1) %out
+ ret void
+}
+
+; A known directed mode is computed in software on the bits, and
+; get.fpmode reads back the value set.
+; CHECK-LABEL: @toward_zero(
+; CHECK-NOT: call {{.*}}constrained
+; CHECK-NOT: fadd float %a, %b{{$}}
+; CHECK: bitcast float %a to i32
+; CHECK-NOT: llvm.agx.{{[sg]}}et.fpmode
+; CHECK: store i32 0, ptr addrspace(1) %mode
+define void @toward_zero(ptr addrspace(1) %out, ptr addrspace(1) %mode, float %a, float %b) #0 {
+ call void @llvm.agx.set.fpmode(i32 0)
+ %r = call float @llvm.experimental.constrained.fadd.f32(float %a, float %b, metadata !"round.dynamic", metadata !"fpexcept.ignore") #0
+ store float %r, ptr addrspace(1) %out
+ %m = call i32 @llvm.agx.get.fpmode()
+ store i32 %m, ptr addrspace(1) %mode
+ ret void
+}
+
+; A mode only known at run time is read from its value: the f32 rounding
+; field (bits 0-1) and the f32 flush bit (bit 4).
+; CHECK-LABEL: @unknown_mode(
+; CHECK-NOT: llvm.agx.{{[sg]}}et.fpmode
+; CHECK-DAG: and i32 %rm, 3
+; CHECK-DAG: and i32 %rm, 16
+define void @unknown_mode(ptr addrspace(1) %out, i32 %rm, float %a, float %b) #0 {
+ call void @llvm.agx.set.fpmode(i32 %rm)
+ %r = call float @llvm.experimental.constrained.fadd.f32(float %a, float %b, metadata !"round.dynamic", metadata !"fpexcept.ignore") #0
+ store float %r, ptr addrspace(1) %out
+ ret void
+}
+
+declare float @llvm.experimental.constrained.fadd.f32(float, float, metadata, metadata)
+declare float @llvm.experimental.constrained.fmul.f32(float, float, metadata, metadata)
+declare void @llvm.agx.set.fpmode(i32)
+declare i32 @llvm.agx.get.fpmode()
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/AGX/fp.ll b/llvm/test/CodeGen/AGX/fp.ll
index c87d98ccc02b6..c5fefcbb0a8e7 100644
--- a/llvm/test/CodeGen/AGX/fp.ll
+++ b/llvm/test/CodeGen/AGX/fp.ll
@@ -38,14 +38,19 @@ define agx_kernel void @arith(ptr addrspace(1) %out, float %a, float %b, float %
define agx_kernel void @div(ptr addrspace(1) %out, float %a, float %b) {
; CHECK-LABEL: div:
; CHECK: // uniform registers: 14 x 16-bit (threadgroups_per_grid at u4)
-; CHECK-NEXT: // general purpose registers: 4 x 32-bit
+; CHECK-NEXT: // general purpose registers: 6 x 32-bit
; CHECK-NEXT: // threadgroup memory: 0 bytes
; CHECK-NEXT: // stack size: 0 bytes
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: rcp r1, u3
; CHECK-NEXT: fmul32 r2, u2, r1
; CHECK-NEXT: fmadd32 r3, u3.neg, r2, u2
-; CHECK-NEXT: fmadd32 r2, r3, r1, r2
+; CHECK-NEXT: mov_imm r4, 2139095040
+; CHECK-NEXT: fcmpsel eq, r4, r2.abs, r4, 1, 0
+; CHECK-NEXT: fcmpsel eq, r5, r2.abs, 0.0, 1, 0
+; CHECK-NEXT: fmadd32 r3, r3, r1, r2
+; CHECK-NEXT: or r4, r5, r4
+; CHECK-NEXT: icmpsel ueq, r2, r4, 0, r3, r2
; CHECK-NEXT: fmul32 r1, r2, r1
; CHECK-NEXT: rcp r1, r1
; CHECK-NEXT: device_store 0, i32, x, r1, u0_u1, 0, signed
>From 616b95fe8e0a4d8d774fd2ba88408d651cd16ae9 Mon Sep 17 00:00:00 2001
From: Claude <noreply at anthropic.com>
Date: Mon, 19 Oct 2026 00:34:44 +0000
Subject: [PATCH 13/22] [AGX] Select negative constant device offsets as signed
immediates
SelectDeviceAddr built the immediate index of a negative offset with
getTargetConstant, which asserts on a value that does not fit unsigned in
32 bits; it is now a signed target constant (the index signedness field is
already 0, signed).
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
Claude-Session: https://claude.ai/code/session_01E56KEecg6edNaS23myRWER
---
llvm/lib/Target/AGX/AGXISelDAGToDAG.cpp | 2 +-
llvm/test/CodeGen/AGX/neg-offset.ll | 10 ++++++++++
2 files changed, 11 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/AGX/neg-offset.ll
diff --git a/llvm/lib/Target/AGX/AGXISelDAGToDAG.cpp b/llvm/lib/Target/AGX/AGXISelDAGToDAG.cpp
index 0ed8d58b88dfa..c29aa7f357b20 100644
--- a/llvm/lib/Target/AGX/AGXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AGX/AGXISelDAGToDAG.cpp
@@ -287,7 +287,7 @@ bool AGXDAGToDAGISel::SelectDeviceAddr(SDValue Addr, unsigned ElemLog2,
if (Off % (int64_t(1) << ElemLog2) == 0 &&
isInt<16>(Off >> ElemLog2)) {
Base = B;
- Index = CurDAG->getTargetConstant(Off >> ElemLog2, DL, MVT::i32);
+ Index = CurDAG->getSignedTargetConstant(Off >> ElemLog2, DL, MVT::i32);
return true;
}
continue;
diff --git a/llvm/test/CodeGen/AGX/neg-offset.ll b/llvm/test/CodeGen/AGX/neg-offset.ll
new file mode 100644
index 0000000000000..4b1e106d543b4
--- /dev/null
+++ b/llvm/test/CodeGen/AGX/neg-offset.ll
@@ -0,0 +1,10 @@
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -verify-machineinstrs < %s | FileCheck %s
+
+; A negative constant offset is a signed immediate index.
+; CHECK-LABEL: neg_offset:
+; CHECK: device_store {{.*}}, u0_u1, -1, signed
+define agx_kernel void @neg_offset(ptr addrspace(1) %p) {
+ %q = getelementptr inbounds i8, ptr addrspace(1) %p, i64 -4
+ store i32 22, ptr addrspace(1) %q, align 4
+ ret void
+}
>From 7aeb0c8773471a13740d9d627e3ff45879eab598 Mon Sep 17 00:00:00 2001
From: Claude <noreply at anthropic.com>
Date: Mon, 19 Oct 2026 00:51:36 +0000
Subject: [PATCH 14/22] [AGX] Compute double precision arithmetic in software
AGX has no double precision hardware. The new agx-soft-f64 pass replaces
double operations (arithmetic, fma, sqrt, comparisons, conversions,
min/max, rounding intrinsics, is.fpclass, saturating conversions) with
integer arithmetic on the bits: exact sign, 128-bit significand and
exponent, an exact result with a sticky bit, and one rounding. The results
are correctly rounded IEEE 754 with denormals in all four rounding modes;
agx-lower-fp-env hands double constrained operations with a directed or
dynamic rounding to the same code. Moved double values become i64 in type
legalization; frem and vector double arithmetic stay diagnosed.
ZLUDA's ptx/tools/agx_f64_check.py checks every operation bit by bit on
the functional simulator against an exact reference, in all rounding modes
(static and dynamic) and with and without canonical-nan.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
Claude-Session: https://claude.ai/code/session_01E56KEecg6edNaS23myRWER
---
llvm/docs/AGXUsage.rst | 33 +-
llvm/lib/Target/AGX/AGX.h | 2 +
llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp | 4 +-
llvm/lib/Target/AGX/AGXISelLowering.cpp | 3 +-
llvm/lib/Target/AGX/AGXSoftF64.cpp | 809 ++++++++++++++++++++++
llvm/lib/Target/AGX/AGXSoftF64.h | 31 +
llvm/lib/Target/AGX/AGXStrictFP.cpp | 12 +-
llvm/lib/Target/AGX/AGXTargetMachine.cpp | 2 +
llvm/lib/Target/AGX/CMakeLists.txt | 1 +
llvm/test/CodeGen/AGX/errors-f64.ll | 10 +-
llvm/test/CodeGen/AGX/soft-f64.ll | 61 ++
11 files changed, 958 insertions(+), 10 deletions(-)
create mode 100644 llvm/lib/Target/AGX/AGXSoftF64.cpp
create mode 100644 llvm/lib/Target/AGX/AGXSoftF64.h
create mode 100644 llvm/test/CodeGen/AGX/soft-f64.ll
diff --git a/llvm/docs/AGXUsage.rst b/llvm/docs/AGXUsage.rst
index 6d13e0dd43961..034f6982ded76 100644
--- a/llvm/docs/AGXUsage.rst
+++ b/llvm/docs/AGXUsage.rst
@@ -251,7 +251,8 @@ in functions whose ``denormal_fpenv`` flushes ``half`` denormals, the
operands and results of ``half`` arithmetic, comparisons and conversions are
flushed explicitly.
-Not supported: ``double`` (diagnosed as unsupported), vector memory
+Not supported: ``frem`` and vector arithmetic on ``double`` (see `Double
+precision`_; diagnosed as unsupported), vector memory
operations (vectors are scalarized), textures and graphics shaders, calls
and recursion.
@@ -332,6 +333,36 @@ normal operands, ``ieee-fp32`` adds about 8 instructions to an addition, 12
to a multiplication, 22 to a fused multiply-add, 13 to a comparison, 65 to a
division and 39 to a square root.
+Double precision
+----------------
+
+The hardware has no double precision arithmetic. The ``agx-soft-f64`` pass
+replaces the ``double`` operations with integer arithmetic on their bits:
+``fadd``, ``fsub``, ``fmul``, ``fdiv``, ``fneg``, ``fcmp``, ``fpext`` and
+``fptrunc`` (from and to ``float``, ``half`` and ``bfloat``), ``sitofp``,
+``uitofp``, ``fptosi``, ``fptoui``, and ``llvm.fma``, ``llvm.fmuladd``,
+``llvm.sqrt``, ``llvm.fabs``, ``llvm.copysign``, the minimum and maximum
+intrinsics, the rounding intrinsics (``floor``, ``ceil``, ``trunc``,
+``rint``, ``nearbyint``, ``roundeven``, ``round``), ``llvm.canonicalize``,
+``llvm.is.fpclass`` and the saturating conversions. Each operand becomes an
+exact sign, 128-bit significand and exponent, the exact result (with a sticky
+bit for the bits dropped) is computed with integer operations, and a single
+rounding gives the result, so all operations are correctly rounded IEEE 754
+operations with denormals, in the four rounding directions (see `Rounding
+modes`_; the ``double`` rounding mode is the ``half`` field). Division uses an
+expanded 128-bit integer division and the square root a loop that produces a
+bit of the root per iteration. NaN results are ``0x7fffffffffffffff`` with
+``canonical-nan``, and the first NaN operand, quieted, otherwise. Conversions
+to integers saturate, and NaN converts to 0.
+
+Values that are only moved (loads, stores, ``phi``, ``select``, arguments)
+stay ``double`` and become ``i64`` in type legalization. ``frem`` and vector
+arithmetic on ``double`` are diagnosed as unsupported. The code is large: in
+instructions per operation, about 590 for an addition, 350 for a
+multiplication, 690 for a fused multiply-add, 780 for a division (plus the
+iterations of the division loop), 460 for a square root (and 64 iterations),
+230 for a conversion and 20 for a comparison.
+
Rounding modes
--------------
diff --git a/llvm/lib/Target/AGX/AGX.h b/llvm/lib/Target/AGX/AGX.h
index 5dc429c4f61c0..44f7b61b84903 100644
--- a/llvm/lib/Target/AGX/AGX.h
+++ b/llvm/lib/Target/AGX/AGX.h
@@ -27,6 +27,7 @@ ModulePass *createAGXAlwaysInlinePass();
FunctionPass *createAGXCodeGenPreparePass();
FunctionPass *createAGXStrictFPPass();
FunctionPass *createAGXLowerFPEnvPass();
+FunctionPass *createAGXSoftF64Pass();
FunctionPass *createAGXAnnotateControlFlowPass();
// Instruction selection.
@@ -41,6 +42,7 @@ void initializeAGXAlwaysInlinePass(PassRegistry &);
void initializeAGXCodeGenPreparePass(PassRegistry &);
void initializeAGXStrictFPPass(PassRegistry &);
void initializeAGXLowerFPEnvPass(PassRegistry &);
+void initializeAGXSoftF64Pass(PassRegistry &);
void initializeAGXAnnotateControlFlowPass(PassRegistry &);
void initializeAGXDAGToDAGISelLegacyPass(PassRegistry &);
void initializeAGXFoldOperandsPass(PassRegistry &);
diff --git a/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp b/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp
index cf09418d0da87..0b94e0493921b 100644
--- a/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp
@@ -24,8 +24,8 @@
// * The hardware flushes 32-bit denormals but keeps 16-bit ones. Functions
// whose denormal mode for half asks for flushing get explicit flushes of
// the operands and results of their half operations.
-// * There is no double precision hardware: double precision arithmetic is
-// diagnosed as unsupported.
+// * There is no double precision hardware: the double precision arithmetic
+// that AGXSoftF64 does not implement is diagnosed as unsupported.
//
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AGX/AGXISelLowering.cpp b/llvm/lib/Target/AGX/AGXISelLowering.cpp
index 67d77a2eb4fa8..84d46a1c601d4 100644
--- a/llvm/lib/Target/AGX/AGXISelLowering.cpp
+++ b/llvm/lib/Target/AGX/AGXISelLowering.cpp
@@ -8,7 +8,8 @@
//
// Lowering of LLVM IR to the AGX selection DAG.
//
-// Legal types are i16, i32, i64, f16 and f32. There is no f64 support and no
+// Legal types are i16, i32, i64, f16 and f32. f64 values are softened to i64
+// (their arithmetic is integer code from AGXSoftF64), and there are no
// vector types: vectors are scalarized. Booleans are 0/1 in 32-bit registers.
// 64-bit integers are native for add, sub, multiply and bitwise operations;
// shifts, compares and bit counts are split into 32-bit halves here.
diff --git a/llvm/lib/Target/AGX/AGXSoftF64.cpp b/llvm/lib/Target/AGX/AGXSoftF64.cpp
new file mode 100644
index 0000000000000..f2d7342aa6b43
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXSoftF64.cpp
@@ -0,0 +1,809 @@
+//===-- AGXSoftF64.cpp - Double precision in software for AGX -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// AGX has no double precision hardware. This pass replaces double precision
+// arithmetic with integer arithmetic on the bits of the values: each operand
+// becomes an exact sign * M * 2^E with a 128-bit M, the exact result (with a
+// sticky bit where bits are dropped) is computed with integer operations, and
+// a single rounding produces the bits of the result. The results are those of
+// IEEE 754 with denormals, in all four rounding directions. Values that are
+// only moved (loads, stores, phis, selects, arguments) stay double, and the
+// type legalizer turns them into i64.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AGXSoftF64.h"
+#include "AGX.h"
+#include "AGXSubtarget.h"
+#include "llvm/ADT/FloatingPointMode.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstIterator.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Transforms/Utils/IntegerDivision.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "agx-soft-f64"
+
+namespace {
+
+/// A binary interchange format.
+struct Fmt {
+ unsigned P; // significand bits, including the implicit one
+ int Bias;
+ unsigned W;
+ int minLSB() const { return 2 - Bias - int(P); }
+ uint64_t expAll() const { return 2 * Bias + 1; }
+ uint64_t fracMask() const { return (uint64_t(1) << (P - 1)) - 1; }
+ uint64_t infBits() const { return expAll() << (P - 1); }
+ uint64_t signBit() const { return uint64_t(1) << (W - 1); }
+ uint64_t maxFinite() const { return infBits() - 1; }
+ uint64_t quietBit() const { return uint64_t(1) << (P - 2); }
+};
+constexpr Fmt F64 = {53, 1023, 64};
+constexpr Fmt F32 = {24, 127, 32};
+constexpr Fmt F16 = {11, 15, 16};
+constexpr Fmt BF16 = {8, 127, 16};
+
+enum RoundingCode : uint32_t {
+ RoundTowardZero = 0,
+ RoundNearestEven = 1,
+ RoundUp = 2,
+ RoundDown = 3,
+};
+
+/// sign * M * 2^E, with Sign i1, M i128 and E i32.
+struct Term {
+ Value *Sign;
+ Value *M;
+ Value *E;
+};
+
+class SoftDouble {
+ IRBuilderBase &B;
+ Type *I1, *I32, *I64, *I128;
+ Value *Rounding;
+ bool CanonicalNaN;
+
+public:
+ /// The 128-bit divisions built, which the caller expands into integer
+ /// arithmetic once the replacement is complete.
+ SmallVector<BinaryOperator *, 1> Divisions;
+
+ SoftDouble(IRBuilderBase &B, Value *Rounding, bool CanonicalNaN)
+ : B(B), I1(B.getInt1Ty()), I32(B.getInt32Ty()), I64(B.getInt64Ty()),
+ I128(B.getInt128Ty()), Rounding(Rounding), CanonicalNaN(CanonicalNaN) {}
+
+ Value *c32(int64_t V) { return ConstantInt::getSigned(I32, V); }
+ Value *c64(uint64_t V) { return B.getInt64(V); }
+ Value *c128(uint64_t V) { return ConstantInt::get(I128, V); }
+ Value *roundingIs(RoundingCode C) {
+ return B.CreateICmpEQ(Rounding, B.getInt32(C));
+ }
+ Value *select(Value *C, Value *T, Value *F) { return B.CreateSelect(C, T, F); }
+ Value *smax(Value *A, Value *X) {
+ return B.CreateBinaryIntrinsic(Intrinsic::smax, A, X);
+ }
+ Value *umin(Value *A, Value *X) {
+ return B.CreateBinaryIntrinsic(Intrinsic::umin, A, X);
+ }
+
+ /// Bit length of a 128-bit value, as i32.
+ Value *bitLength(Value *M) {
+ Value *Lz = B.CreateBinaryIntrinsic(Intrinsic::ctlz, M, B.getFalse());
+ return B.CreateTrunc(B.CreateSub(c128(128), Lz), I32);
+ }
+ /// M << S and M >> S for an i32 S >= 0; amounts of 128 or more give 0.
+ Value *shl(Value *M, Value *S) {
+ Value *R = B.CreateShl(M, B.CreateZExt(umin(S, c32(127)), I128));
+ return select(B.CreateICmpUGT(S, c32(127)), c128(0), R);
+ }
+ Value *lshr(Value *M, Value *S) {
+ Value *R = B.CreateLShr(M, B.CreateZExt(umin(S, c32(127)), I128));
+ return select(B.CreateICmpUGT(S, c32(127)), c128(0), R);
+ }
+ /// M >> S with the bits shifted out ORed into the lowest bit.
+ Value *lshrSticky(Value *M, Value *S) {
+ Value *R = lshr(M, S);
+ Value *Lost = B.CreateICmpNE(shl(R, S), M);
+ return B.CreateOr(R, B.CreateZExt(Lost, I128));
+ }
+
+ // Classification of bits (an i64 holding a value of format F).
+ Value *magnitude(Value *Bits, Fmt F) {
+ return B.CreateAnd(Bits, c64(F.signBit() - 1));
+ }
+ Value *signOf(Value *Bits, Fmt F) {
+ return B.CreateICmpNE(B.CreateAnd(Bits, c64(F.signBit())), c64(0));
+ }
+ Value *isNaN(Value *Bits, Fmt F) {
+ return B.CreateICmpUGT(magnitude(Bits, F), c64(F.infBits()));
+ }
+ Value *isInf(Value *Bits, Fmt F) {
+ return B.CreateICmpEQ(magnitude(Bits, F), c64(F.infBits()));
+ }
+ Value *isZero(Value *Bits, Fmt F) {
+ return B.CreateICmpEQ(magnitude(Bits, F), c64(0));
+ }
+ Value *isInfOrNaN(Value *Bits, Fmt F) {
+ return B.CreateICmpUGE(magnitude(Bits, F), c64(F.infBits()));
+ }
+
+ /// The NaN result of an operation with the given operands of format In,
+ /// in format Out: the canonical NaN, or the first NaN operand quieted
+ /// (the default NaN if there is none).
+ Value *nanResult(ArrayRef<Value *> Ops, Fmt In, Fmt Out) {
+ if (CanonicalNaN)
+ return c64(Out.signBit() - 1);
+ Value *R = c64(Out.infBits() | Out.quietBit());
+ for (Value *X : reverse(Ops)) {
+ Value *Payload = B.CreateAnd(X, c64(In.fracMask()));
+ Payload = In.P > Out.P ? B.CreateLShr(Payload, In.P - Out.P)
+ : B.CreateShl(Payload, Out.P - In.P);
+ Value *Q = B.CreateOr(B.CreateOr(Payload, c64(Out.infBits() | Out.quietBit())),
+ select(signOf(X, In), c64(Out.signBit()), c64(0)));
+ R = select(isNaN(X, In), Q, R);
+ }
+ return R;
+ }
+
+ Term unpack(Value *Bits, Fmt F) {
+ Value *Exp = B.CreateAnd(B.CreateLShr(Bits, F.P - 1), c64(F.expAll()));
+ Value *Frac = B.CreateAnd(Bits, c64(F.fracMask()));
+ Value *Denormal = B.CreateICmpEQ(Exp, c64(0));
+ Value *M = select(Denormal, Frac, B.CreateOr(Frac, c64(F.fracMask() + 1)));
+ Value *E = B.CreateTrunc(select(Denormal, c64(1), Exp), I32);
+ E = B.CreateSub(E, c32(F.Bias + int(F.P) - 1));
+ return {signOf(Bits, F), B.CreateZExt(M, I128), E};
+ }
+
+ /// T rounded to format F in the rounding direction, as bits in an i64.
+ /// T.M may hold a sticky bit below the bits that matter.
+ Value *round(Term T, Fmt F) {
+ Value *L = bitLength(T.M);
+ Value *Shift = smax(B.CreateSub(L, c32(F.P)), B.CreateSub(c32(F.minLSB()), T.E));
+ Value *Right = B.CreateICmpSGT(Shift, c32(0));
+ Value *RS = smax(Shift, c32(0));
+ Value *Q = select(Right, lshr(T.M, RS),
+ shl(T.M, B.CreateNeg(B.CreateBinaryIntrinsic(Intrinsic::smin, Shift, c32(0)))));
+ Value *Rem = B.CreateSub(T.M, shl(lshr(T.M, RS), RS));
+ Value *GuardS = B.CreateSub(RS, c32(1));
+ Value *Guard = B.CreateTrunc(lshr(Rem, umin(GuardS, c32(128))), I1);
+ // The bits below the guard bit; all of them if the guard bit is above
+ // bit 127.
+ Value *Half = shl(c128(1), GuardS);
+ Value *Sticky = B.CreateICmpNE(B.CreateAnd(Rem, B.CreateSub(Half, c128(1))), c128(0));
+ Guard = B.CreateAnd(Guard, Right);
+ Sticky = B.CreateAnd(Sticky, Right);
+ Value *Odd = B.CreateTrunc(Q, I1);
+ Value *Inexact = B.CreateOr(Guard, Sticky);
+ Value *IncNearest = B.CreateAnd(Guard, B.CreateOr(Sticky, Odd));
+ Value *Inc = select(roundingIs(RoundNearestEven), IncNearest,
+ select(roundingIs(RoundUp), B.CreateAnd(Inexact, B.CreateNot(T.Sign)),
+ select(roundingIs(RoundDown), B.CreateAnd(Inexact, T.Sign),
+ B.getFalse())));
+ Q = B.CreateAdd(Q, B.CreateZExt(Inc, I128));
+ // bits = k << (P - 1) + Q, with k the biased exponent of the lowest bit
+ // of Q; a carry out of Q increments the exponent.
+ Value *K = B.CreateSub(B.CreateAdd(T.E, Shift), c32(F.minLSB()));
+ K = umin(K, c32(F.expAll() + 1));
+ Value *Bits = B.CreateAdd(B.CreateShl(B.CreateZExt(K, I64), F.P - 1),
+ B.CreateTrunc(Q, I64));
+ Bits = select(B.CreateICmpEQ(T.M, c128(0)), c64(0), Bits);
+ Value *Overflow = B.CreateICmpUGE(Bits, c64(F.infBits()));
+ // Overflows round to infinity or to the largest finite value.
+ Value *ToInf = select(roundingIs(RoundNearestEven), B.getTrue(),
+ select(roundingIs(RoundUp), B.CreateNot(T.Sign),
+ select(roundingIs(RoundDown), T.Sign, B.getFalse())));
+ Bits = select(Overflow, select(ToInf, c64(F.infBits()), c64(F.maxFinite())), Bits);
+ return B.CreateOr(Bits, select(T.Sign, c64(F.signBit()), c64(0)));
+ }
+
+ /// The exact sum of A and B (with M of at most 126 bits), with a sticky bit.
+ Term addTerms(Term X, Term Y) {
+ auto Normalize = [&](Term T) {
+ Value *S = B.CreateSub(c32(126), bitLength(T.M));
+ return Term{T.Sign, shl(T.M, S), B.CreateSub(T.E, S)};
+ };
+ X = Normalize(X);
+ Y = Normalize(Y);
+ // Order by magnitude; zeros are the smallest.
+ Value *XZero = B.CreateICmpEQ(X.M, c128(0));
+ Value *YZero = B.CreateICmpEQ(Y.M, c128(0));
+ Value *YBigger = B.CreateOr(
+ B.CreateICmpSGT(Y.E, X.E),
+ B.CreateAnd(B.CreateICmpEQ(Y.E, X.E), B.CreateICmpUGT(Y.M, X.M)));
+ YBigger = B.CreateAnd(B.CreateNot(YZero), B.CreateOr(XZero, YBigger));
+ Term Big{select(YBigger, Y.Sign, X.Sign), select(YBigger, Y.M, X.M),
+ select(YBigger, Y.E, X.E)};
+ Term Small{select(YBigger, X.Sign, Y.Sign), select(YBigger, X.M, Y.M),
+ select(YBigger, X.E, Y.E)};
+ Value *D = B.CreateSub(Big.E, Small.E);
+ Value *SM = select(B.CreateICmpEQ(Small.M, c128(0)), c128(0),
+ lshrSticky(Small.M, D));
+ Value *Same = B.CreateICmpEQ(Big.Sign, Small.Sign);
+ Value *M = select(Same, B.CreateAdd(Big.M, SM), B.CreateSub(Big.M, SM));
+ return {Big.Sign, M, Big.E};
+ }
+
+ /// The sign of an exact zero sum of operands with signs SX and SY that
+ /// are both zero (BothZero) or cancel.
+ Value *zeroSumSign(Value *BothZero, Value *SX, Value *SY) {
+ Value *Down = roundingIs(RoundDown);
+ return select(BothZero, select(Down, B.CreateOr(SX, SY), B.CreateAnd(SX, SY)), Down);
+ }
+
+ Value *withZeroSign(Term T, Value *ZeroSign) {
+ // round() gives the sign of T to a zero.
+ return round({select(B.CreateICmpEQ(T.M, c128(0)), ZeroSign, T.Sign), T.M, T.E}, F64);
+ }
+
+ Value *add(Value *X, Value *Y, bool Sub) {
+ if (Sub)
+ Y = B.CreateXor(Y, c64(F64.signBit()));
+ Term TX = unpack(X, F64), TY = unpack(Y, F64);
+ Term S = addTerms(TX, TY);
+ Value *R = withZeroSign(
+ S, zeroSumSign(B.CreateAnd(isZero(X, F64), isZero(Y, F64)), TX.Sign, TY.Sign));
+ // Infinities: the infinite operand, or NaN for opposite infinities.
+ Value *XInf = isInf(X, F64), *YInf = isInf(Y, F64);
+ Value *Special = select(XInf, X, Y);
+ Value *NaN = B.CreateOr(B.CreateOr(isNaN(X, F64), isNaN(Y, F64)),
+ B.CreateAnd(B.CreateAnd(XInf, YInf), B.CreateICmpNE(X, Y)));
+ R = select(B.CreateOr(XInf, YInf), Special, R);
+ Value *OrigY = Sub ? B.CreateXor(Y, c64(F64.signBit())) : Y;
+ return select(NaN, nanResult({X, OrigY}, F64, F64), R);
+ }
+
+ Value *mul(Value *X, Value *Y) {
+ Term TX = unpack(X, F64), TY = unpack(Y, F64);
+ Value *Sign = B.CreateXor(TX.Sign, TY.Sign);
+ Value *R = round({Sign, B.CreateMul(TX.M, TY.M), B.CreateAdd(TX.E, TY.E)}, F64);
+ Value *XInf = isInf(X, F64), *YInf = isInf(Y, F64);
+ Value *Inf = B.CreateOr(c64(F64.infBits()), select(Sign, c64(F64.signBit()), c64(0)));
+ Value *NaN = B.CreateOr(
+ B.CreateOr(isNaN(X, F64), isNaN(Y, F64)),
+ B.CreateOr(B.CreateAnd(XInf, isZero(Y, F64)), B.CreateAnd(YInf, isZero(X, F64))));
+ R = select(B.CreateOr(XInf, YInf), Inf, R);
+ return select(NaN, nanResult({X, Y}, F64, F64), R);
+ }
+
+ Value *fma(Value *X, Value *Y, Value *Z) {
+ Term TX = unpack(X, F64), TY = unpack(Y, F64), TZ = unpack(Z, F64);
+ Term P{B.CreateXor(TX.Sign, TY.Sign), B.CreateMul(TX.M, TY.M),
+ B.CreateAdd(TX.E, TY.E)};
+ Term S = addTerms(P, TZ);
+ Value *PZero = B.CreateOr(isZero(X, F64), isZero(Y, F64));
+ Value *R = withZeroSign(
+ S, zeroSumSign(B.CreateAnd(PZero, isZero(Z, F64)), P.Sign, TZ.Sign));
+ Value *XInf = isInf(X, F64), *YInf = isInf(Y, F64), *ZInf = isInf(Z, F64);
+ Value *PInf = B.CreateOr(XInf, YInf);
+ Value *PInfBits = B.CreateOr(c64(F64.infBits()),
+ select(P.Sign, c64(F64.signBit()), c64(0)));
+ Value *NaN = B.CreateOr(B.CreateOr(isNaN(X, F64), isNaN(Y, F64)), isNaN(Z, F64));
+ NaN = B.CreateOr(NaN, B.CreateOr(B.CreateAnd(XInf, isZero(Y, F64)),
+ B.CreateAnd(YInf, isZero(X, F64))));
+ NaN = B.CreateOr(NaN, B.CreateAnd(B.CreateAnd(PInf, ZInf),
+ B.CreateICmpNE(P.Sign, TZ.Sign)));
+ R = select(ZInf, Z, R);
+ R = select(PInf, PInfBits, R);
+ return select(NaN, nanResult({X, Y, Z}, F64, F64), R);
+ }
+
+ Value *div(Value *X, Value *Y) {
+ Term TX = unpack(X, F64), TY = unpack(Y, F64);
+ Value *Sign = B.CreateXor(TX.Sign, TY.Sign);
+ // The dividend with its top bit at 126 over the divisor with its top bit
+ // at 63: a quotient of at least 63 bits.
+ Value *SX = B.CreateSub(c32(126), bitLength(TX.M));
+ Value *SY = B.CreateSub(c32(63), bitLength(TY.M));
+ Value *N = shl(TX.M, SX);
+ Value *Dv = shl(TY.M, SY);
+ Value *SafeDv = select(B.CreateICmpEQ(Dv, c128(0)), c128(1), Dv);
+ Value *Q = B.CreateUDiv(N, SafeDv);
+ Divisions.push_back(cast<BinaryOperator>(Q));
+ Value *Inexact = B.CreateICmpNE(B.CreateMul(Q, SafeDv), N);
+ Value *M = B.CreateOr(B.CreateShl(Q, 1), B.CreateZExt(Inexact, I128));
+ Value *E = B.CreateSub(B.CreateSub(TX.E, SX), B.CreateSub(TY.E, SY));
+ E = B.CreateSub(E, c32(1));
+ Value *R = round({Sign, M, E}, F64);
+ Value *SignBits = select(Sign, c64(F64.signBit()), c64(0));
+ Value *XZero = isZero(X, F64), *YZero = isZero(Y, F64);
+ Value *XInf = isInf(X, F64), *YInf = isInf(Y, F64);
+ R = select(B.CreateOr(XZero, YInf), SignBits, R);
+ R = select(B.CreateOr(XInf, YZero), B.CreateOr(c64(F64.infBits()), SignBits), R);
+ Value *NaN = B.CreateOr(B.CreateOr(isNaN(X, F64), isNaN(Y, F64)),
+ B.CreateOr(B.CreateAnd(XZero, YZero), B.CreateAnd(XInf, YInf)));
+ return select(NaN, nanResult({X, Y}, F64, F64), R);
+ }
+
+ /// The integer square root of the i128 N (and whether it is exact), with a
+ /// loop that produces a bit per iteration.
+ std::pair<Value *, Value *> isqrt(Value *N) {
+ BasicBlock *Pre = B.GetInsertBlock();
+ Instruction *At = &*B.GetInsertPoint();
+ BasicBlock *Tail = Pre->splitBasicBlock(At->getIterator(), "isqrt.end");
+ Function *F = Pre->getParent();
+ BasicBlock *Loop = BasicBlock::Create(B.getContext(), "isqrt", F, Tail);
+ Pre->getTerminator()->setSuccessor(0, Loop);
+ B.SetInsertPoint(Loop);
+ PHINode *I = B.CreatePHI(I32, 2);
+ PHINode *Rem = B.CreatePHI(I128, 2);
+ PHINode *Root = B.CreatePHI(I128, 2);
+ I->addIncoming(c32(63), Pre);
+ Rem->addIncoming(c128(0), Pre);
+ Root->addIncoming(c128(0), Pre);
+ Value *Pair = B.CreateAnd(
+ B.CreateLShr(N, B.CreateZExt(B.CreateShl(I, 1), I128)), c128(3));
+ Value *R2 = B.CreateOr(B.CreateShl(Rem, 2), Pair);
+ Value *Rt = B.CreateShl(Root, 1);
+ Value *T = B.CreateOr(B.CreateShl(Rt, 1), c128(1));
+ Value *Ge = B.CreateICmpUGE(R2, T);
+ Value *NewRem = select(Ge, B.CreateSub(R2, T), R2);
+ Value *NewRoot = B.CreateOr(Rt, B.CreateZExt(Ge, I128));
+ Value *NextI = B.CreateSub(I, c32(1));
+ I->addIncoming(NextI, Loop);
+ Rem->addIncoming(NewRem, Loop);
+ Root->addIncoming(NewRoot, Loop);
+ B.CreateCondBr(B.CreateICmpEQ(I, c32(0)), Tail, Loop);
+ B.SetInsertPoint(At);
+ PHINode *RootOut = B.CreatePHI(I128, 1);
+ RootOut->addIncoming(NewRoot, Loop);
+ PHINode *RemOut = B.CreatePHI(I128, 1);
+ RemOut->addIncoming(NewRem, Loop);
+ return {RootOut, B.CreateICmpEQ(RemOut, c128(0))};
+ }
+
+ Value *sqrt(Value *X) {
+ Term TX = unpack(X, F64);
+ // The radicand with its top bit at 125 or 126 and an even exponent.
+ Value *S = B.CreateSub(c32(126), bitLength(TX.M));
+ Value *Odd = B.CreateTrunc(B.CreateSub(TX.E, S), I1);
+ S = B.CreateSub(S, B.CreateZExt(Odd, I32));
+ Value *N = shl(TX.M, S);
+ Value *E = B.CreateSub(TX.E, S);
+ auto [Root, Exact] = isqrt(N);
+ Value *M = B.CreateOr(B.CreateShl(Root, 1), B.CreateZExt(B.CreateNot(Exact), I128));
+ Value *R = round({B.getFalse(), M, B.CreateSub(B.CreateAShr(E, 1), c32(1))}, F64);
+ R = select(B.CreateOr(isZero(X, F64), B.CreateICmpEQ(X, c64(F64.infBits()))), X, R);
+ Value *NaN = B.CreateOr(isNaN(X, F64),
+ B.CreateAnd(signOf(X, F64), B.CreateNot(isZero(X, F64))));
+ return select(NaN, nanResult({X}, F64, F64), R);
+ }
+
+ /// A value of format In converted to format Out.
+ Value *convert(Value *X, Fmt In, Fmt Out) {
+ Value *R = round(unpack(X, In), Out);
+ Value *Inf = B.CreateOr(c64(Out.infBits()),
+ select(signOf(X, In), c64(Out.signBit()), c64(0)));
+ R = select(isInf(X, In), Inf, R);
+ return select(isNaN(X, In), nanResult({X}, In, Out), R);
+ }
+
+ Value *fromInt(Value *V, bool Signed) {
+ unsigned N = V->getType()->getIntegerBitWidth();
+ Value *Neg = Signed ? B.CreateICmpSLT(V, ConstantInt::get(V->getType(), 0))
+ : B.getFalse();
+ Value *Mag = Signed ? select(Neg, B.CreateNeg(V), V) : V;
+ Value *M = N < 128 ? B.CreateZExt(Mag, I128) : Mag;
+ return round({Neg, M, c32(0)}, F64);
+ }
+
+ /// X truncated toward zero to an integer of the type Ty, saturated (NaN
+ /// gives 0).
+ Value *toInt(Value *X, Type *Ty, bool Signed) {
+ unsigned N = Ty->getIntegerBitWidth();
+ Term T = unpack(X, F64);
+ Value *E = T.E;
+ Value *Big = B.CreateICmpSGT(E, c32(int(N)));
+ Value *V = select(B.CreateICmpSGE(E, c32(0)),
+ shl(T.M, umin(E, c32(127))),
+ lshr(T.M, B.CreateNeg(B.CreateBinaryIntrinsic(Intrinsic::smin, E, c32(0)))));
+ APInt Max = Signed ? APInt::getSignedMaxValue(N) : APInt::getMaxValue(N);
+ APInt MinMag = Signed ? APInt::getOneBitSet(N, N - 1) : APInt(N, 0);
+ Value *MaxV = ConstantInt::get(I128, Max.zext(128));
+ Value *MinMagV = ConstantInt::get(I128, MinMag.zext(128));
+ Value *Lim = select(T.Sign, MinMagV, MaxV);
+ Value *Mag = select(B.CreateOr(Big, B.CreateICmpUGT(V, Lim)), Lim, V);
+ Value *R = B.CreateTrunc(Mag, Ty);
+ R = select(T.Sign, B.CreateNeg(R), R);
+ if (!Signed)
+ R = select(T.Sign, ConstantInt::get(Ty, 0), R);
+ return select(isNaN(X, F64), ConstantInt::get(Ty, 0), R);
+ }
+
+ enum Integral { Floor, Ceil, Trunc, Nearest, Even, Away };
+ Value *integral(Value *X, Integral Mode) {
+ Term T = unpack(X, F64);
+ Value *NE = B.CreateNeg(B.CreateBinaryIntrinsic(Intrinsic::smin, T.E, c32(0)));
+ Value *Q = lshr(T.M, NE);
+ Value *Rem = B.CreateSub(T.M, shl(Q, NE));
+ Value *Half = shl(c128(1), B.CreateSub(NE, c32(1)));
+ Value *HasFrac = B.CreateICmpNE(Rem, c128(0));
+ // Below 2^-75 there is no half to reach (Half is then 0).
+ Value *HasHalf = B.CreateAnd(B.CreateICmpNE(NE, c32(0)), B.CreateICmpULE(NE, c32(128)));
+ Value *AboveHalf = B.CreateAnd(HasHalf, B.CreateICmpUGT(Rem, Half));
+ Value *IsHalf = B.CreateAnd(HasHalf, B.CreateICmpEQ(Rem, Half));
+ Value *Odd = B.CreateTrunc(Q, I1);
+ Value *IncEven = B.CreateOr(AboveHalf, B.CreateAnd(IsHalf, Odd));
+ Value *Inc;
+ switch (Mode) {
+ case Floor:
+ Inc = B.CreateAnd(HasFrac, T.Sign);
+ break;
+ case Ceil:
+ Inc = B.CreateAnd(HasFrac, B.CreateNot(T.Sign));
+ break;
+ case Trunc:
+ Inc = B.getFalse();
+ break;
+ case Even:
+ Inc = IncEven;
+ break;
+ case Away:
+ Inc = B.CreateOr(AboveHalf, IsHalf);
+ break;
+ case Nearest:
+ Inc = select(roundingIs(RoundNearestEven), IncEven,
+ select(roundingIs(RoundUp), B.CreateAnd(HasFrac, B.CreateNot(T.Sign)),
+ select(roundingIs(RoundDown), B.CreateAnd(HasFrac, T.Sign),
+ B.getFalse())));
+ break;
+ }
+ Q = B.CreateAdd(Q, B.CreateZExt(Inc, I128));
+ Value *R = round({T.Sign, Q, c32(0)}, F64);
+ // Values of at least 2^52 are integers.
+ Value *Integer = B.CreateICmpUGE(magnitude(X, F64), c64(uint64_t(1075) << 52));
+ R = select(Integer, X, R);
+ return select(isNaN(X, F64), nanResult({X}, F64, F64), R);
+ }
+
+ /// A signed integer whose order is that of the non-NaN values, with -0 and
+ /// +0 equal (or -0 below +0 with ZeroOrder).
+ Value *orderKey(Value *X, bool ZeroOrder) {
+ Value *Mag = magnitude(X, F64);
+ Value *Neg = ZeroOrder ? B.CreateNot(Mag) : B.CreateNeg(Mag);
+ return select(signOf(X, F64), Neg, Mag);
+ }
+
+ Value *compare(CmpInst::Predicate P, Value *X, Value *Y) {
+ Value *Uno = B.CreateOr(isNaN(X, F64), isNaN(Y, F64));
+ Value *KX = orderKey(X, false), *KY = orderKey(Y, false);
+ Value *R;
+ switch (P) {
+ case CmpInst::FCMP_FALSE:
+ return B.getFalse();
+ case CmpInst::FCMP_TRUE:
+ return B.getTrue();
+ case CmpInst::FCMP_ORD:
+ return B.CreateNot(Uno);
+ case CmpInst::FCMP_UNO:
+ return Uno;
+ case CmpInst::FCMP_OEQ:
+ case CmpInst::FCMP_UEQ:
+ R = B.CreateICmpEQ(KX, KY);
+ break;
+ case CmpInst::FCMP_ONE:
+ case CmpInst::FCMP_UNE:
+ R = B.CreateICmpNE(KX, KY);
+ break;
+ case CmpInst::FCMP_OGT:
+ case CmpInst::FCMP_UGT:
+ R = B.CreateICmpSGT(KX, KY);
+ break;
+ case CmpInst::FCMP_OGE:
+ case CmpInst::FCMP_UGE:
+ R = B.CreateICmpSGE(KX, KY);
+ break;
+ case CmpInst::FCMP_OLT:
+ case CmpInst::FCMP_ULT:
+ R = B.CreateICmpSLT(KX, KY);
+ break;
+ case CmpInst::FCMP_OLE:
+ case CmpInst::FCMP_ULE:
+ R = B.CreateICmpSLE(KX, KY);
+ break;
+ default:
+ llvm_unreachable("not a floating point predicate");
+ }
+ return CmpInst::isUnordered(P) ? B.CreateOr(Uno, R)
+ : B.CreateAnd(B.CreateNot(Uno), R);
+ }
+
+ /// minnum, maxnum and the IEEE 754-2019 minimum, maximum, minimumNumber
+ /// and maximumNumber.
+ Value *minMax(Value *X, Value *Y, bool Min, bool PropagateNaN, bool ZeroOrder) {
+ Value *KX = orderKey(X, ZeroOrder), *KY = orderKey(Y, ZeroOrder);
+ Value *PickX = Min ? B.CreateICmpSLE(KX, KY) : B.CreateICmpSGE(KX, KY);
+ Value *R = select(PickX, X, Y);
+ Value *XNaN = isNaN(X, F64), *YNaN = isNaN(Y, F64);
+ if (PropagateNaN)
+ return select(B.CreateOr(XNaN, YNaN), nanResult({X, Y}, F64, F64), R);
+ R = select(XNaN, Y, select(YNaN, X, R));
+ return select(B.CreateAnd(XNaN, YNaN), nanResult({X, Y}, F64, F64), R);
+ }
+
+ Value *isFPClass(Value *X, uint64_t Mask) {
+ Value *Sign = signOf(X, F64);
+ Value *Mag = magnitude(X, F64);
+ Value *R = B.getFalse();
+ auto Add = [&](unsigned Test, Value *C) {
+ if (Mask & Test)
+ R = B.CreateOr(R, C);
+ };
+ Value *SNaN = B.CreateAnd(isNaN(X, F64),
+ B.CreateICmpEQ(B.CreateAnd(X, c64(F64.quietBit())), c64(0)));
+ Value *QNaN = B.CreateAnd(isNaN(X, F64), B.CreateNot(SNaN));
+ Value *Inf = isInf(X, F64);
+ Value *Zero = isZero(X, F64);
+ Value *Sub = B.CreateAnd(B.CreateNot(Zero), B.CreateICmpULT(Mag, c64(F64.fracMask() + 1)));
+ Value *Normal = B.CreateAnd(B.CreateICmpUGE(Mag, c64(F64.fracMask() + 1)),
+ B.CreateICmpULT(Mag, c64(F64.infBits())));
+ Value *Pos = B.CreateNot(Sign);
+ Add(fcSNan, SNaN);
+ Add(fcQNan, QNaN);
+ Add(fcNegInf, B.CreateAnd(Inf, Sign));
+ Add(fcNegNormal, B.CreateAnd(Normal, Sign));
+ Add(fcNegSubnormal, B.CreateAnd(Sub, Sign));
+ Add(fcNegZero, B.CreateAnd(Zero, Sign));
+ Add(fcPosZero, B.CreateAnd(Zero, Pos));
+ Add(fcPosSubnormal, B.CreateAnd(Sub, Pos));
+ Add(fcPosNormal, B.CreateAnd(Normal, Pos));
+ Add(fcPosInf, B.CreateAnd(Inf, Pos));
+ return R;
+ }
+};
+
+} // namespace
+
+static bool isDouble(const Value *V) {
+ return V->getType()->isDoubleTy();
+}
+
+static Fmt formatOf(Type *Ty) {
+ if (Ty->isFloatTy())
+ return F32;
+ if (Ty->isBFloatTy())
+ return BF16;
+ return F16;
+}
+
+bool AGX::isDoubleArithmetic(const Instruction &I) {
+ if (I.getType()->isVectorTy())
+ return false;
+ switch (I.getOpcode()) {
+ case Instruction::FAdd:
+ case Instruction::FSub:
+ case Instruction::FMul:
+ case Instruction::FDiv:
+ case Instruction::FNeg:
+ case Instruction::FPTrunc:
+ case Instruction::SIToFP:
+ case Instruction::UIToFP:
+ return isDouble(&I) || isDouble(I.getOperand(0));
+ case Instruction::FCmp:
+ case Instruction::FPToSI:
+ case Instruction::FPToUI:
+ case Instruction::FPExt:
+ return isDouble(I.getOperand(0)) || isDouble(&I);
+ default:
+ break;
+ }
+ if (const auto *II = dyn_cast<IntrinsicInst>(&I)) {
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::fma:
+ case Intrinsic::fmuladd:
+ case Intrinsic::sqrt:
+ case Intrinsic::fabs:
+ case Intrinsic::copysign:
+ case Intrinsic::minnum:
+ case Intrinsic::maxnum:
+ case Intrinsic::minimum:
+ case Intrinsic::maximum:
+ case Intrinsic::minimumnum:
+ case Intrinsic::maximumnum:
+ case Intrinsic::floor:
+ case Intrinsic::ceil:
+ case Intrinsic::trunc:
+ case Intrinsic::rint:
+ case Intrinsic::nearbyint:
+ case Intrinsic::roundeven:
+ case Intrinsic::round:
+ case Intrinsic::canonicalize:
+ return isDouble(II);
+ case Intrinsic::is_fpclass:
+ case Intrinsic::fptosi_sat:
+ case Intrinsic::fptoui_sat:
+ return isDouble(II->getArgOperand(0));
+ default:
+ break;
+ }
+ }
+ return false;
+}
+
+Value *AGX::lowerDoubleArithmetic(Instruction &I, Value *Rounding,
+ bool CanonicalNaN) {
+ if (!isDoubleArithmetic(I))
+ return nullptr;
+ IRBuilder<> B(&I);
+ SoftDouble SD(B, Rounding, CanonicalNaN);
+ Type *I64 = B.getInt64Ty();
+ auto Bits = [&](unsigned N) {
+ Value *V = I.getOperand(N);
+ if (V->getType()->isDoubleTy())
+ return B.CreateBitCast(V, I64);
+ unsigned W = V->getType()->getPrimitiveSizeInBits();
+ return B.CreateZExt(B.CreateBitCast(V, B.getIntNTy(W)), I64);
+ };
+ auto Dbl = [&](Value *V) { return B.CreateBitCast(V, B.getDoubleTy()); };
+ Value *R = nullptr;
+ switch (I.getOpcode()) {
+ case Instruction::FAdd:
+ case Instruction::FSub:
+ R = Dbl(SD.add(Bits(0), Bits(1), I.getOpcode() == Instruction::FSub));
+ break;
+ case Instruction::FMul:
+ R = Dbl(SD.mul(Bits(0), Bits(1)));
+ break;
+ case Instruction::FDiv:
+ R = Dbl(SD.div(Bits(0), Bits(1)));
+ break;
+ case Instruction::FNeg:
+ R = Dbl(B.CreateXor(Bits(0), SD.c64(F64.signBit())));
+ break;
+ case Instruction::FPExt: {
+ Fmt In = formatOf(I.getOperand(0)->getType());
+ R = Dbl(SD.convert(Bits(0), In, F64));
+ break;
+ }
+ case Instruction::FPTrunc: {
+ Type *Ty = I.getType();
+ Fmt Out = formatOf(Ty);
+ Value *V = SD.convert(Bits(0), F64, Out);
+ R = B.CreateBitCast(B.CreateTrunc(V, B.getIntNTy(Out.W)), Ty);
+ break;
+ }
+ case Instruction::SIToFP:
+ case Instruction::UIToFP:
+ if (!isDouble(&I))
+ return nullptr;
+ R = Dbl(SD.fromInt(I.getOperand(0), I.getOpcode() == Instruction::SIToFP));
+ break;
+ case Instruction::FPToSI:
+ case Instruction::FPToUI:
+ R = SD.toInt(Bits(0), I.getType(), I.getOpcode() == Instruction::FPToSI);
+ break;
+ case Instruction::FCmp:
+ R = SD.compare(cast<FCmpInst>(I).getPredicate(), Bits(0), Bits(1));
+ break;
+ default: {
+ auto &II = cast<IntrinsicInst>(I);
+ switch (II.getIntrinsicID()) {
+ case Intrinsic::fma:
+ case Intrinsic::fmuladd:
+ R = Dbl(SD.fma(Bits(0), Bits(1), Bits(2)));
+ break;
+ case Intrinsic::sqrt:
+ R = Dbl(SD.sqrt(Bits(0)));
+ break;
+ case Intrinsic::fabs:
+ R = Dbl(SD.magnitude(Bits(0), F64));
+ break;
+ case Intrinsic::copysign:
+ R = Dbl(B.CreateOr(SD.magnitude(Bits(0), F64),
+ B.CreateAnd(Bits(1), SD.c64(F64.signBit()))));
+ break;
+ case Intrinsic::minnum:
+ case Intrinsic::maxnum:
+ R = Dbl(SD.minMax(Bits(0), Bits(1), II.getIntrinsicID() == Intrinsic::minnum,
+ false, true));
+ break;
+ case Intrinsic::minimum:
+ case Intrinsic::maximum:
+ R = Dbl(SD.minMax(Bits(0), Bits(1), II.getIntrinsicID() == Intrinsic::minimum,
+ true, true));
+ break;
+ case Intrinsic::minimumnum:
+ case Intrinsic::maximumnum:
+ R = Dbl(SD.minMax(Bits(0), Bits(1),
+ II.getIntrinsicID() == Intrinsic::minimumnum, false, true));
+ break;
+ case Intrinsic::floor:
+ R = Dbl(SD.integral(Bits(0), SoftDouble::Floor));
+ break;
+ case Intrinsic::ceil:
+ R = Dbl(SD.integral(Bits(0), SoftDouble::Ceil));
+ break;
+ case Intrinsic::trunc:
+ R = Dbl(SD.integral(Bits(0), SoftDouble::Trunc));
+ break;
+ case Intrinsic::rint:
+ case Intrinsic::nearbyint:
+ R = Dbl(SD.integral(Bits(0), SoftDouble::Nearest));
+ break;
+ case Intrinsic::roundeven:
+ R = Dbl(SD.integral(Bits(0), SoftDouble::Even));
+ break;
+ case Intrinsic::round:
+ R = Dbl(SD.integral(Bits(0), SoftDouble::Away));
+ break;
+ case Intrinsic::canonicalize: {
+ Value *X = Bits(0);
+ R = Dbl(SD.select(SD.isNaN(X, F64), SD.nanResult({X}, F64, F64), X));
+ break;
+ }
+ case Intrinsic::is_fpclass:
+ R = SD.isFPClass(Bits(0),
+ cast<ConstantInt>(II.getArgOperand(1))->getZExtValue());
+ break;
+ case Intrinsic::fptosi_sat:
+ case Intrinsic::fptoui_sat:
+ R = SD.toInt(Bits(0), I.getType(),
+ II.getIntrinsicID() == Intrinsic::fptosi_sat);
+ break;
+ default:
+ return nullptr;
+ }
+ }
+ }
+ R->takeName(&I);
+ I.replaceAllUsesWith(R);
+ I.eraseFromParent();
+ for (BinaryOperator *Div : SD.Divisions)
+ expandDivision(Div);
+ return R;
+}
+
+namespace {
+
+class AGXSoftF64 : public FunctionPass {
+public:
+ static char ID;
+ AGXSoftF64() : FunctionPass(ID) {}
+
+ StringRef getPassName() const override { return "AGX Soft F64"; }
+ bool runOnFunction(Function &F) override;
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<TargetPassConfig>();
+ }
+};
+
+} // namespace
+
+char AGXSoftF64::ID = 0;
+
+INITIALIZE_PASS_BEGIN(AGXSoftF64, DEBUG_TYPE, "AGX Soft F64", false, false)
+INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
+INITIALIZE_PASS_END(AGXSoftF64, DEBUG_TYPE, "AGX Soft F64", false, false)
+
+FunctionPass *llvm::createAGXSoftF64Pass() { return new AGXSoftF64(); }
+
+bool AGXSoftF64::runOnFunction(Function &F) {
+ if (skipFunction(F))
+ return false;
+ const TargetMachine &TM =
+ getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
+ const auto &ST = TM.getSubtarget<AGXSubtarget>(F);
+ SmallVector<Instruction *, 16> Worklist;
+ for (Instruction &I : instructions(F))
+ if (AGX::isDoubleArithmetic(I))
+ Worklist.push_back(&I);
+ bool Changed = false;
+ Value *Nearest = ConstantInt::get(Type::getInt32Ty(F.getContext()),
+ RoundNearestEven);
+ for (Instruction *I : Worklist)
+ Changed |= AGX::lowerDoubleArithmetic(*I, Nearest, ST.hasCanonicalNaN()) != nullptr;
+ return Changed;
+}
diff --git a/llvm/lib/Target/AGX/AGXSoftF64.h b/llvm/lib/Target/AGX/AGXSoftF64.h
new file mode 100644
index 0000000000000..6c9f4827a24e0
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXSoftF64.h
@@ -0,0 +1,31 @@
+//===-- AGXSoftF64.h - Double precision in software for AGX -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AGX_AGXSOFTF64_H
+#define LLVM_LIB_TARGET_AGX_AGXSOFTF64_H
+
+namespace llvm {
+class Instruction;
+class Value;
+
+namespace AGX {
+
+/// True if I computes with double precision values (beyond moving them).
+bool isDoubleArithmetic(const Instruction &I);
+
+/// Replaces I, a double precision operation, with integer arithmetic on the
+/// bits of its operands, correctly rounded in the direction Rounding (an i32
+/// value numbered like FLT_ROUNDS). Returns the replacement, or null (and
+/// leaves I) if the operation is not supported.
+Value *lowerDoubleArithmetic(Instruction &I, Value *Rounding,
+ bool CanonicalNaN);
+
+} // namespace AGX
+} // namespace llvm
+
+#endif
diff --git a/llvm/lib/Target/AGX/AGXStrictFP.cpp b/llvm/lib/Target/AGX/AGXStrictFP.cpp
index 0cd6b310dd4e0..cf2822a35017a 100644
--- a/llvm/lib/Target/AGX/AGXStrictFP.cpp
+++ b/llvm/lib/Target/AGX/AGXStrictFP.cpp
@@ -34,6 +34,7 @@
//===----------------------------------------------------------------------===//
#include "AGX.h"
+#include "AGXSoftF64.h"
#include "AGXSubtarget.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/IR/IRBuilder.h"
@@ -1355,10 +1356,19 @@ bool AGXLowerFPEnv::runOnFunction(Function &F) {
bool Nearest = isa<ConstantInt>(Rounding) &&
cast<ConstantInt>(Rounding)->getZExtValue() == RoundNearestEven;
- Value *R = Nearest ? nullptr : lowerRounded(CI, Rounding, ST.hasCanonicalNaN());
+ bool Double = ResTy->isDoubleTy() || OpTy->isDoubleTy();
+ Value *R = Nearest || Double
+ ? nullptr
+ : lowerRounded(CI, Rounding, ST.hasCanonicalNaN());
if (!R) {
B.SetInsertPoint(CI);
R = lowerToPlain(CI, B);
+ // Double precision is computed in software in any rounding direction.
+ if (Double && !Nearest)
+ if (auto *RI = dyn_cast_or_null<Instruction>(R))
+ if (Value *D = AGX::lowerDoubleArithmetic(*RI, Rounding,
+ ST.hasCanonicalNaN()))
+ R = D;
}
if (!R)
continue;
diff --git a/llvm/lib/Target/AGX/AGXTargetMachine.cpp b/llvm/lib/Target/AGX/AGXTargetMachine.cpp
index d79914d611b02..21f4b9718051d 100644
--- a/llvm/lib/Target/AGX/AGXTargetMachine.cpp
+++ b/llvm/lib/Target/AGX/AGXTargetMachine.cpp
@@ -38,6 +38,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAGXTarget() {
initializeAGXCodeGenPreparePass(PR);
initializeAGXStrictFPPass(PR);
initializeAGXLowerFPEnvPass(PR);
+ initializeAGXSoftF64Pass(PR);
initializeAGXAnnotateControlFlowPass(PR);
initializeAGXDAGToDAGISelLegacyPass(PR);
initializeAGXFoldOperandsPass(PR);
@@ -153,6 +154,7 @@ void AGXPassConfig::addIRPasses() {
// The floating point mode register and constrained floating point
// intrinsics (directed rounding in software).
addPass(createAGXLowerFPEnvPass());
+ addPass(createAGXSoftF64Pass());
addPass(createAGXCodeGenPreparePass());
// Software IEEE binary32 arithmetic and canonical NaNs, when enabled.
addPass(createAGXStrictFPPass());
diff --git a/llvm/lib/Target/AGX/CMakeLists.txt b/llvm/lib/Target/AGX/CMakeLists.txt
index 0c5515668c967..9d91d36725564 100644
--- a/llvm/lib/Target/AGX/CMakeLists.txt
+++ b/llvm/lib/Target/AGX/CMakeLists.txt
@@ -31,6 +31,7 @@ add_llvm_target(AGXCodeGen
AGXMCInstLower.cpp
AGXRegisterInfo.cpp
AGXSelectionDAGInfo.cpp
+ AGXSoftF64.cpp
AGXStrictFP.cpp
AGXSubtarget.cpp
AGXTargetMachine.cpp
diff --git a/llvm/test/CodeGen/AGX/errors-f64.ll b/llvm/test/CodeGen/AGX/errors-f64.ll
index 310e1a887ac9c..a25347043fc89 100644
--- a/llvm/test/CodeGen/AGX/errors-f64.ll
+++ b/llvm/test/CodeGen/AGX/errors-f64.ll
@@ -1,12 +1,12 @@
; RUN: not llc -mtriple=agx -mcpu=agx-g13g < %s 2>&1 | FileCheck %s
+; Double precision arithmetic is computed in software (see soft-f64.ll),
+; except for the remainder and vector operations.
; CHECK: error: {{.*}}in function doubles {{.*}}: double precision floating point arithmetic (AGX has no double precision hardware)
; CHECK-NOT: error
-define agx_kernel void @doubles(ptr addrspace(1) %out, double %a, float %b) {
- %e = fpext float %b to double
- %s = fadd double %a, %e
- %m = fmul double %s, %a
- store double %m, ptr addrspace(1) %out
+define agx_kernel void @doubles(ptr addrspace(1) %out, double %a, double %b) {
+ %r = frem double %a, %b
+ store double %r, ptr addrspace(1) %out
ret void
}
diff --git a/llvm/test/CodeGen/AGX/soft-f64.ll b/llvm/test/CodeGen/AGX/soft-f64.ll
new file mode 100644
index 0000000000000..e77c57f3ee05d
--- /dev/null
+++ b/llvm/test/CodeGen/AGX/soft-f64.ll
@@ -0,0 +1,61 @@
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -o /dev/null -print-after=agx-soft-f64 < %s 2>&1 | FileCheck %s
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -verify-machineinstrs < %s | FileCheck %s --check-prefix=ASM
+
+; Double precision arithmetic becomes integer arithmetic on the bits, with
+; 128-bit intermediate values; values that are only moved stay double.
+
+; CHECK-LABEL: @arith(
+; CHECK-NOT: {{fadd|fmul|fdiv|fpext|fptrunc|fcmp}} double
+; CHECK: mul i128
+; CHECK: call i128 @llvm.ctlz.i128
+; CHECK: bitcast i64 {{.*}} to double
+; ASM-LABEL: arith:
+; ASM-NOT: error
+define agx_kernel void @arith(ptr addrspace(1) %out, double %a, float %b) {
+ %e = fpext float %b to double
+ %s = fadd double %a, %e
+ %m = fmul double %s, %a
+ %d = fdiv double %m, %e
+ %t = fptrunc double %d to float
+ store float %t, ptr addrspace(1) %out
+ %o = getelementptr double, ptr addrspace(1) %out, i32 1
+ store double %d, ptr addrspace(1) %o
+ ret void
+}
+
+; The square root is a loop producing a bit of the root per iteration.
+; CHECK-LABEL: @root(
+; CHECK: isqrt:
+; CHECK: phi i128
+; CHECK: br i1 {{.*}}, label %isqrt.end, label %isqrt
+define agx_kernel void @root(ptr addrspace(1) %out, double %a) {
+ %r = call double @llvm.sqrt.f64(double %a)
+ store double %r, ptr addrspace(1) %out
+ ret void
+}
+
+; A directed rounding mode is applied to the software operation.
+; CHECK-LABEL: @toward_zero(
+; CHECK-NOT: call {{.*}}constrained
+; CHECK-NOT: fadd double
+define agx_kernel void @toward_zero(ptr addrspace(1) %out, double %a, double %b) #0 {
+ %r = call double @llvm.experimental.constrained.fadd.f64(double %a, double %b, metadata !"round.towardzero", metadata !"fpexcept.ignore") #0
+ store double %r, ptr addrspace(1) %out
+ ret void
+}
+
+; Comparisons are integer comparisons of an order key.
+; CHECK-LABEL: @compare(
+; CHECK-NOT: fcmp
+; CHECK: icmp slt i64
+define agx_kernel void @compare(ptr addrspace(1) %out, double %a, double %b) {
+ %c = fcmp olt double %a, %b
+ %z = zext i1 %c to i32
+ store i32 %z, ptr addrspace(1) %out
+ ret void
+}
+
+declare double @llvm.sqrt.f64(double)
+declare double @llvm.experimental.constrained.fadd.f64(double, double, metadata, metadata)
+
+attributes #0 = { strictfp }
>From cecf499691ef33327ccb402ee44db700d7355cee Mon Sep 17 00:00:00 2001
From: Claude <noreply at anthropic.com>
Date: Mon, 19 Oct 2026 01:04:35 +0000
Subject: [PATCH 15/22] [AGX] Strict mode for vectors and bfloat; fix fneg,
min/max and bf16 rounding
- Vector arithmetic is scalarized (Scalarizer) before the floating point
lowerings, and agx-lower-fp-env splits vector constrained intrinsics into
scalar calls, so strict IEEE, directed rounding and software double
precision apply lane by lane.
- With ieee-fp32, bfloat operations become float operations on extended
operands tagged agx.ieee (bfloat keeps denormals), rounded back once;
bfloat fma is computed exactly (AGX::buildSoftFMA).
- float to bfloat conversions rounded by truncation; they now round to
nearest even (custom FP_TO_BF16). bf16 extending loads and truncating
stores are expanded.
- A standalone float fneg or fabs was an addition with a source modifier,
which flushes denormals; it is now a bit operation.
- The hardware min/max return a denormal operand unchanged: their results
are flushed in flushing functions.
Checked bit by bit with ZLUDA's ptx/tools/agx_vector_bf16_check.py (all
v2f32 operations in four rounding modes, and bfloat add, sub, mul, div,
fma, sqrt, minnum, fcmp).
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
Claude-Session: https://claude.ai/code/session_01E56KEecg6edNaS23myRWER
---
llvm/docs/AGXUsage.rst | 16 +++
llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp | 38 ++++++
llvm/lib/Target/AGX/AGXISelLowering.cpp | 32 +++++-
llvm/lib/Target/AGX/AGXISelLowering.h | 1 +
llvm/lib/Target/AGX/AGXInstructions.td | 7 +-
llvm/lib/Target/AGX/AGXSoftF64.cpp | 39 ++++---
llvm/lib/Target/AGX/AGXSoftF64.h | 6 +
llvm/lib/Target/AGX/AGXStrictFP.cpp | 134 +++++++++++++++++++++-
llvm/lib/Target/AGX/AGXTargetMachine.cpp | 4 +
llvm/test/CodeGen/AGX/float-misc.ll | 84 ++++++++++++++
10 files changed, 343 insertions(+), 18 deletions(-)
create mode 100644 llvm/test/CodeGen/AGX/float-misc.ll
diff --git a/llvm/docs/AGXUsage.rst b/llvm/docs/AGXUsage.rst
index 034f6982ded76..66d0dbfb24982 100644
--- a/llvm/docs/AGXUsage.rst
+++ b/llvm/docs/AGXUsage.rst
@@ -323,6 +323,13 @@ PTX behavior in software (``AGXStrictFP.cpp``):
correctly rounded in the same way, on flushed operands and with a flushed
result.
+ ``bfloat`` arithmetic (which keeps denormals) is computed as ``float``
+ arithmetic on the extended operands with the IEEE lowering, rounded back
+ once; ``float`` has more than twice the precision, so the result is
+ correctly rounded. ``llvm.fma`` on ``bfloat`` is computed exactly with
+ integer arithmetic (see `Double precision`_), since a ``float`` fma would
+ round twice.
+
``canonical-nan``
NaN results of floating point arithmetic and conversions are ``0x7fffffff``
(``0x7fff`` for ``half`` and ``bfloat``); minimum and maximum keep the bits
@@ -363,6 +370,15 @@ multiplication, 690 for a fused multiply-add, 780 for a division (plus the
iterations of the division loop), 460 for a square root (and 64 iterations),
230 for a conversion and 20 for a comparison.
+Vector floating point arithmetic is scalarized before the floating point
+lowerings (the Scalarizer pass), and constrained vector operations are rounded
+lane by lane, so the strict modes, the rounding modes and the software double
+precision apply to vectors too. A ``fneg`` or ``fabs`` of ``float`` that is
+not a source modifier of another operation is a bit operation, and the
+results of ``float`` minimum and maximum are flushed in functions that flush
+``float`` denormals (the hardware returns a denormal operand unchanged).
+Conversions of ``float`` to ``bfloat`` round to nearest even.
+
Rounding modes
--------------
diff --git a/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp b/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp
index 0b94e0493921b..75d88f0522153 100644
--- a/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp
@@ -131,6 +131,7 @@ class AGXCodeGenPrepare : public FunctionPass {
bool lowerConstrainedFP(Function &F);
bool lowerDivergentShuffles(Function &F, const UniformityInfo &UI);
bool flushHalfDenormals(Function &F);
+ bool flushFloatMinMax(Function &F);
bool diagnoseDoubles(Function &F);
public:
@@ -458,6 +459,42 @@ static bool isFPArithmetic(const Instruction &I) {
static bool isHalf(const Type *Ty) { return Ty->getScalarType()->isHalfTy(); }
+/// The hardware minimum and maximum return a denormal operand as it is, also
+/// where float denormals are flushed: flush their results.
+bool AGXCodeGenPrepare::flushFloatMinMax(Function &F) {
+ if (F.getDenormalMode(APFloat::IEEEsingle()).Output == DenormalMode::IEEE)
+ return false;
+ SmallVector<IntrinsicInst *, 8> MinMax;
+ for (Instruction &I : instructions(F)) {
+ auto *II = dyn_cast<IntrinsicInst>(&I);
+ if (!II || !II->getType()->isFloatTy() || II->hasMetadata("agx.ieee") ||
+ II->hasMetadata("agx.exact"))
+ continue;
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::minnum:
+ case Intrinsic::maxnum:
+ case Intrinsic::minimum:
+ case Intrinsic::maximum:
+ case Intrinsic::minimumnum:
+ case Intrinsic::maximumnum:
+ MinMax.push_back(II);
+ break;
+ default:
+ break;
+ }
+ }
+ for (IntrinsicInst *II : MinMax) {
+ IRBuilder<> B(II->getNextNode());
+ SmallVector<Use *, 4> Uses;
+ for (Use &U : II->uses())
+ Uses.push_back(&U);
+ Value *R = B.CreateUnaryIntrinsic(Intrinsic::canonicalize, II);
+ for (Use *U : Uses)
+ U->set(R);
+ }
+ return !MinMax.empty();
+}
+
bool AGXCodeGenPrepare::flushHalfDenormals(Function &F) {
DenormalMode Mode = F.getDenormalMode(APFloat::IEEEhalf());
bool FlushInputs = Mode.Input != DenormalMode::IEEE &&
@@ -586,6 +623,7 @@ bool AGXCodeGenPrepare::runOnFunction(Function &F) {
F, getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo());
Changed |= lowerConstrainedFP(F);
Changed |= flushHalfDenormals(F);
+ Changed |= flushFloatMinMax(F);
Changed |= expandDivisions(F);
Changed |= expandMemIntrinsics(F, TTI);
Changed |= unifyExits(F);
diff --git a/llvm/lib/Target/AGX/AGXISelLowering.cpp b/llvm/lib/Target/AGX/AGXISelLowering.cpp
index 84d46a1c601d4..4f82fd13ea0b0 100644
--- a/llvm/lib/Target/AGX/AGXISelLowering.cpp
+++ b/llvm/lib/Target/AGX/AGXISelLowering.cpp
@@ -141,6 +141,8 @@ AGXTargetLowering::AGXTargetLowering(const TargetMachine &TM,
// bfloat16 is not a legal type: conversions are integer operations.
setOperationAction({ISD::BF16_TO_FP, ISD::FP_TO_BF16},
{MVT::i16, MVT::f16, MVT::f32}, Expand);
+ // The generic expansion truncates; round to nearest even instead.
+ setOperationAction(ISD::FP_TO_BF16, {MVT::i16, MVT::f16, MVT::f32}, Custom);
// 64-bit conversions are expanded with 32-bit conversions.
setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::SINT_TO_FP,
@@ -160,8 +162,9 @@ AGXTargetLowering::AGXTargetLowering(const TargetMachine &TM,
{MVT::i8, MVT::i16, MVT::i32}, Expand);
for (MVT MemVT : {MVT::i8, MVT::i16, MVT::i32})
setTruncStoreAction(MVT::i64, MemVT, Expand);
- setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
+ setLoadExtAction(ISD::EXTLOAD, MVT::f32, {MVT::f16, MVT::bf16}, Expand);
setTruncStoreAction(MVT::f32, MVT::f16, Expand);
+ setTruncStoreAction(MVT::f32, MVT::bf16, Expand);
setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i32, Expand);
@@ -374,6 +377,8 @@ SDValue AGXTargetLowering::LowerOperation(SDValue Op,
case ISD::CTLZ:
case ISD::CTTZ:
return lowerCTLZ_CTTZ64(Op, DAG);
+ case ISD::FP_TO_BF16:
+ return lowerFP_TO_BF16(Op, DAG);
case ISD::FDIV:
return lowerFDIV(Op, DAG);
case ISD::FREM:
@@ -778,6 +783,31 @@ SDValue AGXTargetLowering::lowerFCOPYSIGN(SDValue Op,
return DAG.getBitcast(VT, DAG.getNode(ISD::OR, DL, IntVT, M, S));
}
+/// float to bfloat bits, rounded to nearest even: (x + 0x7fff + lsb) >> 16,
+/// with NaNs quieted.
+SDValue AGXTargetLowering::lowerFP_TO_BF16(SDValue Op,
+ SelectionDAG &DAG) const {
+ SDLoc DL(Op);
+ SDValue Src = Op.getOperand(0);
+ if (Src.getValueType() != MVT::f32)
+ Src = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src);
+ SDValue X = DAG.getBitcast(MVT::i32, Src);
+ auto C = [&](uint32_t V) { return DAG.getConstant(V, DL, MVT::i32); };
+ SDValue Lsb = DAG.getNode(ISD::AND, DL, MVT::i32,
+ DAG.getNode(ISD::SRL, DL, MVT::i32, X, C(16)), C(1));
+ SDValue R = DAG.getNode(ISD::ADD, DL, MVT::i32, X,
+ DAG.getNode(ISD::ADD, DL, MVT::i32, Lsb, C(0x7fff)));
+ R = DAG.getNode(ISD::SRL, DL, MVT::i32, R, C(16));
+ SDValue Quiet = DAG.getNode(ISD::OR, DL, MVT::i32,
+ DAG.getNode(ISD::SRL, DL, MVT::i32, X, C(16)), C(0x40));
+ EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i32);
+ SDValue IsNaN = DAG.getSetCC(DL, CCVT,
+ DAG.getNode(ISD::AND, DL, MVT::i32, X, C(0x7fffffff)),
+ C(0x7f800000), ISD::SETUGT);
+ R = DAG.getSelect(DL, MVT::i32, IsNaN, Quiet, R);
+ return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), R);
+}
+
SDValue AGXTargetLowering::lowerFREM(SDValue Op, SelectionDAG &DAG) const {
// a - trunc(a / b) * b
SDLoc DL(Op);
diff --git a/llvm/lib/Target/AGX/AGXISelLowering.h b/llvm/lib/Target/AGX/AGXISelLowering.h
index 35f3f8143f23b..0957fd4720e4b 100644
--- a/llvm/lib/Target/AGX/AGXISelLowering.h
+++ b/llvm/lib/Target/AGX/AGXISelLowering.h
@@ -34,6 +34,7 @@ class AGXTargetLowering final : public TargetLowering {
SDValue lowerCTLZ_CTTZ64(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFDIV(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFREM(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerFP_TO_BF16(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerTrig(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerExpLog(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/AGX/AGXInstructions.td b/llvm/lib/Target/AGX/AGXInstructions.td
index d5ad46a41e9a0..4da8316ad2463 100644
--- a/llvm/lib/Target/AGX/AGXInstructions.td
+++ b/llvm/lib/Target/AGX/AGXInstructions.td
@@ -616,10 +616,13 @@ def : Pat<(f16 (fadd (AGXFPSrc16 f16:$a, i32:$am), (AGXFPSrc16 f16:$b, i32:$bm))
def : Pat<(f16 (fmul (AGXFPSrc16 f16:$a, i32:$am), (AGXFPSrc16 f16:$b, i32:$bm))),
(FMUL16_F16 $am, $a, $bm, $b)>;
+// A negation or absolute value of its own is a bit operation: an addition
+// with a source modifier would flush a denormal operand. (Those that feed a
+// floating point operation are its source modifiers.)
def : Pat<(f32 (fneg f32:$a)),
- (FADD32_F32 MOD_NEG, $a, 0, (i32 FPIMM_NEG_ZERO))>;
+ (BITOP_I32 TT_XOR, 0, $a, 0, (MOV_IMM32 0x80000000))>;
def : Pat<(f32 (fabs f32:$a)),
- (FADD32_F32 MOD_ABS, $a, 0, (i32 FPIMM_NEG_ZERO))>;
+ (BITOP_I32 TT_AND, 0, $a, 0, (MOV_IMM32 0x7fffffff))>;
def : Pat<(f16 (fneg f16:$a)),
(FADD16_F16 MOD_NEG, $a, 0, (i16 FPIMM_NEG_ZERO))>;
def : Pat<(f16 (fabs f16:$a)),
diff --git a/llvm/lib/Target/AGX/AGXSoftF64.cpp b/llvm/lib/Target/AGX/AGXSoftF64.cpp
index f2d7342aa6b43..d11aca2ea654e 100644
--- a/llvm/lib/Target/AGX/AGXSoftF64.cpp
+++ b/llvm/lib/Target/AGX/AGXSoftF64.cpp
@@ -244,9 +244,9 @@ class SoftDouble {
return select(BothZero, select(Down, B.CreateOr(SX, SY), B.CreateAnd(SX, SY)), Down);
}
- Value *withZeroSign(Term T, Value *ZeroSign) {
+ Value *withZeroSign(Term T, Value *ZeroSign, Fmt F = F64) {
// round() gives the sign of T to a zero.
- return round({select(B.CreateICmpEQ(T.M, c128(0)), ZeroSign, T.Sign), T.M, T.E}, F64);
+ return round({select(B.CreateICmpEQ(T.M, c128(0)), ZeroSign, T.Sign), T.M, T.E}, F);
}
Value *add(Value *X, Value *Y, bool Sub) {
@@ -279,26 +279,26 @@ class SoftDouble {
return select(NaN, nanResult({X, Y}, F64, F64), R);
}
- Value *fma(Value *X, Value *Y, Value *Z) {
- Term TX = unpack(X, F64), TY = unpack(Y, F64), TZ = unpack(Z, F64);
+ Value *fma(Value *X, Value *Y, Value *Z, Fmt F = F64) {
+ Term TX = unpack(X, F), TY = unpack(Y, F), TZ = unpack(Z, F);
Term P{B.CreateXor(TX.Sign, TY.Sign), B.CreateMul(TX.M, TY.M),
B.CreateAdd(TX.E, TY.E)};
Term S = addTerms(P, TZ);
- Value *PZero = B.CreateOr(isZero(X, F64), isZero(Y, F64));
+ Value *PZero = B.CreateOr(isZero(X, F), isZero(Y, F));
Value *R = withZeroSign(
- S, zeroSumSign(B.CreateAnd(PZero, isZero(Z, F64)), P.Sign, TZ.Sign));
- Value *XInf = isInf(X, F64), *YInf = isInf(Y, F64), *ZInf = isInf(Z, F64);
+ S, zeroSumSign(B.CreateAnd(PZero, isZero(Z, F)), P.Sign, TZ.Sign), F);
+ Value *XInf = isInf(X, F), *YInf = isInf(Y, F), *ZInf = isInf(Z, F);
Value *PInf = B.CreateOr(XInf, YInf);
- Value *PInfBits = B.CreateOr(c64(F64.infBits()),
- select(P.Sign, c64(F64.signBit()), c64(0)));
- Value *NaN = B.CreateOr(B.CreateOr(isNaN(X, F64), isNaN(Y, F64)), isNaN(Z, F64));
- NaN = B.CreateOr(NaN, B.CreateOr(B.CreateAnd(XInf, isZero(Y, F64)),
- B.CreateAnd(YInf, isZero(X, F64))));
+ Value *PInfBits = B.CreateOr(c64(F.infBits()),
+ select(P.Sign, c64(F.signBit()), c64(0)));
+ Value *NaN = B.CreateOr(B.CreateOr(isNaN(X, F), isNaN(Y, F)), isNaN(Z, F));
+ NaN = B.CreateOr(NaN, B.CreateOr(B.CreateAnd(XInf, isZero(Y, F)),
+ B.CreateAnd(YInf, isZero(X, F))));
NaN = B.CreateOr(NaN, B.CreateAnd(B.CreateAnd(PInf, ZInf),
B.CreateICmpNE(P.Sign, TZ.Sign)));
R = select(ZInf, Z, R);
R = select(PInf, PInfBits, R);
- return select(NaN, nanResult({X, Y, Z}, F64, F64), R);
+ return select(NaN, nanResult({X, Y, Z}, F, F), R);
}
Value *div(Value *X, Value *Y) {
@@ -633,6 +633,19 @@ bool AGX::isDoubleArithmetic(const Instruction &I) {
return false;
}
+Value *AGX::buildSoftFMA(IRBuilderBase &B, Value *X, Value *Y, Value *Z,
+ Value *Rounding, bool CanonicalNaN) {
+ Type *Ty = X->getType();
+ Fmt F = Ty->isDoubleTy() ? F64 : formatOf(Ty);
+ Type *IntTy = B.getIntNTy(F.W);
+ auto Bits = [&](Value *V) {
+ return B.CreateZExt(B.CreateBitCast(V, IntTy), B.getInt64Ty());
+ };
+ SoftDouble SD(B, Rounding, CanonicalNaN);
+ Value *R = SD.fma(Bits(X), Bits(Y), Bits(Z), F);
+ return B.CreateBitCast(B.CreateTrunc(R, IntTy), Ty);
+}
+
Value *AGX::lowerDoubleArithmetic(Instruction &I, Value *Rounding,
bool CanonicalNaN) {
if (!isDoubleArithmetic(I))
diff --git a/llvm/lib/Target/AGX/AGXSoftF64.h b/llvm/lib/Target/AGX/AGXSoftF64.h
index 6c9f4827a24e0..3ba68106577bf 100644
--- a/llvm/lib/Target/AGX/AGXSoftF64.h
+++ b/llvm/lib/Target/AGX/AGXSoftF64.h
@@ -10,6 +10,7 @@
#define LLVM_LIB_TARGET_AGX_AGXSOFTF64_H
namespace llvm {
+class IRBuilderBase;
class Instruction;
class Value;
@@ -25,6 +26,11 @@ bool isDoubleArithmetic(const Instruction &I);
Value *lowerDoubleArithmetic(Instruction &I, Value *Rounding,
bool CanonicalNaN);
+/// The fused multiply-add of X, Y and Z (half, bfloat, float or double),
+/// computed exactly with integer arithmetic and rounded once.
+Value *buildSoftFMA(IRBuilderBase &B, Value *X, Value *Y, Value *Z,
+ Value *Rounding, bool CanonicalNaN);
+
} // namespace AGX
} // namespace llvm
diff --git a/llvm/lib/Target/AGX/AGXStrictFP.cpp b/llvm/lib/Target/AGX/AGXStrictFP.cpp
index cf2822a35017a..f41f3b1315915 100644
--- a/llvm/lib/Target/AGX/AGXStrictFP.cpp
+++ b/llvm/lib/Target/AGX/AGXStrictFP.cpp
@@ -1038,6 +1038,79 @@ static bool lowerFlushing(Instruction &I) {
return true;
}
+/// bfloat arithmetic as float arithmetic on the extended operands, tagged
+/// for the IEEE lowering (bfloat operations keep denormals, which the float
+/// hardware flushes), with the result rounded back. Float has more than
+/// twice the precision of bfloat, so rounding twice gives the correctly
+/// rounded result. Returns false if I is not a bfloat operation.
+static bool promoteBFloat(Instruction &I, bool CanonicalNaN) {
+ bool IsCmp = isa<FCmpInst>(I);
+ Type *Ty = IsCmp ? I.getOperand(0)->getType() : I.getType();
+ if (!Ty->isBFloatTy())
+ return false;
+ Intrinsic::ID ID = getIntrinsicID(I);
+ switch (ID) {
+ case Intrinsic::not_intrinsic:
+ if (!isa<BinaryOperator>(I) && !IsCmp)
+ return false;
+ if (isa<BinaryOperator>(I) && I.getOpcode() == Instruction::FRem)
+ return false;
+ break;
+ case Intrinsic::fma:
+ case Intrinsic::fmuladd:
+ case Intrinsic::sqrt:
+ case Intrinsic::minnum:
+ case Intrinsic::maxnum:
+ case Intrinsic::minimum:
+ case Intrinsic::maximum:
+ case Intrinsic::minimumnum:
+ case Intrinsic::maximumnum:
+ case Intrinsic::floor:
+ case Intrinsic::ceil:
+ case Intrinsic::trunc:
+ case Intrinsic::rint:
+ case Intrinsic::nearbyint:
+ case Intrinsic::roundeven:
+ case Intrinsic::round:
+ case Intrinsic::canonicalize:
+ break;
+ default:
+ return false;
+ }
+ IRBuilder<> B(&I);
+ if (ID == Intrinsic::fma || ID == Intrinsic::fmuladd) {
+ // A float fma would round twice: compute it exactly.
+ Value *R = AGX::buildSoftFMA(B, I.getOperand(0), I.getOperand(1),
+ I.getOperand(2), B.getInt32(RoundNearestEven),
+ CanonicalNaN);
+ R->takeName(&I);
+ I.replaceAllUsesWith(R);
+ I.eraseFromParent();
+ return true;
+ }
+ Type *F32 = B.getFloatTy();
+ SmallVector<Value *, 3> Ops;
+ unsigned N = isa<CallInst>(I) ? cast<CallInst>(I).arg_size() : I.getNumOperands();
+ for (unsigned K = 0; K < N; ++K)
+ Ops.push_back(B.CreateFPExt(I.getOperand(K), F32));
+ Value *New;
+ if (IsCmp)
+ New = B.CreateFCmp(cast<FCmpInst>(I).getPredicate(), Ops[0], Ops[1]);
+ else if (auto *BO = dyn_cast<BinaryOperator>(&I))
+ New = B.CreateBinOp(BO->getOpcode(), Ops[0], Ops[1]);
+ else
+ New = B.CreateIntrinsic(ID, {F32}, Ops);
+ auto *NI = cast<Instruction>(New);
+ if (isa<FPMathOperator>(I))
+ NI->copyFastMathFlags(&I);
+ NI->setMetadata(IEEETag, MDNode::get(I.getContext(), {}));
+ Value *R = IsCmp ? New : B.CreateFPTrunc(New, I.getType());
+ R->takeName(&I);
+ I.replaceAllUsesWith(R);
+ I.eraseFromParent();
+ return true;
+}
+
bool AGXStrictFP::runOnFunction(Function &F) {
if (skipFunction(F))
return false;
@@ -1050,12 +1123,20 @@ bool AGXStrictFP::runOnFunction(Function &F) {
if (!ST.hasIEEEFP32() && !CanonicalNaN)
return false;
+ bool Changed = false;
+ if (ST.hasIEEEFP32()) {
+ SmallVector<Instruction *, 8> BFloat;
+ for (Instruction &I : instructions(F))
+ BFloat.push_back(&I);
+ for (Instruction *I : BFloat)
+ Changed |= promoteBFloat(*I, CanonicalNaN);
+ }
+
SmallVector<Instruction *, 32> Worklist;
for (Instruction &I : instructions(F))
if (!I.hasMetadata(ExactTag))
Worklist.push_back(&I);
- bool Changed = false;
// NaN canonicalization first: the IEEE lowering then replaces the
// canonicalized value.
if (CanonicalNaN)
@@ -1268,6 +1349,42 @@ static Value *lowerRounded(ConstrainedFPIntrinsic *CI, Value *Rounding,
return R;
}
+/// A vector constrained intrinsic as one scalar call per lane: the scalar
+/// calls, or none if CI is not a vector operation.
+static SmallVector<ConstrainedFPIntrinsic *, 4>
+scalarizeConstrained(ConstrainedFPIntrinsic *CI) {
+ SmallVector<ConstrainedFPIntrinsic *, 4> Lanes;
+ auto *VT = dyn_cast<FixedVectorType>(CI->getType());
+ if (!VT && CI->arg_size())
+ VT = dyn_cast<FixedVectorType>(CI->getArgOperand(0)->getType());
+ if (!VT)
+ return Lanes;
+ IRBuilder<> B(CI);
+ Type *ResTy = CI->getType()->getScalarType();
+ SmallVector<Type *, 4> ArgTys;
+ for (Value *A : CI->args())
+ ArgTys.push_back(A->getType()->getScalarType());
+ SmallVector<Type *, 2> OverloadTys;
+ FunctionType *FT = FunctionType::get(ResTy, ArgTys, false);
+ if (!Intrinsic::isSignatureValid(CI->getIntrinsicID(), FT, OverloadTys))
+ return Lanes;
+ Function *Decl = Intrinsic::getOrInsertDeclaration(
+ CI->getModule(), CI->getIntrinsicID(), OverloadTys);
+ Value *R = PoisonValue::get(CI->getType());
+ for (unsigned L = 0, E = VT->getNumElements(); L < E; ++L) {
+ SmallVector<Value *, 4> Args;
+ for (Value *A : CI->args())
+ Args.push_back(A->getType()->isVectorTy() ? B.CreateExtractElement(A, L) : A);
+ auto *Lane = cast<ConstrainedFPIntrinsic>(B.CreateCall(Decl, Args));
+ Lane->setAttributes(CI->getAttributes());
+ Lanes.push_back(Lane);
+ R = CI->getType()->isVectorTy() ? B.CreateInsertElement(R, Lane, L) : Lane;
+ }
+ CI->replaceAllUsesWith(R);
+ CI->eraseFromParent();
+ return Lanes;
+}
+
bool AGXLowerFPEnv::runOnFunction(Function &F) {
if (skipFunction(F))
return false;
@@ -1277,14 +1394,27 @@ bool AGXLowerFPEnv::runOnFunction(Function &F) {
SmallVector<IntrinsicInst *, 8> ModeOps;
SmallVector<ConstrainedFPIntrinsic *, 16> Constrained;
+ SmallVector<ConstrainedFPIntrinsic *, 4> Vector;
for (Instruction &I : instructions(F)) {
if (auto *CI = dyn_cast<ConstrainedFPIntrinsic>(&I))
- Constrained.push_back(CI);
+ if (CI->getType()->isVectorTy() ||
+ (CI->arg_size() && CI->getArgOperand(0)->getType()->isVectorTy()))
+ Vector.push_back(CI);
+ else
+ Constrained.push_back(CI);
else if (auto *II = dyn_cast<IntrinsicInst>(&I);
II && (II->getIntrinsicID() == Intrinsic::agx_set_fpmode ||
II->getIntrinsicID() == Intrinsic::agx_get_fpmode))
ModeOps.push_back(II);
}
+ // Vector operations are rounded lane by lane.
+ for (ConstrainedFPIntrinsic *CI : Vector) {
+ SmallVector<ConstrainedFPIntrinsic *, 4> Lanes = scalarizeConstrained(CI);
+ if (Lanes.empty())
+ Constrained.push_back(CI);
+ else
+ Constrained.append(Lanes.begin(), Lanes.end());
+ }
if (ModeOps.empty() && Constrained.empty())
return false;
diff --git a/llvm/lib/Target/AGX/AGXTargetMachine.cpp b/llvm/lib/Target/AGX/AGXTargetMachine.cpp
index 21f4b9718051d..eabe994a180d2 100644
--- a/llvm/lib/Target/AGX/AGXTargetMachine.cpp
+++ b/llvm/lib/Target/AGX/AGXTargetMachine.cpp
@@ -27,6 +27,7 @@
#include "llvm/Support/Compiler.h"
#include "llvm/Transforms/IPO/AlwaysInliner.h"
#include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Scalar/Scalarizer.h"
#include "llvm/Transforms/Utils.h"
using namespace llvm;
@@ -151,6 +152,9 @@ void AGXPassConfig::addIRPasses() {
addPass(createInferAddressSpacesPass());
addPass(createSROAPass());
}
+ // Vector arithmetic is scalar code (as in instruction selection), which the
+ // software floating point lowerings below then see.
+ addPass(createScalarizerPass());
// The floating point mode register and constrained floating point
// intrinsics (directed rounding in software).
addPass(createAGXLowerFPEnvPass());
diff --git a/llvm/test/CodeGen/AGX/float-misc.ll b/llvm/test/CodeGen/AGX/float-misc.ll
new file mode 100644
index 0000000000000..09376a81ada83
--- /dev/null
+++ b/llvm/test/CodeGen/AGX/float-misc.ll
@@ -0,0 +1,84 @@
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -mattr=+ieee-fp32 -o /dev/null -print-after=agx-strict-fp < %s 2>&1 | FileCheck %s --check-prefix=STRICT
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -o /dev/null -print-after=agx-lower-fp-env < %s 2>&1 | FileCheck %s --check-prefix=ENV
+
+; A negation or absolute value of its own is a bit operation, which keeps
+; denormals.
+; CHECK-LABEL: neg:
+; CHECK: xor
+; CHECK-NOT: fadd
+define agx_kernel void @neg(ptr addrspace(1) %o, float %a) #0 {
+ %n = fneg float %a
+ store float %n, ptr addrspace(1) %o
+ ret void
+}
+
+; CHECK-LABEL: abs:
+; CHECK: and
+; CHECK-NOT: fadd
+define agx_kernel void @abs(ptr addrspace(1) %o, float %a) #0 {
+ %n = call float @llvm.fabs.f32(float %a)
+ store float %n, ptr addrspace(1) %o
+ ret void
+}
+
+; The hardware minimum keeps a denormal operand: its result is flushed in a
+; function that flushes float denormals.
+; CHECK-LABEL: min_ftz:
+; CHECK: fcmpsel
+; CHECK-NEXT: fmul32 {{.*}}, 1.0
+define agx_kernel void @min_ftz(ptr addrspace(1) %o, float %a, float %b) #0 {
+ %m = call float @llvm.minnum.f32(float %a, float %b)
+ store float %m, ptr addrspace(1) %o
+ ret void
+}
+
+; float to bfloat rounds to nearest even.
+; CHECK-LABEL: to_bf16:
+; CHECK: iadd
+; CHECK: 32767
+define agx_kernel void @to_bf16(ptr addrspace(1) %o, float %a) {
+ %b = fptrunc float %a to bfloat
+ store bfloat %b, ptr addrspace(1) %o
+ ret void
+}
+
+; With ieee-fp32, bfloat arithmetic is float arithmetic on the extended
+; operands, tagged for the IEEE lowering, rounded back once; fma is computed
+; exactly with integer arithmetic.
+; STRICT-LABEL: @bf16_add(
+; STRICT: fpext bfloat %a to float
+; STRICT: fptrunc float {{.*}} to bfloat
+; STRICT-NOT: fadd bfloat
+define agx_kernel void @bf16_add(ptr addrspace(1) %o, bfloat %a, bfloat %b) {
+ %s = fadd bfloat %a, %b
+ store bfloat %s, ptr addrspace(1) %o
+ ret void
+}
+
+; STRICT-LABEL: @bf16_fma(
+; STRICT-NOT: @llvm.fma
+; STRICT: mul i128
+define agx_kernel void @bf16_fma(ptr addrspace(1) %o, bfloat %a, bfloat %b, bfloat %c) {
+ %s = call bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
+ store bfloat %s, ptr addrspace(1) %o
+ ret void
+}
+
+; Vector constrained operations are rounded lane by lane.
+; ENV-LABEL: @vector_rz(
+; ENV-NOT: call {{.*}}constrained
+; ENV: insertelement <2 x float>
+define agx_kernel void @vector_rz(ptr addrspace(1) %o, <2 x float> %a, <2 x float> %b) #1 {
+ %s = call <2 x float> @llvm.experimental.constrained.fadd.v2f32(<2 x float> %a, <2 x float> %b, metadata !"round.towardzero", metadata !"fpexcept.ignore") #1
+ store <2 x float> %s, ptr addrspace(1) %o
+ ret void
+}
+
+declare float @llvm.fabs.f32(float)
+declare float @llvm.minnum.f32(float, float)
+declare bfloat @llvm.fma.bf16(bfloat, bfloat, bfloat)
+declare <2 x float> @llvm.experimental.constrained.fadd.v2f32(<2 x float>, <2 x float>, metadata, metadata)
+
+attributes #0 = { denormal_fpenv(preservesign) }
+attributes #1 = { strictfp }
>From 20f01fec3e62ac349f4dc027b46568bfd9247f71 Mon Sep 17 00:00:00 2001
From: Claude <noreply at anthropic.com>
Date: Mon, 19 Oct 2026 01:09:42 +0000
Subject: [PATCH 16/22] [AGX] Emulate 64- and 128-bit atomics with locks
The hardware atomics are 32-bit, so wider atomics were rejected. The new
agx-expand-wide-atomics pass (before AtomicExpand) gives 64- and 128-bit
atomic loads, stores, read-modify-write operations and compare and swaps a
lock from a 1024-entry table in device memory, chosen by the 16-byte block
of the address, and accesses the value with 32-bit atomics between fences
while it holds it. The critical section is inside the loop that takes the
lock, so threads of one SIMD-group waiting for the same lock make
progress. Floating point read-modify-write operations go through the usual
lowerings (software double precision for double).
Checked on the functional simulator: 256 threads (8 SIMD-groups) doing a
64-bit atomic add, a 64-bit atomic max and a 128-bit compare and swap
increment on shared locations give the exact totals and leave all locks
free.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
Claude-Session: https://claude.ai/code/session_01E56KEecg6edNaS23myRWER
---
llvm/docs/AGXUsage.rst | 8 +
llvm/lib/Target/AGX/AGX.h | 2 +
llvm/lib/Target/AGX/AGXExpandWideAtomics.cpp | 234 +++++++++++++++++++
llvm/lib/Target/AGX/AGXTargetMachine.cpp | 3 +
llvm/lib/Target/AGX/CMakeLists.txt | 1 +
llvm/test/CodeGen/AGX/wide-atomics.ll | 58 +++++
6 files changed, 306 insertions(+)
create mode 100644 llvm/lib/Target/AGX/AGXExpandWideAtomics.cpp
create mode 100644 llvm/test/CodeGen/AGX/wide-atomics.ll
diff --git a/llvm/docs/AGXUsage.rst b/llvm/docs/AGXUsage.rst
index 66d0dbfb24982..de26c942de531 100644
--- a/llvm/docs/AGXUsage.rst
+++ b/llvm/docs/AGXUsage.rst
@@ -241,6 +241,14 @@ Supported:
intrinsics (inline or as loops).
* Device and threadgroup 32-bit atomics. Other atomic operations become
compare-and-swap loops; atomic loads and stores become atomic operations.
+ The hardware has no 64-bit or 128-bit atomics (beyond 64-bit ``umin`` and
+ ``umax`` without a result): those take a lock from a table of 1024 device
+ memory locks (the ``__agx_atomic_locks`` global, in the global data
+ buffer), chosen by the address of the 16-byte block they access, and read
+ and write the value with 32-bit atomics between fences while they hold it.
+ The critical section is in the loop that takes the lock, so the threads of
+ a SIMD-group that wait for the same lock do not deadlock. They are atomic
+ with respect to each other, not to 32-bit atomics on the same memory.
* SIMD-group and quad shuffles, ballots, reductions and prefixes, derivatives,
barriers and special registers through the ``llvm.agx.*`` intrinsics.
diff --git a/llvm/lib/Target/AGX/AGX.h b/llvm/lib/Target/AGX/AGX.h
index 44f7b61b84903..41b9aecc46f5b 100644
--- a/llvm/lib/Target/AGX/AGX.h
+++ b/llvm/lib/Target/AGX/AGX.h
@@ -28,6 +28,7 @@ FunctionPass *createAGXCodeGenPreparePass();
FunctionPass *createAGXStrictFPPass();
FunctionPass *createAGXLowerFPEnvPass();
FunctionPass *createAGXSoftF64Pass();
+FunctionPass *createAGXExpandWideAtomicsPass();
FunctionPass *createAGXAnnotateControlFlowPass();
// Instruction selection.
@@ -43,6 +44,7 @@ void initializeAGXCodeGenPreparePass(PassRegistry &);
void initializeAGXStrictFPPass(PassRegistry &);
void initializeAGXLowerFPEnvPass(PassRegistry &);
void initializeAGXSoftF64Pass(PassRegistry &);
+void initializeAGXExpandWideAtomicsPass(PassRegistry &);
void initializeAGXAnnotateControlFlowPass(PassRegistry &);
void initializeAGXDAGToDAGISelLegacyPass(PassRegistry &);
void initializeAGXFoldOperandsPass(PassRegistry &);
diff --git a/llvm/lib/Target/AGX/AGXExpandWideAtomics.cpp b/llvm/lib/Target/AGX/AGXExpandWideAtomics.cpp
new file mode 100644
index 0000000000000..73079aafb11f0
--- /dev/null
+++ b/llvm/lib/Target/AGX/AGXExpandWideAtomics.cpp
@@ -0,0 +1,234 @@
+//===-- AGXExpandWideAtomics.cpp - 64- and 128-bit atomics for AGX --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The hardware atomics are 32-bit. Atomic operations on wider values (64-bit
+// and 128-bit loads, stores, read-modify-write operations and compare and
+// swaps) take a lock from a table of device memory locks, chosen by the
+// address of the 16-byte block they access, and access the value with 32-bit
+// atomics, between fences, while they hold it:
+//
+// loop: ; per thread, until it has done its work
+// if (cmpxchg(lock, 0, 1) succeeds) {
+// fence; read; compute; write; fence
+// xchg(lock, 0)
+// done
+// }
+//
+// The critical section is inside the loop, so a thread that holds a lock
+// releases it in the same iteration, before the other threads of its
+// SIMD-group, which may wait for the same lock, try again.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AGX.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstIterator.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Module.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Transforms/Utils/LowerAtomic.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "agx-expand-wide-atomics"
+
+static constexpr unsigned NumLocks = 1024;
+
+namespace {
+
+class AGXExpandWideAtomics : public FunctionPass {
+public:
+ static char ID;
+ AGXExpandWideAtomics() : FunctionPass(ID) {}
+
+ StringRef getPassName() const override { return "AGX Expand Wide Atomics"; }
+ bool runOnFunction(Function &F) override;
+};
+
+} // namespace
+
+char AGXExpandWideAtomics::ID = 0;
+
+INITIALIZE_PASS(AGXExpandWideAtomics, DEBUG_TYPE, "AGX Expand Wide Atomics",
+ false, false)
+
+FunctionPass *llvm::createAGXExpandWideAtomicsPass() {
+ return new AGXExpandWideAtomics();
+}
+
+static GlobalVariable *getLocks(Module &M) {
+ const char *Name = "__agx_atomic_locks";
+ if (GlobalVariable *G = M.getNamedGlobal(Name))
+ return G;
+ auto *Ty = ArrayType::get(Type::getInt32Ty(M.getContext()), NumLocks);
+ return new GlobalVariable(M, Ty, false, GlobalValue::LinkOnceODRLinkage,
+ Constant::getNullValue(Ty), Name, nullptr,
+ GlobalValue::NotThreadLocal, /*AddressSpace=*/1);
+}
+
+/// The size in bits of the value of a wide atomic instruction, or 0.
+static unsigned wideAtomicSize(const Instruction &I, const DataLayout &DL) {
+ Type *Ty = nullptr;
+ if (auto *LI = dyn_cast<LoadInst>(&I); LI && LI->isAtomic())
+ Ty = LI->getType();
+ else if (auto *SI = dyn_cast<StoreInst>(&I); SI && SI->isAtomic())
+ Ty = SI->getValueOperand()->getType();
+ else if (auto *RMW = dyn_cast<AtomicRMWInst>(&I))
+ Ty = RMW->getType();
+ else if (auto *CX = dyn_cast<AtomicCmpXchgInst>(&I))
+ Ty = CX->getCompareOperand()->getType();
+ if (!Ty)
+ return 0;
+ unsigned Bits = DL.getTypeSizeInBits(Ty);
+ return Bits == 64 || Bits == 128 ? Bits : 0;
+}
+
+static Value *getPointer(Instruction &I) {
+ if (auto *LI = dyn_cast<LoadInst>(&I))
+ return LI->getPointerOperand();
+ if (auto *SI = dyn_cast<StoreInst>(&I))
+ return SI->getPointerOperand();
+ if (auto *RMW = dyn_cast<AtomicRMWInst>(&I))
+ return RMW->getPointerOperand();
+ return cast<AtomicCmpXchgInst>(I).getPointerOperand();
+}
+
+/// V (of any type of Bits bits) as an integer, and back.
+static Value *toInt(IRBuilderBase &B, Value *V, unsigned Bits) {
+ Type *IntTy = B.getIntNTy(Bits);
+ if (V->getType()->isPointerTy())
+ return B.CreatePtrToInt(V, IntTy);
+ return B.CreateBitCast(V, IntTy);
+}
+
+static Value *fromInt(IRBuilderBase &B, Value *V, Type *Ty) {
+ if (Ty->isPointerTy())
+ return B.CreateIntToPtr(V, Ty);
+ return B.CreateBitCast(V, Ty);
+}
+
+static void expand(Instruction &I, unsigned Bits, GlobalVariable *Locks) {
+ const DataLayout &DL = I.getDataLayout();
+ LLVMContext &Ctx = I.getContext();
+ Value *Ptr = getPointer(I);
+ unsigned Words = Bits / 32;
+ Type *I32 = Type::getInt32Ty(Ctx);
+ Type *ValTy = isa<StoreInst>(I) ? cast<StoreInst>(I).getValueOperand()->getType()
+ : isa<AtomicCmpXchgInst>(I)
+ ? cast<AtomicCmpXchgInst>(I).getCompareOperand()->getType()
+ : I.getType();
+
+ BasicBlock *Pre = I.getParent();
+ BasicBlock *Exit = Pre->splitBasicBlock(I.getIterator(), "wide.atomic.end");
+ Function *F = Pre->getParent();
+ BasicBlock *Head = BasicBlock::Create(Ctx, "wide.atomic", F, Exit);
+ BasicBlock *Crit = BasicBlock::Create(Ctx, "wide.atomic.locked", F, Exit);
+ BasicBlock *Latch = BasicBlock::Create(Ctx, "wide.atomic.latch", F, Exit);
+ Pre->getTerminator()->setSuccessor(0, Head);
+
+ IRBuilder<> B(Pre->getTerminator());
+ unsigned PtrBits = DL.getPointerSizeInBits(Ptr->getType()->getPointerAddressSpace());
+ Value *Addr = B.CreatePtrToInt(Ptr, B.getIntNTy(PtrBits));
+ Value *Slot = B.CreateAnd(B.CreateLShr(Addr, 4), NumLocks - 1);
+ Value *Lock = B.CreateInBoundsGEP(Locks->getValueType(), Locks,
+ {B.getInt64(0), B.CreateZExtOrTrunc(Slot, B.getInt64Ty())});
+
+ B.SetInsertPoint(Head);
+ Value *Acq = B.CreateAtomicCmpXchg(Lock, B.getInt32(0), B.getInt32(1), MaybeAlign(4),
+ AtomicOrdering::Monotonic,
+ AtomicOrdering::Monotonic);
+ B.CreateCondBr(B.CreateExtractValue(Acq, 1), Crit, Latch);
+
+ B.SetInsertPoint(Crit);
+ B.CreateFence(AtomicOrdering::SequentiallyConsistent);
+ auto Word = [&](unsigned K) {
+ return B.CreateConstGEP1_32(B.getInt8Ty(), Ptr, 4 * K);
+ };
+ Value *Loaded = ConstantInt::get(B.getIntNTy(Bits), 0);
+ for (unsigned K = 0; K < Words; ++K) {
+ Value *W = B.CreateAtomicRMW(AtomicRMWInst::Or, Word(K), B.getInt32(0),
+ MaybeAlign(4), AtomicOrdering::Monotonic);
+ W = B.CreateShl(B.CreateZExt(W, B.getIntNTy(Bits)), 32 * K);
+ Loaded = B.CreateOr(Loaded, W);
+ }
+ Value *Old = fromInt(B, Loaded, ValTy);
+ Value *Result = Old;
+ Value *New = nullptr; // the value to store
+ Value *DoStore = nullptr; // whether to store it
+ if (auto *SI = dyn_cast<StoreInst>(&I)) {
+ New = SI->getValueOperand();
+ } else if (auto *RMW = dyn_cast<AtomicRMWInst>(&I)) {
+ New = buildAtomicRMWValue(RMW->getOperation(), B, Old, RMW->getValOperand());
+ } else if (auto *CX = dyn_cast<AtomicCmpXchgInst>(&I)) {
+ Value *Eq = B.CreateICmpEQ(Loaded, toInt(B, CX->getCompareOperand(), Bits));
+ New = CX->getNewValOperand();
+ DoStore = Eq;
+ Value *Pair = B.CreateInsertValue(PoisonValue::get(CX->getType()), Old, 0);
+ Result = B.CreateInsertValue(Pair, Eq, 1);
+ }
+ if (New) {
+ Value *NewInt = toInt(B, New, Bits);
+ BasicBlock *StoreBB = nullptr, *After = nullptr;
+ if (DoStore) {
+ StoreBB = BasicBlock::Create(Ctx, "wide.atomic.store", F, Latch);
+ After = BasicBlock::Create(Ctx, "wide.atomic.unlock", F, Latch);
+ B.CreateCondBr(DoStore, StoreBB, After);
+ B.SetInsertPoint(StoreBB);
+ }
+ for (unsigned K = 0; K < Words; ++K) {
+ Value *W = B.CreateTrunc(B.CreateLShr(NewInt, 32 * K), I32);
+ B.CreateAtomicRMW(AtomicRMWInst::Xchg, Word(K), W, MaybeAlign(4),
+ AtomicOrdering::Monotonic);
+ }
+ if (DoStore) {
+ B.CreateBr(After);
+ B.SetInsertPoint(After);
+ }
+ }
+ B.CreateFence(AtomicOrdering::SequentiallyConsistent);
+ B.CreateAtomicRMW(AtomicRMWInst::Xchg, Lock, B.getInt32(0), MaybeAlign(4),
+ AtomicOrdering::Monotonic);
+ BasicBlock *CritEnd = B.GetInsertBlock();
+ B.CreateBr(Latch);
+
+ Type *ResTy = isa<AtomicCmpXchgInst>(I) ? I.getType() : ValTy;
+ B.SetInsertPoint(Latch);
+ PHINode *Got = B.CreatePHI(B.getInt1Ty(), 2);
+ Got->addIncoming(B.getTrue(), CritEnd);
+ Got->addIncoming(B.getFalse(), Head);
+ PHINode *Res = B.CreatePHI(ResTy, 2);
+ Res->addIncoming(Result, CritEnd);
+ // The result of a thread that has not done its work yet is not used.
+ Res->addIncoming(PoisonValue::get(ResTy), Head);
+ B.CreateCondBr(Got, Exit, Head);
+
+ if (!I.getType()->isVoidTy()) {
+ B.SetInsertPoint(&I);
+ PHINode *Out = B.CreatePHI(ResTy, 1);
+ Out->addIncoming(Res, Latch);
+ I.replaceAllUsesWith(Out);
+ }
+ I.eraseFromParent();
+}
+
+bool AGXExpandWideAtomics::runOnFunction(Function &F) {
+ if (skipFunction(F))
+ return false;
+ const DataLayout &DL = F.getDataLayout();
+ SmallVector<std::pair<Instruction *, unsigned>, 4> Wide;
+ for (Instruction &I : instructions(F))
+ if (unsigned Bits = wideAtomicSize(I, DL))
+ Wide.push_back({&I, Bits});
+ if (Wide.empty())
+ return false;
+ GlobalVariable *Locks = getLocks(*F.getParent());
+ for (auto [I, Bits] : Wide)
+ expand(*I, Bits, Locks);
+ return true;
+}
diff --git a/llvm/lib/Target/AGX/AGXTargetMachine.cpp b/llvm/lib/Target/AGX/AGXTargetMachine.cpp
index eabe994a180d2..cc8a9fd2d0a70 100644
--- a/llvm/lib/Target/AGX/AGXTargetMachine.cpp
+++ b/llvm/lib/Target/AGX/AGXTargetMachine.cpp
@@ -40,6 +40,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAGXTarget() {
initializeAGXStrictFPPass(PR);
initializeAGXLowerFPEnvPass(PR);
initializeAGXSoftF64Pass(PR);
+ initializeAGXExpandWideAtomicsPass(PR);
initializeAGXAnnotateControlFlowPass(PR);
initializeAGXDAGToDAGISelLegacyPass(PR);
initializeAGXFoldOperandsPass(PR);
@@ -147,6 +148,8 @@ void AGXPassConfig::addIRPasses() {
addPass(createAGXAlwaysInlinePass());
addPass(createAlwaysInlinerLegacyPass());
+ // 64- and 128-bit atomics take locks; the others are hardware atomics.
+ addPass(createAGXExpandWideAtomicsPass());
addPass(createAtomicExpandLegacyPass());
if (getOptLevel() != CodeGenOptLevel::None) {
addPass(createInferAddressSpacesPass());
diff --git a/llvm/lib/Target/AGX/CMakeLists.txt b/llvm/lib/Target/AGX/CMakeLists.txt
index 9d91d36725564..f955118e57dec 100644
--- a/llvm/lib/Target/AGX/CMakeLists.txt
+++ b/llvm/lib/Target/AGX/CMakeLists.txt
@@ -31,6 +31,7 @@ add_llvm_target(AGXCodeGen
AGXMCInstLower.cpp
AGXRegisterInfo.cpp
AGXSelectionDAGInfo.cpp
+ AGXExpandWideAtomics.cpp
AGXSoftF64.cpp
AGXStrictFP.cpp
AGXSubtarget.cpp
diff --git a/llvm/test/CodeGen/AGX/wide-atomics.ll b/llvm/test/CodeGen/AGX/wide-atomics.ll
new file mode 100644
index 0000000000000..6d31e7fd2c5d2
--- /dev/null
+++ b/llvm/test/CodeGen/AGX/wide-atomics.ll
@@ -0,0 +1,58 @@
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -o /dev/null -print-after=agx-expand-wide-atomics < %s 2>&1 | FileCheck %s
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -verify-machineinstrs < %s | FileCheck %s --check-prefix=ASM
+
+; 64- and 128-bit atomics take a lock from a table chosen by the address of
+; the 16-byte block, and access the value with 32-bit atomics while they
+; hold it, inside the loop that takes the lock.
+
+; CHECK-LABEL: @add64(
+; CHECK: lshr i64 {{.*}}, 4
+; CHECK: and i64 {{.*}}, 1023
+; CHECK: getelementptr inbounds [1024 x i32], ptr addrspace(1) @__agx_atomic_locks
+; CHECK: wide.atomic:
+; CHECK: cmpxchg ptr addrspace(1) %{{.*}}, i32 0, i32 1 monotonic monotonic
+; CHECK: wide.atomic.locked:
+; CHECK: fence seq_cst
+; CHECK-COUNT-2: atomicrmw or ptr addrspace(1) %{{.*}}, i32 0 monotonic
+; CHECK: add i64
+; CHECK-COUNT-2: atomicrmw xchg ptr addrspace(1) %{{.*}}, i32 %{{.*}} monotonic
+; CHECK: fence seq_cst
+; CHECK: atomicrmw xchg ptr addrspace(1) %{{.*}}, i32 0 monotonic
+; CHECK: wide.atomic.latch:
+; ASM-LABEL: add64:
+; ASM: global data buffer at
+; ASM: atomic cmpxchg
+define agx_kernel void @add64(ptr addrspace(1) %p, ptr addrspace(1) %o, i64 %v) {
+ %a = atomicrmw add ptr addrspace(1) %p, i64 %v monotonic
+ store i64 %a, ptr addrspace(1) %o
+ ret void
+}
+
+; A compare and swap stores only when the comparison succeeds.
+; CHECK-LABEL: @cas128(
+; CHECK-COUNT-4: atomicrmw or
+; CHECK: icmp eq i128
+; CHECK: br i1 %{{.*}}, label %wide.atomic.store, label %wide.atomic.unlock
+; CHECK: wide.atomic.store:
+; CHECK-COUNT-4: atomicrmw xchg
+define agx_kernel void @cas128(ptr addrspace(1) %p, ptr addrspace(1) %o, i128 %c, i128 %n) {
+ %x = cmpxchg ptr addrspace(1) %p, i128 %c, i128 %n seq_cst seq_cst
+ %v = extractvalue {i128, i1} %x, 0
+ store i128 %v, ptr addrspace(1) %o
+ ret void
+}
+
+; Atomic loads only read; double read-modify-write operations compute in
+; software.
+; CHECK-LABEL: @load_fadd(
+; CHECK-COUNT-2: atomicrmw or
+; CHECK-NOT: atomicrmw xchg ptr addrspace(1) %p
+; CHECK: fadd double
+define agx_kernel void @load_fadd(ptr addrspace(1) %p, ptr addrspace(1) %q, ptr addrspace(1) %o, double %d) {
+ %l = load atomic i64, ptr addrspace(1) %p seq_cst, align 8
+ store i64 %l, ptr addrspace(1) %o
+ %f = atomicrmw fadd ptr addrspace(1) %q, double %d monotonic
+ %o2 = getelementptr i64, ptr addrspace(1) %o, i32 1
+ store double %f, ptr addrspace(1) %o2
+ ret void
+}
>From a87e083e5ce204c97cafaf05c3d961c736188fcb Mon Sep 17 00:00:00 2001
From: Claude <noreply at anthropic.com>
Date: Mon, 19 Oct 2026 01:11:32 +0000
Subject: [PATCH 17/22] [AGX] Support function pointers and indirect calls
AGX has no calls, and everything is inlined. agx-always-inline now turns
each indirect call into tests of the callee against the address-taken
functions of its type, with direct (inlinable) calls, and a trap when none
matches; the addresses of those functions, as values, become their
numbers. Checked on the functional simulator with a table of function
pointers in device memory selected per thread.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
Claude-Session: https://claude.ai/code/session_01E56KEecg6edNaS23myRWER
---
llvm/docs/AGXUsage.rst | 8 +++-
llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp | 51 ++++++++++++++++++++++-
llvm/test/CodeGen/AGX/indirect-calls.ll | 41 ++++++++++++++++++
3 files changed, 98 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/CodeGen/AGX/indirect-calls.ll
diff --git a/llvm/docs/AGXUsage.rst b/llvm/docs/AGXUsage.rst
index de26c942de531..b0fab723163b8 100644
--- a/llvm/docs/AGXUsage.rst
+++ b/llvm/docs/AGXUsage.rst
@@ -262,7 +262,13 @@ flushed explicitly.
Not supported: ``frem`` and vector arithmetic on ``double`` (see `Double
precision`_; diagnosed as unsupported), vector memory
operations (vectors are scalarized), textures and graphics shaders, calls
-and recursion.
+to functions that are not defined in the module, and recursion.
+
+The hardware has no calls: all functions are inlined into the kernels. An
+indirect call becomes a test of the callee against each address-taken
+function of the module with the same type, with a direct call (inlined) for
+each, and a trap if none matches. The address of such a function, as a
+value (stored, compared or passed), is its number in the module (1, 2, ...).
.. _agx-apple9:
diff --git a/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp b/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp
index 75d88f0522153..fbe950ef51d02 100644
--- a/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp
@@ -39,6 +39,7 @@
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
+#include "llvm/Transforms/Utils/CallPromotionUtils.h"
#include "llvm/IR/InstIterator.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
@@ -81,14 +82,62 @@ INITIALIZE_PASS(AGXAlwaysInline, "agx-always-inline", "AGX Always Inline",
ModulePass *llvm::createAGXAlwaysInlinePass() { return new AGXAlwaysInline(); }
+/// There are no calls in the hardware, direct or indirect: an indirect call
+/// becomes a test of the callee against each address-taken function of its
+/// type, with a direct call (inlined later) for each, and a trap if none
+/// matches. The addresses of these functions, as values, become their
+/// numbers (1, 2, ...), so that they can be stored, compared and passed.
+static bool lowerIndirectCalls(Module &M) {
+ SmallVector<CallBase *, 8> Indirect;
+ for (Function &F : M)
+ for (Instruction &I : instructions(F))
+ if (auto *CB = dyn_cast<CallBase>(&I); CB && CB->isIndirectCall() &&
+ !CB->isInlineAsm())
+ Indirect.push_back(CB);
+ SmallVector<Function *, 8> Taken;
+ for (Function &F : M)
+ if (!F.isIntrinsic() && F.getCallingConv() != CallingConv::AGX_KERNEL &&
+ F.hasAddressTaken(nullptr, /*IgnoreCallbackUses=*/false,
+ /*IgnoreAssumeLikeCalls=*/true,
+ /*IngoreLLVMUsed=*/true))
+ Taken.push_back(&F);
+ if (Indirect.empty() && Taken.empty())
+ return false;
+ for (CallBase *CB : Indirect) {
+ CallBase *Rest = CB;
+ for (Function *F : Taken) {
+ if (F->getFunctionType() != CB->getFunctionType())
+ continue;
+ // The direct call is in the then block; the indirect one stays in the
+ // else block.
+ promoteCallWithIfThenElse(*Rest, F);
+ }
+ // No address-taken function of this type matches.
+ IRBuilder<> B(Rest);
+ B.CreateIntrinsic(Intrinsic::trap, {});
+ if (!Rest->getType()->isVoidTy())
+ Rest->replaceAllUsesWith(PoisonValue::get(Rest->getType()));
+ Rest->eraseFromParent();
+ }
+ for (auto [N, F] : enumerate(Taken)) {
+ Constant *Id = ConstantExpr::getIntToPtr(
+ ConstantInt::get(Type::getInt64Ty(M.getContext()), N + 1), F->getType());
+ F->replaceUsesWithIf(Id, [](Use &U) {
+ auto *CB = dyn_cast<CallBase>(U.getUser());
+ return !CB || !CB->isCallee(&U);
+ });
+ }
+ return true;
+}
+
bool AGXAlwaysInline::runOnModule(Module &M) {
+ bool Changed = lowerIndirectCalls(M);
// Modules without any agx_kernel function (for example tests of single
// functions) compile every external function as an entry point.
bool HasKernels = any_of(M, [](const Function &F) {
return F.getCallingConv() == CallingConv::AGX_KERNEL;
});
- bool Changed = false;
SmallVector<Function *, 8> Dead;
for (Function &F : M) {
if (F.isDeclaration() || F.getCallingConv() == CallingConv::AGX_KERNEL)
diff --git a/llvm/test/CodeGen/AGX/indirect-calls.ll b/llvm/test/CodeGen/AGX/indirect-calls.ll
new file mode 100644
index 0000000000000..fbaa7a8e358d8
--- /dev/null
+++ b/llvm/test/CodeGen/AGX/indirect-calls.ll
@@ -0,0 +1,41 @@
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -o /dev/null -print-after=agx-always-inline < %s 2>&1 | FileCheck %s
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -verify-machineinstrs < %s | FileCheck %s --check-prefix=ASM
+
+; An indirect call tests the callee against each address-taken function of
+; its type and calls it directly (the calls are inlined later); none
+; matching traps. Function addresses as values are the functions' numbers.
+
+; CHECK: @table = internal addrspace(1) global [3 x ptr] [ptr inttoptr (i64 1 to ptr), ptr inttoptr (i64 2 to ptr), ptr inttoptr (i64 3 to ptr)]
+ at table = internal addrspace(1) global [3 x ptr] [ptr @twice, ptr @square, ptr @other]
+
+define internal i32 @twice(i32 %x) {
+ %r = shl i32 %x, 1
+ ret i32 %r
+}
+
+define internal i32 @square(i32 %x) {
+ %r = mul i32 %x, %x
+ ret i32 %r
+}
+
+define internal float @other(float %x) {
+ ret float %x
+}
+
+; CHECK-LABEL: define agx_kernel void @k(
+; CHECK: icmp eq ptr %f, inttoptr (i64 1 to ptr)
+; CHECK: call i32 @twice(
+; CHECK: icmp eq ptr %f, inttoptr (i64 2 to ptr)
+; CHECK: call i32 @square(
+; CHECK-NOT: call i32 %f
+; CHECK-NOT: @other(
+; CHECK: call void @llvm.trap()
+; ASM-LABEL: k:
+; ASM-NOT: error
+define agx_kernel void @k(ptr addrspace(1) %o, i32 %i, i32 %x) {
+ %pf = getelementptr [3 x ptr], ptr addrspace(1) @table, i32 0, i32 %i
+ %f = load ptr, ptr addrspace(1) %pf
+ %r = call i32 %f(i32 %x)
+ store i32 %r, ptr addrspace(1) %o
+ ret void
+}
>From cc163e366dd7566952cc0bba9de0e937dd356898 Mon Sep 17 00:00:00 2001
From: Claude <noreply at anthropic.com>
Date: Mon, 19 Oct 2026 01:17:34 +0000
Subject: [PATCH 18/22] [AGX] Halve the software double precision code
Significands are now 64-bit (up to 63 bits and a sticky bit), with 128-bit
values only for the products of multiplications and fused multiply-adds,
which are compressed to 64 bits with a sticky bit before rounding. Division
is a restoring division and the square root a digit by digit root, both
loops of 55 iterations of 64-bit operations, instead of an expanded
128-bit division and a 64-iteration 128-bit root.
Instructions per kernel: addition 613 -> 315, division 808 -> 303, square
root 484 -> 252, conversions about 255 -> 165, floor 366 -> 188. All
operations stay bit-exact (ptx/tools/agx_f64_check.py in ZLUDA, all
rounding modes, with and without canonical-nan).
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
Claude-Session: https://claude.ai/code/session_01E56KEecg6edNaS23myRWER
---
llvm/docs/AGXUsage.rst | 15 ++-
llvm/lib/Target/AGX/AGXSoftF64.cpp | 208 ++++++++++++++++-------------
llvm/test/CodeGen/AGX/soft-f64.ll | 11 +-
3 files changed, 131 insertions(+), 103 deletions(-)
diff --git a/llvm/docs/AGXUsage.rst b/llvm/docs/AGXUsage.rst
index b0fab723163b8..ff2581ef56a65 100644
--- a/llvm/docs/AGXUsage.rst
+++ b/llvm/docs/AGXUsage.rst
@@ -370,19 +370,20 @@ exact sign, 128-bit significand and exponent, the exact result (with a sticky
bit for the bits dropped) is computed with integer operations, and a single
rounding gives the result, so all operations are correctly rounded IEEE 754
operations with denormals, in the four rounding directions (see `Rounding
-modes`_; the ``double`` rounding mode is the ``half`` field). Division uses an
-expanded 128-bit integer division and the square root a loop that produces a
-bit of the root per iteration. NaN results are ``0x7fffffffffffffff`` with
+modes`_; the ``double`` rounding mode is the ``half`` field). Significands are 64-bit, with 128-bit
+products in multiplications and fused multiply-adds; division is a restoring
+division and the square root a digit by digit root, each a loop of 55
+iterations of 64-bit operations. NaN results are ``0x7fffffffffffffff`` with
``canonical-nan``, and the first NaN operand, quieted, otherwise. Conversions
to integers saturate, and NaN converts to 0.
Values that are only moved (loads, stores, ``phi``, ``select``, arguments)
stay ``double`` and become ``i64`` in type legalization. ``frem`` and vector
arithmetic on ``double`` are diagnosed as unsupported. The code is large: in
-instructions per operation, about 590 for an addition, 350 for a
-multiplication, 690 for a fused multiply-add, 780 for a division (plus the
-iterations of the division loop), 460 for a square root (and 64 iterations),
-230 for a conversion and 20 for a comparison.
+instructions per operation, about 290 for an addition, 335 for a
+multiplication, 670 for a fused multiply-add, 280 for a division and 230 for
+a square root (plus their 55 loop iterations of about 10 instructions), 140
+for a conversion and 20 for a comparison.
Vector floating point arithmetic is scalarized before the floating point
lowerings (the Scalarizer pass), and constrained vector operations are rounded
diff --git a/llvm/lib/Target/AGX/AGXSoftF64.cpp b/llvm/lib/Target/AGX/AGXSoftF64.cpp
index d11aca2ea654e..f4b8b0c98a5ee 100644
--- a/llvm/lib/Target/AGX/AGXSoftF64.cpp
+++ b/llvm/lib/Target/AGX/AGXSoftF64.cpp
@@ -63,7 +63,7 @@ enum RoundingCode : uint32_t {
RoundDown = 3,
};
-/// sign * M * 2^E, with Sign i1, M i128 and E i32.
+/// sign * M * 2^E, with Sign i1, M i64 (or i128 for products) and E i32.
struct Term {
Value *Sign;
Value *M;
@@ -77,10 +77,6 @@ class SoftDouble {
bool CanonicalNaN;
public:
- /// The 128-bit divisions built, which the caller expands into integer
- /// arithmetic once the replacement is complete.
- SmallVector<BinaryOperator *, 1> Divisions;
-
SoftDouble(IRBuilderBase &B, Value *Rounding, bool CanonicalNaN)
: B(B), I1(B.getInt1Ty()), I32(B.getInt32Ty()), I64(B.getInt64Ty()),
I128(B.getInt128Ty()), Rounding(Rounding), CanonicalNaN(CanonicalNaN) {}
@@ -88,6 +84,8 @@ class SoftDouble {
Value *c32(int64_t V) { return ConstantInt::getSigned(I32, V); }
Value *c64(uint64_t V) { return B.getInt64(V); }
Value *c128(uint64_t V) { return ConstantInt::get(I128, V); }
+ Value *cw(Value *Like, uint64_t V) { return ConstantInt::get(Like->getType(), V); }
+ static unsigned width(Value *M) { return M->getType()->getIntegerBitWidth(); }
Value *roundingIs(RoundingCode C) {
return B.CreateICmpEQ(Rounding, B.getInt32(C));
}
@@ -99,25 +97,69 @@ class SoftDouble {
return B.CreateBinaryIntrinsic(Intrinsic::umin, A, X);
}
- /// Bit length of a 128-bit value, as i32.
+ /// Bit length of M, as i32.
Value *bitLength(Value *M) {
Value *Lz = B.CreateBinaryIntrinsic(Intrinsic::ctlz, M, B.getFalse());
- return B.CreateTrunc(B.CreateSub(c128(128), Lz), I32);
+ return B.CreateTrunc(B.CreateSub(cw(M, width(M)), Lz), I32);
}
- /// M << S and M >> S for an i32 S >= 0; amounts of 128 or more give 0.
+ /// M << S and M >> S for an i32 S >= 0; amounts of the width of M or more
+ /// give 0.
Value *shl(Value *M, Value *S) {
- Value *R = B.CreateShl(M, B.CreateZExt(umin(S, c32(127)), I128));
- return select(B.CreateICmpUGT(S, c32(127)), c128(0), R);
+ unsigned W = width(M);
+ Value *R = B.CreateShl(M, B.CreateZExt(umin(S, c32(W - 1)), M->getType()));
+ return select(B.CreateICmpUGT(S, c32(W - 1)), cw(M, 0), R);
}
Value *lshr(Value *M, Value *S) {
- Value *R = B.CreateLShr(M, B.CreateZExt(umin(S, c32(127)), I128));
- return select(B.CreateICmpUGT(S, c32(127)), c128(0), R);
+ unsigned W = width(M);
+ Value *R = B.CreateLShr(M, B.CreateZExt(umin(S, c32(W - 1)), M->getType()));
+ return select(B.CreateICmpUGT(S, c32(W - 1)), cw(M, 0), R);
}
/// M >> S with the bits shifted out ORed into the lowest bit.
Value *lshrSticky(Value *M, Value *S) {
Value *R = lshr(M, S);
Value *Lost = B.CreateICmpNE(shl(R, S), M);
- return B.CreateOr(R, B.CreateZExt(Lost, I128));
+ return B.CreateOr(R, B.CreateZExt(Lost, M->getType()));
+ }
+ /// A term with a 128-bit M as one with a 64-bit M (and a sticky bit).
+ Term compress(Term T) {
+ Value *S = smax(B.CreateSub(bitLength(T.M), c32(63)), c32(0));
+ return {T.Sign, B.CreateTrunc(lshrSticky(T.M, S), I64), B.CreateAdd(T.E, S)};
+ }
+ Term widen(Term T) { return {T.Sign, B.CreateZExt(T.M, I128), T.E}; }
+
+ /// A counted loop: Count iterations of Body, which maps the iteration
+ /// number (Count - 1 down to 0) and the current values to the next ones,
+ /// starting from Init. Returns the final values.
+ SmallVector<Value *, 2>
+ loop(unsigned Count, ArrayRef<Value *> Init,
+ function_ref<SmallVector<Value *, 2>(Value *, ArrayRef<Value *>)> Body) {
+ BasicBlock *Pre = B.GetInsertBlock();
+ Instruction *At = &*B.GetInsertPoint();
+ BasicBlock *Tail = Pre->splitBasicBlock(At->getIterator(), "soft.loop.end");
+ BasicBlock *Loop = BasicBlock::Create(B.getContext(), "soft.loop", Pre->getParent(), Tail);
+ Pre->getTerminator()->setSuccessor(0, Loop);
+ B.SetInsertPoint(Loop);
+ PHINode *I = B.CreatePHI(I32, 2);
+ I->addIncoming(c32(Count - 1), Pre);
+ SmallVector<Value *, 2> Cur;
+ for (Value *V : Init) {
+ PHINode *P = B.CreatePHI(V->getType(), 2);
+ P->addIncoming(V, Pre);
+ Cur.push_back(P);
+ }
+ SmallVector<Value *, 2> Next = Body(I, Cur);
+ I->addIncoming(B.CreateSub(I, c32(1)), Loop);
+ for (auto [P, V] : zip(Cur, Next))
+ cast<PHINode>(P)->addIncoming(V, Loop);
+ B.CreateCondBr(B.CreateICmpEQ(I, c32(0)), Tail, Loop);
+ B.SetInsertPoint(At);
+ SmallVector<Value *, 2> Out;
+ for (Value *V : Next) {
+ PHINode *P = B.CreatePHI(V->getType(), 1);
+ P->addIncoming(V, Loop);
+ Out.push_back(P);
+ }
+ return Out;
}
// Classification of bits (an i64 holding a value of format F).
@@ -165,7 +207,7 @@ class SoftDouble {
Value *M = select(Denormal, Frac, B.CreateOr(Frac, c64(F.fracMask() + 1)));
Value *E = B.CreateTrunc(select(Denormal, c64(1), Exp), I32);
E = B.CreateSub(E, c32(F.Bias + int(F.P) - 1));
- return {signOf(Bits, F), B.CreateZExt(M, I128), E};
+ return {signOf(Bits, F), M, E};
}
/// T rounded to format F in the rounding direction, as bits in an i64.
@@ -179,11 +221,11 @@ class SoftDouble {
shl(T.M, B.CreateNeg(B.CreateBinaryIntrinsic(Intrinsic::smin, Shift, c32(0)))));
Value *Rem = B.CreateSub(T.M, shl(lshr(T.M, RS), RS));
Value *GuardS = B.CreateSub(RS, c32(1));
- Value *Guard = B.CreateTrunc(lshr(Rem, umin(GuardS, c32(128))), I1);
+ Value *Guard = B.CreateTrunc(lshr(Rem, umin(GuardS, c32(width(T.M)))), I1);
// The bits below the guard bit; all of them if the guard bit is above
- // bit 127.
- Value *Half = shl(c128(1), GuardS);
- Value *Sticky = B.CreateICmpNE(B.CreateAnd(Rem, B.CreateSub(Half, c128(1))), c128(0));
+ // the top bit.
+ Value *Half = shl(cw(T.M, 1), GuardS);
+ Value *Sticky = B.CreateICmpNE(B.CreateAnd(Rem, B.CreateSub(Half, cw(T.M, 1))), cw(T.M, 0));
Guard = B.CreateAnd(Guard, Right);
Sticky = B.CreateAnd(Sticky, Right);
Value *Odd = B.CreateTrunc(Q, I1);
@@ -193,14 +235,14 @@ class SoftDouble {
select(roundingIs(RoundUp), B.CreateAnd(Inexact, B.CreateNot(T.Sign)),
select(roundingIs(RoundDown), B.CreateAnd(Inexact, T.Sign),
B.getFalse())));
- Q = B.CreateAdd(Q, B.CreateZExt(Inc, I128));
+ Q = B.CreateAdd(Q, B.CreateZExt(Inc, Q->getType()));
// bits = k << (P - 1) + Q, with k the biased exponent of the lowest bit
// of Q; a carry out of Q increments the exponent.
Value *K = B.CreateSub(B.CreateAdd(T.E, Shift), c32(F.minLSB()));
K = umin(K, c32(F.expAll() + 1));
Value *Bits = B.CreateAdd(B.CreateShl(B.CreateZExt(K, I64), F.P - 1),
- B.CreateTrunc(Q, I64));
- Bits = select(B.CreateICmpEQ(T.M, c128(0)), c64(0), Bits);
+ B.CreateZExtOrTrunc(Q, I64));
+ Bits = select(B.CreateICmpEQ(T.M, cw(T.M, 0)), c64(0), Bits);
Value *Overflow = B.CreateICmpUGE(Bits, c64(F.infBits()));
// Overflows round to infinity or to the largest finite value.
Value *ToInf = select(roundingIs(RoundNearestEven), B.getTrue(),
@@ -210,17 +252,19 @@ class SoftDouble {
return B.CreateOr(Bits, select(T.Sign, c64(F.signBit()), c64(0)));
}
- /// The exact sum of A and B (with M of at most 126 bits), with a sticky bit.
+ /// The exact sum of A and B (with M of at most 10 bits less than its
+ /// width), with a sticky bit.
Term addTerms(Term X, Term Y) {
+ unsigned Top = width(X.M) - 2;
auto Normalize = [&](Term T) {
- Value *S = B.CreateSub(c32(126), bitLength(T.M));
+ Value *S = B.CreateSub(c32(Top), bitLength(T.M));
return Term{T.Sign, shl(T.M, S), B.CreateSub(T.E, S)};
};
X = Normalize(X);
Y = Normalize(Y);
// Order by magnitude; zeros are the smallest.
- Value *XZero = B.CreateICmpEQ(X.M, c128(0));
- Value *YZero = B.CreateICmpEQ(Y.M, c128(0));
+ Value *XZero = B.CreateICmpEQ(X.M, cw(X.M, 0));
+ Value *YZero = B.CreateICmpEQ(Y.M, cw(Y.M, 0));
Value *YBigger = B.CreateOr(
B.CreateICmpSGT(Y.E, X.E),
B.CreateAnd(B.CreateICmpEQ(Y.E, X.E), B.CreateICmpUGT(Y.M, X.M)));
@@ -230,7 +274,7 @@ class SoftDouble {
Term Small{select(YBigger, X.Sign, Y.Sign), select(YBigger, X.M, Y.M),
select(YBigger, X.E, Y.E)};
Value *D = B.CreateSub(Big.E, Small.E);
- Value *SM = select(B.CreateICmpEQ(Small.M, c128(0)), c128(0),
+ Value *SM = select(B.CreateICmpEQ(Small.M, cw(Small.M, 0)), cw(Small.M, 0),
lshrSticky(Small.M, D));
Value *Same = B.CreateICmpEQ(Big.Sign, Small.Sign);
Value *M = select(Same, B.CreateAdd(Big.M, SM), B.CreateSub(Big.M, SM));
@@ -246,7 +290,7 @@ class SoftDouble {
Value *withZeroSign(Term T, Value *ZeroSign, Fmt F = F64) {
// round() gives the sign of T to a zero.
- return round({select(B.CreateICmpEQ(T.M, c128(0)), ZeroSign, T.Sign), T.M, T.E}, F);
+ return round({select(B.CreateICmpEQ(T.M, cw(T.M, 0)), ZeroSign, T.Sign), T.M, T.E}, F);
}
Value *add(Value *X, Value *Y, bool Sub) {
@@ -269,7 +313,8 @@ class SoftDouble {
Value *mul(Value *X, Value *Y) {
Term TX = unpack(X, F64), TY = unpack(Y, F64);
Value *Sign = B.CreateXor(TX.Sign, TY.Sign);
- Value *R = round({Sign, B.CreateMul(TX.M, TY.M), B.CreateAdd(TX.E, TY.E)}, F64);
+ Value *P = B.CreateMul(B.CreateZExt(TX.M, I128), B.CreateZExt(TY.M, I128));
+ Value *R = round(compress({Sign, P, B.CreateAdd(TX.E, TY.E)}), F64);
Value *XInf = isInf(X, F64), *YInf = isInf(Y, F64);
Value *Inf = B.CreateOr(c64(F64.infBits()), select(Sign, c64(F64.signBit()), c64(0)));
Value *NaN = B.CreateOr(
@@ -281,9 +326,10 @@ class SoftDouble {
Value *fma(Value *X, Value *Y, Value *Z, Fmt F = F64) {
Term TX = unpack(X, F), TY = unpack(Y, F), TZ = unpack(Z, F);
- Term P{B.CreateXor(TX.Sign, TY.Sign), B.CreateMul(TX.M, TY.M),
+ Term P{B.CreateXor(TX.Sign, TY.Sign),
+ B.CreateMul(B.CreateZExt(TX.M, I128), B.CreateZExt(TY.M, I128)),
B.CreateAdd(TX.E, TY.E)};
- Term S = addTerms(P, TZ);
+ Term S = compress(addTerms(P, widen(TZ)));
Value *PZero = B.CreateOr(isZero(X, F), isZero(Y, F));
Value *R = withZeroSign(
S, zeroSumSign(B.CreateAnd(PZero, isZero(Z, F)), P.Sign, TZ.Sign), F);
@@ -304,19 +350,24 @@ class SoftDouble {
Value *div(Value *X, Value *Y) {
Term TX = unpack(X, F64), TY = unpack(Y, F64);
Value *Sign = B.CreateXor(TX.Sign, TY.Sign);
- // The dividend with its top bit at 126 over the divisor with its top bit
- // at 63: a quotient of at least 63 bits.
- Value *SX = B.CreateSub(c32(126), bitLength(TX.M));
- Value *SY = B.CreateSub(c32(63), bitLength(TY.M));
+ // Both significands with their top bit at 52, the dividend doubled if it
+ // is the smaller: a restoring division gives a quotient of 55 bits.
+ Value *SX = B.CreateSub(c32(53), bitLength(TX.M));
+ Value *SY = B.CreateSub(c32(53), bitLength(TY.M));
Value *N = shl(TX.M, SX);
Value *Dv = shl(TY.M, SY);
- Value *SafeDv = select(B.CreateICmpEQ(Dv, c128(0)), c128(1), Dv);
- Value *Q = B.CreateUDiv(N, SafeDv);
- Divisions.push_back(cast<BinaryOperator>(Q));
- Value *Inexact = B.CreateICmpNE(B.CreateMul(Q, SafeDv), N);
- Value *M = B.CreateOr(B.CreateShl(Q, 1), B.CreateZExt(Inexact, I128));
+ Value *Less = B.CreateICmpULT(N, Dv);
+ N = select(Less, B.CreateShl(N, 1), N);
+ SmallVector<Value *, 2> QR = loop(55, {c64(0), N}, [&](Value *, ArrayRef<Value *> V) {
+ Value *Ge = B.CreateICmpUGE(V[1], Dv);
+ Value *Q = B.CreateOr(B.CreateShl(V[0], 1), B.CreateZExt(Ge, I64));
+ Value *R = B.CreateShl(select(Ge, B.CreateSub(V[1], Dv), V[1]), 1);
+ return SmallVector<Value *, 2>{Q, R};
+ });
+ Value *Inexact = B.CreateICmpNE(QR[1], c64(0));
+ Value *M = B.CreateOr(B.CreateShl(QR[0], 1), B.CreateZExt(Inexact, I64));
Value *E = B.CreateSub(B.CreateSub(TX.E, SX), B.CreateSub(TY.E, SY));
- E = B.CreateSub(E, c32(1));
+ E = B.CreateSub(E, B.CreateAdd(c32(55), B.CreateZExt(Less, I32)));
Value *R = round({Sign, M, E}, F64);
Value *SignBits = select(Sign, c64(F64.signBit()), c64(0));
Value *XZero = isZero(X, F64), *YZero = isZero(Y, F64);
@@ -328,54 +379,29 @@ class SoftDouble {
return select(NaN, nanResult({X, Y}, F64, F64), R);
}
- /// The integer square root of the i128 N (and whether it is exact), with a
- /// loop that produces a bit per iteration.
- std::pair<Value *, Value *> isqrt(Value *N) {
- BasicBlock *Pre = B.GetInsertBlock();
- Instruction *At = &*B.GetInsertPoint();
- BasicBlock *Tail = Pre->splitBasicBlock(At->getIterator(), "isqrt.end");
- Function *F = Pre->getParent();
- BasicBlock *Loop = BasicBlock::Create(B.getContext(), "isqrt", F, Tail);
- Pre->getTerminator()->setSuccessor(0, Loop);
- B.SetInsertPoint(Loop);
- PHINode *I = B.CreatePHI(I32, 2);
- PHINode *Rem = B.CreatePHI(I128, 2);
- PHINode *Root = B.CreatePHI(I128, 2);
- I->addIncoming(c32(63), Pre);
- Rem->addIncoming(c128(0), Pre);
- Root->addIncoming(c128(0), Pre);
- Value *Pair = B.CreateAnd(
- B.CreateLShr(N, B.CreateZExt(B.CreateShl(I, 1), I128)), c128(3));
- Value *R2 = B.CreateOr(B.CreateShl(Rem, 2), Pair);
- Value *Rt = B.CreateShl(Root, 1);
- Value *T = B.CreateOr(B.CreateShl(Rt, 1), c128(1));
- Value *Ge = B.CreateICmpUGE(R2, T);
- Value *NewRem = select(Ge, B.CreateSub(R2, T), R2);
- Value *NewRoot = B.CreateOr(Rt, B.CreateZExt(Ge, I128));
- Value *NextI = B.CreateSub(I, c32(1));
- I->addIncoming(NextI, Loop);
- Rem->addIncoming(NewRem, Loop);
- Root->addIncoming(NewRoot, Loop);
- B.CreateCondBr(B.CreateICmpEQ(I, c32(0)), Tail, Loop);
- B.SetInsertPoint(At);
- PHINode *RootOut = B.CreatePHI(I128, 1);
- RootOut->addIncoming(NewRoot, Loop);
- PHINode *RemOut = B.CreatePHI(I128, 1);
- RemOut->addIncoming(NewRem, Loop);
- return {RootOut, B.CreateICmpEQ(RemOut, c128(0))};
- }
-
Value *sqrt(Value *X) {
Term TX = unpack(X, F64);
- // The radicand with its top bit at 125 or 126 and an even exponent.
- Value *S = B.CreateSub(c32(126), bitLength(TX.M));
+ // The radicand with its top bit at 52 or 53 and an even exponent. Its
+ // root times 2^28 has 55 bits; the bit pairs of the radicand times 2^56
+ // are produced from the top, one per iteration.
+ Value *S = B.CreateSub(c32(53), bitLength(TX.M));
Value *Odd = B.CreateTrunc(B.CreateSub(TX.E, S), I1);
- S = B.CreateSub(S, B.CreateZExt(Odd, I32));
+ S = B.CreateAdd(S, B.CreateZExt(Odd, I32));
Value *N = shl(TX.M, S);
Value *E = B.CreateSub(TX.E, S);
- auto [Root, Exact] = isqrt(N);
- Value *M = B.CreateOr(B.CreateShl(Root, 1), B.CreateZExt(B.CreateNot(Exact), I128));
- Value *R = round({B.getFalse(), M, B.CreateSub(B.CreateAShr(E, 1), c32(1))}, F64);
+ SmallVector<Value *, 2> RR = loop(55, {c64(0), c64(0)}, [&](Value *I, ArrayRef<Value *> V) {
+ Value *Sh = B.CreateZExt(B.CreateSub(B.CreateShl(I, 1), c32(56)), I64);
+ Value *Pair = select(B.CreateICmpUGE(I, c32(28)),
+ B.CreateAnd(B.CreateLShr(N, Sh), c64(3)), c64(0));
+ Value *Rem = B.CreateOr(B.CreateShl(V[1], 2), Pair);
+ Value *T = B.CreateOr(B.CreateShl(V[0], 2), c64(1));
+ Value *Ge = B.CreateICmpUGE(Rem, T);
+ Value *Root = B.CreateOr(B.CreateShl(V[0], 1), B.CreateZExt(Ge, I64));
+ return SmallVector<Value *, 2>{Root, select(Ge, B.CreateSub(Rem, T), Rem)};
+ });
+ Value *Inexact = B.CreateICmpNE(RR[1], c64(0));
+ Value *M = B.CreateOr(B.CreateShl(RR[0], 1), B.CreateZExt(Inexact, I64));
+ Value *R = round({B.getFalse(), M, B.CreateSub(B.CreateAShr(E, 1), c32(29))}, F64);
R = select(B.CreateOr(isZero(X, F64), B.CreateICmpEQ(X, c64(F64.infBits()))), X, R);
Value *NaN = B.CreateOr(isNaN(X, F64),
B.CreateAnd(signOf(X, F64), B.CreateNot(isZero(X, F64))));
@@ -396,7 +422,7 @@ class SoftDouble {
Value *Neg = Signed ? B.CreateICmpSLT(V, ConstantInt::get(V->getType(), 0))
: B.getFalse();
Value *Mag = Signed ? select(Neg, B.CreateNeg(V), V) : V;
- Value *M = N < 128 ? B.CreateZExt(Mag, I128) : Mag;
+ Value *M = N < 64 ? B.CreateZExt(Mag, I64) : Mag;
return round({Neg, M, c32(0)}, F64);
}
@@ -404,7 +430,7 @@ class SoftDouble {
/// gives 0).
Value *toInt(Value *X, Type *Ty, bool Signed) {
unsigned N = Ty->getIntegerBitWidth();
- Term T = unpack(X, F64);
+ Term T = widen(unpack(X, F64));
Value *E = T.E;
Value *Big = B.CreateICmpSGT(E, c32(int(N)));
Value *V = select(B.CreateICmpSGE(E, c32(0)),
@@ -429,10 +455,10 @@ class SoftDouble {
Value *NE = B.CreateNeg(B.CreateBinaryIntrinsic(Intrinsic::smin, T.E, c32(0)));
Value *Q = lshr(T.M, NE);
Value *Rem = B.CreateSub(T.M, shl(Q, NE));
- Value *Half = shl(c128(1), B.CreateSub(NE, c32(1)));
- Value *HasFrac = B.CreateICmpNE(Rem, c128(0));
- // Below 2^-75 there is no half to reach (Half is then 0).
- Value *HasHalf = B.CreateAnd(B.CreateICmpNE(NE, c32(0)), B.CreateICmpULE(NE, c32(128)));
+ Value *Half = shl(c64(1), B.CreateSub(NE, c32(1)));
+ Value *HasFrac = B.CreateICmpNE(Rem, c64(0));
+ // Far below 1/2 there is no half to reach (Half is then 0).
+ Value *HasHalf = B.CreateAnd(B.CreateICmpNE(NE, c32(0)), B.CreateICmpULE(NE, c32(64)));
Value *AboveHalf = B.CreateAnd(HasHalf, B.CreateICmpUGT(Rem, Half));
Value *IsHalf = B.CreateAnd(HasHalf, B.CreateICmpEQ(Rem, Half));
Value *Odd = B.CreateTrunc(Q, I1);
@@ -461,7 +487,7 @@ class SoftDouble {
B.getFalse())));
break;
}
- Q = B.CreateAdd(Q, B.CreateZExt(Inc, I128));
+ Q = B.CreateAdd(Q, B.CreateZExt(Inc, I64));
Value *R = round({T.Sign, Q, c32(0)}, F64);
// Values of at least 2^52 are integers.
Value *Integer = B.CreateICmpUGE(magnitude(X, F64), c64(uint64_t(1075) << 52));
@@ -774,8 +800,6 @@ Value *AGX::lowerDoubleArithmetic(Instruction &I, Value *Rounding,
R->takeName(&I);
I.replaceAllUsesWith(R);
I.eraseFromParent();
- for (BinaryOperator *Div : SD.Divisions)
- expandDivision(Div);
return R;
}
diff --git a/llvm/test/CodeGen/AGX/soft-f64.ll b/llvm/test/CodeGen/AGX/soft-f64.ll
index e77c57f3ee05d..9c43277db5eea 100644
--- a/llvm/test/CodeGen/AGX/soft-f64.ll
+++ b/llvm/test/CodeGen/AGX/soft-f64.ll
@@ -6,6 +6,7 @@
; CHECK-LABEL: @arith(
; CHECK-NOT: {{fadd|fmul|fdiv|fpext|fptrunc|fcmp}} double
+; CHECK: call i64 @llvm.ctlz.i64
; CHECK: mul i128
; CHECK: call i128 @llvm.ctlz.i128
; CHECK: bitcast i64 {{.*}} to double
@@ -23,11 +24,13 @@ define agx_kernel void @arith(ptr addrspace(1) %out, double %a, float %b) {
ret void
}
-; The square root is a loop producing a bit of the root per iteration.
+; The square root is a loop producing a bit of the root per iteration, on
+; 64-bit values.
; CHECK-LABEL: @root(
-; CHECK: isqrt:
-; CHECK: phi i128
-; CHECK: br i1 {{.*}}, label %isqrt.end, label %isqrt
+; CHECK: soft.loop:
+; CHECK: phi i64
+; CHECK-NOT: i128
+; CHECK: br i1 {{.*}}, label %soft.loop.end, label %soft.loop
define agx_kernel void @root(ptr addrspace(1) %out, double %a) {
%r = call double @llvm.sqrt.f64(double %a)
store double %r, ptr addrspace(1) %out
>From 939c63ad63dccc9d172c4eb0221c90f0533547e2 Mon Sep 17 00:00:00 2001
From: Claude <noreply at anthropic.com>
Date: Mon, 19 Oct 2026 01:22:36 +0000
Subject: [PATCH 19/22] [AGX] Exact cases of exp2, sin and cos in strict mode;
f16 fneg and fabs as bit operations
- With ieee-fp32, llvm.exp2 is 2^f (hardware, f in [0, 1)) scaled by
2^floor(x) exactly, so integral operands give exact powers of 2 and
denormal results are rounded once; llvm.sin and llvm.cos of operands
below 2^-12 are x and 1 (keeping -0 and denormals).
- A standalone fneg or fabs of half was an addition, which canonicalizes
NaNs; it is now a bit operation like float's.
PTX-formalization fmisc-0.ptx (ex2.approx, sin.approx) and half-2.ptx
(neg.f16 of a NaN) now pass with --strict-fp.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
Claude-Session: https://claude.ai/code/session_01E56KEecg6edNaS23myRWER
---
llvm/docs/AGXUsage.rst | 9 ++++--
llvm/lib/Target/AGX/AGXInstructions.td | 8 ++---
llvm/lib/Target/AGX/AGXStrictFP.cpp | 42 ++++++++++++++++++++------
llvm/test/CodeGen/AGX/float-misc.ll | 13 ++++++++
4 files changed, 57 insertions(+), 15 deletions(-)
diff --git a/llvm/docs/AGXUsage.rst b/llvm/docs/AGXUsage.rst
index ff2581ef56a65..03eddcc0c0d4a 100644
--- a/llvm/docs/AGXUsage.rst
+++ b/llvm/docs/AGXUsage.rst
@@ -330,8 +330,13 @@ PTX behavior in software (``AGXStrictFP.cpp``):
the normal range use integer arithmetic.
* Comparisons, ``minnum``/``maxnum``/``minimum``/``maximum`` (and the
``num`` variants) compare zeros and denormals as integers, and rounding
- functions, ``llvm.agx.rcp``, ``llvm.agx.rsqrt``, ``llvm.log2`` and
- ``llvm.exp2`` handle denormal operands or results explicitly.
+ functions, ``llvm.agx.rcp``, ``llvm.agx.rsqrt`` and ``llvm.log2`` handle
+ denormal operands or results explicitly. ``llvm.exp2`` computes
+ 2\ :sup:`f` in hardware for the fraction f of x and scales it by
+ 2\ :sup:`floor(x)` exactly, so that integral operands give exact powers of
+ 2 and denormal results are rounded once; ``llvm.sin`` and ``llvm.cos`` of
+ operands below 2\ :sup:`-12` are x and 1 (keeping the sign of zero and
+ denormal operands). They are otherwise the hardware approximations.
In functions that flush ``float`` denormals, ``fdiv`` and ``llvm.sqrt`` are
correctly rounded in the same way, on flushed operands and with a flushed
diff --git a/llvm/lib/Target/AGX/AGXInstructions.td b/llvm/lib/Target/AGX/AGXInstructions.td
index 4da8316ad2463..bb3de420d5420 100644
--- a/llvm/lib/Target/AGX/AGXInstructions.td
+++ b/llvm/lib/Target/AGX/AGXInstructions.td
@@ -617,16 +617,16 @@ def : Pat<(f16 (fmul (AGXFPSrc16 f16:$a, i32:$am), (AGXFPSrc16 f16:$b, i32:$bm))
(FMUL16_F16 $am, $a, $bm, $b)>;
// A negation or absolute value of its own is a bit operation: an addition
-// with a source modifier would flush a denormal operand. (Those that feed a
-// floating point operation are its source modifiers.)
+// with a source modifier would flush a denormal operand and change NaNs.
+// (Those that feed a floating point operation are its source modifiers.)
def : Pat<(f32 (fneg f32:$a)),
(BITOP_I32 TT_XOR, 0, $a, 0, (MOV_IMM32 0x80000000))>;
def : Pat<(f32 (fabs f32:$a)),
(BITOP_I32 TT_AND, 0, $a, 0, (MOV_IMM32 0x7fffffff))>;
def : Pat<(f16 (fneg f16:$a)),
- (FADD16_F16 MOD_NEG, $a, 0, (i16 FPIMM_NEG_ZERO))>;
+ (BITOP_I16 TT_XOR, 0, $a, 0, (MOV_IMM16 0x8000))>;
def : Pat<(f16 (fabs f16:$a)),
- (FADD16_F16 MOD_ABS, $a, 0, (i16 FPIMM_NEG_ZERO))>;
+ (BITOP_I16 TT_AND, 0, $a, 0, (MOV_IMM16 0x7fff))>;
def : Pat<(f32 (fcanonicalize (AGXFPSrc32 f32:$a, i32:$am))),
(FADD32_F32 $am, $a, 0, (i32 FPIMM_NEG_ZERO))>;
diff --git a/llvm/lib/Target/AGX/AGXStrictFP.cpp b/llvm/lib/Target/AGX/AGXStrictFP.cpp
index f41f3b1315915..e8c57450ab293 100644
--- a/llvm/lib/Target/AGX/AGXStrictFP.cpp
+++ b/llvm/lib/Target/AGX/AGXStrictFP.cpp
@@ -979,21 +979,45 @@ static bool lowerIEEE(Instruction &I) {
return true;
}
case Intrinsic::exp2: {
- // Results below 2^-126 are denormals: 2^(x + 64) * 2^-64.
+ // 2^x = 2^f * 2^n with n = floor(x): the hardware computes 2^f in
+ // [1, 2), and the scaling by 2^n is exact (rounded once for denormal
+ // results), so integral x give exact powers of 2.
Value *X = II->getArgOperand(0);
SmallVector<Use *, 4> Uses;
for (Use &U : I.uses())
Uses.push_back(&U);
IRBuilder<> B(I.getNextNode());
SoftFloat SF(B);
- Value *Scaled = B.CreateIntrinsic(Intrinsic::exp2, {B.getFloatTy()},
- {B.CreateFAdd(X, SF.fconst(64.0f))});
- Value *N = B.CreateUnaryIntrinsic(
- Intrinsic::rint, B.CreateFMul(Scaled, SF.fconst(0x1p85f)));
- Value *Den = SF.flt(B.CreateFPToUI(N, B.getInt32Ty()));
- Value *InRange = B.CreateAnd(B.CreateFCmpOLT(X, SF.fconst(-126.0f)),
- B.CreateFCmpOGT(X, SF.fconst(-151.0f)));
- Value *R = B.CreateSelect(InRange, Den, &I);
+ // Below -152 the result rounds to 0 anyway; clamp so that n fits.
+ Value *XC = B.CreateBinaryIntrinsic(Intrinsic::maxnum, X, SF.fconst(-160.0f));
+ XC = B.CreateBinaryIntrinsic(Intrinsic::minnum, XC, SF.fconst(160.0f));
+ Value *N = B.CreateUnaryIntrinsic(Intrinsic::floor, XC);
+ Value *Frac = B.CreateFSub(XC, N);
+ Value *P = B.CreateIntrinsic(Intrinsic::exp2, {B.getFloatTy()}, {Frac});
+ Term T = SF.unpack(SF.bits(P), false);
+ T.E = B.CreateAdd(T.E, B.CreateFPToSI(N, B.getInt32Ty()));
+ Value *R = SF.flt(SF.convert(T));
+ // NaN and infinities (2^-inf = 0) as the hardware gives them.
+ Value *Special = B.CreateFCmpUEQ(B.CreateUnaryIntrinsic(Intrinsic::fabs, X),
+ SF.fconst(INFINITY));
+ R = B.CreateSelect(Special, &I, R);
+ for (Use *U : Uses)
+ U->set(R);
+ return true;
+ }
+ case Intrinsic::sin:
+ case Intrinsic::cos: {
+ // Below 2^-12, sin x and cos x round to x and 1 (the hardware flushes
+ // denormal operands and loses the sign of zero).
+ Value *X = II->getArgOperand(0);
+ SmallVector<Use *, 4> Uses;
+ for (Use &U : I.uses())
+ Uses.push_back(&U);
+ IRBuilder<> B(I.getNextNode());
+ SoftFloat SF(B);
+ Value *Small = B.CreateFCmpOLT(B.CreateUnaryIntrinsic(Intrinsic::fabs, X),
+ SF.fconst(0x1p-12f));
+ Value *R = B.CreateSelect(Small, ID == Intrinsic::sin ? X : SF.fconst(1.0f), &I);
for (Use *U : Uses)
U->set(R);
return true;
diff --git a/llvm/test/CodeGen/AGX/float-misc.ll b/llvm/test/CodeGen/AGX/float-misc.ll
index 09376a81ada83..d812b09dc90fc 100644
--- a/llvm/test/CodeGen/AGX/float-misc.ll
+++ b/llvm/test/CodeGen/AGX/float-misc.ll
@@ -82,3 +82,16 @@ declare <2 x float> @llvm.experimental.constrained.fadd.v2f32(<2 x float>, <2 x
attributes #0 = { denormal_fpenv(preservesign) }
attributes #1 = { strictfp }
+
+; With ieee-fp32, 2^x is 2^f * 2^n with n = floor(x), scaled exactly, and
+; sin x is x for tiny x.
+; STRICT-LABEL: @exp2_sin(
+; STRICT: call float @llvm.floor.f32
+; STRICT: call float @llvm.exp2.f32
+; STRICT: fcmp olt float {{.*}}, f0x39800000
+define agx_kernel void @exp2_sin(ptr addrspace(1) %o, float %a) {
+ %e = call float @llvm.exp2.f32(float %a)
+ %s = call float @llvm.sin.f32(float %e)
+ store float %s, ptr addrspace(1) %o
+ ret void
+}
>From a3e07e1a1d34863056f659f3b1fc5650454e3343 Mon Sep 17 00:00:00 2001
From: Claude <noreply at anthropic.com>
Date: Mon, 19 Oct 2026 01:25:43 +0000
Subject: [PATCH 20/22] [AGX] Lower llvm.agx.simdgroup.matrix.fma
The 8x8 SIMD-group matrix multiply-accumulate intrinsic had no lowering.
AGXCodeGenPrepare now computes it with shuffles and fused multiply-adds,
with a fixed fragment layout (documented in AGXUsage.rst). Checked on the
functional simulator for float and half on G13 and G15 against a
reference.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
Claude-Session: https://claude.ai/code/session_01E56KEecg6edNaS23myRWER
---
llvm/docs/AGXUsage.rst | 13 +++++
llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp | 60 +++++++++++++++++++++++
llvm/test/CodeGen/AGX/matrix-fma.ll | 20 ++++++++
3 files changed, 93 insertions(+)
create mode 100644 llvm/test/CodeGen/AGX/matrix-fma.ll
diff --git a/llvm/docs/AGXUsage.rst b/llvm/docs/AGXUsage.rst
index 03eddcc0c0d4a..f86c2ca36b502 100644
--- a/llvm/docs/AGXUsage.rst
+++ b/llvm/docs/AGXUsage.rst
@@ -428,6 +428,19 @@ exact zero sum that it requires. An operation whose denormal mode differs
from its function's flushes its operands and result explicitly. The software
paths do not depend on the ``ieee-fp32`` feature.
+SIMD-group matrices
+-------------------
+
+``llvm.agx.simdgroup.matrix.fma(a, b, c)`` computes D = A * B + C on 8x8
+matrices spread over the SIMD-group, of ``half``, ``bfloat`` or ``float``
+elements. Thread t holds the elements (r, c) and (r, c + 1) of each matrix
+in a ``<2 x T>``, with r = 4 * (t / 16) + (t / 2) % 4 and
+c = 4 * ((t / 8) % 2) + 2 * (t % 2). It is computed with shuffles (each
+thread gathers its row of A and its two columns of B, 24 shuffles; four times
+as many on subtargets with ``quad-shuffle-index-merge``) and 16 fused
+multiply-adds per thread, in order of k starting from C; the matrix unit of
+Apple9 GPUs is not used yet.
+
SIMD-group shuffles
-------------------
diff --git a/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp b/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp
index fbe950ef51d02..49d22ba81d731 100644
--- a/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AGX/AGXCodeGenPrepare.cpp
@@ -179,6 +179,7 @@ class AGXCodeGenPrepare : public FunctionPass {
bool unifyExits(Function &F);
bool lowerConstrainedFP(Function &F);
bool lowerDivergentShuffles(Function &F, const UniformityInfo &UI);
+ bool lowerMatrixFMA(Function &F);
bool flushHalfDenormals(Function &F);
bool flushFloatMinMax(Function &F);
bool diagnoseDoubles(Function &F);
@@ -508,6 +509,64 @@ static bool isFPArithmetic(const Instruction &I) {
static bool isHalf(const Type *Ty) { return Ty->getScalarType()->isHalfTy(); }
+/// llvm.agx.simdgroup.matrix.fma: D = A * B + C on 8x8 matrices spread over
+/// the SIMD-group. Thread t holds the elements (r, c) and (r, c + 1) of each
+/// matrix, with r = 4 * (t / 16) + (t / 2) % 4 and c = 4 * ((t / 8) % 2) +
+/// 2 * (t % 2). Each thread gathers its row of A and its two columns of B
+/// with shuffles and accumulates in order of k with fused multiply-adds.
+bool AGXCodeGenPrepare::lowerMatrixFMA(Function &F) {
+ SmallVector<IntrinsicInst *, 2> Calls;
+ for (Instruction &I : instructions(F))
+ if (auto *II = dyn_cast<IntrinsicInst>(&I);
+ II && II->getIntrinsicID() == Intrinsic::agx_simdgroup_matrix_fma)
+ Calls.push_back(II);
+ for (IntrinsicInst *II : Calls) {
+ IRBuilder<> B(II);
+ Type *VT = II->getType();
+ Value *A = II->getArgOperand(0), *Bm = II->getArgOperand(1),
+ *C = II->getArgOperand(2);
+ Value *T = B.CreateIntrinsic(Intrinsic::agx_thread_index_in_simdgroup, {});
+ auto Bits = [&](Value *V, unsigned Shift, unsigned Mask) {
+ return B.CreateAnd(B.CreateLShr(V, Shift), Mask);
+ };
+ // The thread holding the elements 2j and 2j + 1 of row r.
+ auto Holder = [&](Value *R, Value *J) {
+ Value *L = B.CreateShl(B.CreateLShr(R, 2), 4);
+ L = B.CreateOr(L, B.CreateShl(B.CreateLShr(J, 1), 3));
+ L = B.CreateOr(L, B.CreateShl(B.CreateAnd(R, 3), 1));
+ return B.CreateOr(L, B.CreateAnd(J, 1));
+ };
+ auto Shuffle = [&](Value *V, unsigned Elt, Value *Lane) {
+ Value *X = B.CreateExtractElement(V, Elt);
+ return B.CreateIntrinsic(Intrinsic::agx_simd_shuffle, {X->getType()}, {X, Lane});
+ };
+ Value *Row = B.CreateOr(B.CreateShl(Bits(T, 4, 1), 2), Bits(T, 1, 3));
+ Value *ColPair = B.CreateOr(B.CreateShl(Bits(T, 3, 1), 1), Bits(T, 0, 1));
+ SmallVector<Value *, 8> RowA, ColB0, ColB1;
+ for (unsigned J = 0; J < 4; ++J) {
+ Value *Lane = Holder(Row, B.getInt32(J));
+ RowA.push_back(Shuffle(A, 0, Lane));
+ RowA.push_back(Shuffle(A, 1, Lane));
+ }
+ for (unsigned K = 0; K < 8; ++K) {
+ Value *Lane = Holder(B.getInt32(K), ColPair);
+ ColB0.push_back(Shuffle(Bm, 0, Lane));
+ ColB1.push_back(Shuffle(Bm, 1, Lane));
+ }
+ Value *D0 = B.CreateExtractElement(C, uint64_t(0));
+ Value *D1 = B.CreateExtractElement(C, 1);
+ for (unsigned K = 0; K < 8; ++K) {
+ D0 = B.CreateIntrinsic(Intrinsic::fma, {D0->getType()}, {RowA[K], ColB0[K], D0});
+ D1 = B.CreateIntrinsic(Intrinsic::fma, {D1->getType()}, {RowA[K], ColB1[K], D1});
+ }
+ Value *R = B.CreateInsertElement(PoisonValue::get(VT), D0, uint64_t(0));
+ R = B.CreateInsertElement(R, D1, 1);
+ II->replaceAllUsesWith(R);
+ II->eraseFromParent();
+ }
+ return !Calls.empty();
+}
+
/// The hardware minimum and maximum return a denormal operand as it is, also
/// where float denormals are flushed: flush their results.
bool AGXCodeGenPrepare::flushFloatMinMax(Function &F) {
@@ -667,6 +726,7 @@ bool AGXCodeGenPrepare::runOnFunction(Function &F) {
getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
const auto &ST = TM.getSubtarget<AGXSubtarget>(F);
bool Changed = diagnoseDoubles(F);
+ Changed |= lowerMatrixFMA(F);
if (ST.hasQuadShuffleIndexMerge())
Changed |= lowerDivergentShuffles(
F, getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo());
diff --git a/llvm/test/CodeGen/AGX/matrix-fma.ll b/llvm/test/CodeGen/AGX/matrix-fma.ll
new file mode 100644
index 0000000000000..bc31e472b7bcc
--- /dev/null
+++ b/llvm/test/CodeGen/AGX/matrix-fma.ll
@@ -0,0 +1,20 @@
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -o /dev/null -print-after=agx-codegenprepare < %s 2>&1 | FileCheck %s
+; RUN: llc -mtriple=agx -mcpu=agx-g13g -verify-machineinstrs < %s | FileCheck %s --check-prefix=ASM
+
+; The 8x8 SIMD-group matrix multiply-accumulate gathers the row of A and the
+; two columns of B of each thread with 24 shuffles and accumulates with 16
+; fused multiply-adds.
+; CHECK-LABEL: @mma(
+; CHECK-NOT: llvm.agx.simdgroup.matrix.fma
+; CHECK-COUNT-24: call float @llvm.agx.simd.shuffle.f32
+; CHECK-COUNT-16: call float @llvm.fma.f32
+; ASM-LABEL: mma:
+; ASM: simd_shuffle
+; ASM: fmadd32
+define agx_kernel void @mma(ptr addrspace(1) %o, <2 x float> %a, <2 x float> %b, <2 x float> %c) {
+ %r = call <2 x float> @llvm.agx.simdgroup.matrix.fma.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c)
+ store <2 x float> %r, ptr addrspace(1) %o
+ ret void
+}
+
+declare <2 x float> @llvm.agx.simdgroup.matrix.fma.v2f32(<2 x float>, <2 x float>, <2 x float>)
>From 35a770830f566817521bd20a6bc1b1d14cc0dbe7 Mon Sep 17 00:00:00 2001
From: Claude <noreply at anthropic.com>
Date: Mon, 19 Oct 2026 01:52:50 +0000
Subject: [PATCH 21/22] [mlir][AGX] Add matrix, memory barrier and floating
point mode operations, and end-to-end tests
- agx: agx.mma (8x8 SIMD-group matrix multiply-accumulate, like
amdgpu.mfma), agx.memory_barrier (threadgroup or device scope) and
agx.set_fpmode / agx.get_fpmode, lowered by -convert-agx-to-agxir.
- agxir: core.id, active.thread.index.in.quadgroup,
total.active.threads.in.quadgroup, set.fpmode, get.fpmode,
simdgroup.matrix.fma and rsqrt.special wrap the corresponding
llvm.agx intrinsics.
- test/Integration/Dialect/AGX: kernels converted from the gpu and agx
dialects, compiled with llc and run on the functional AGX simulator
(agx_run, from AGX_RUN or the PATH) with run-agx.py: a SIMD-group
reduction, a shuffle and a ballot, the matrix multiply-accumulate, and
software double precision.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
Claude-Session: https://claude.ai/code/session_01E56KEecg6edNaS23myRWER
---
mlir/include/mlir/Dialect/AGX/IR/AGX.td | 76 +++++++++++
mlir/include/mlir/Dialect/LLVMIR/AGXIROps.td | 63 ++++++++++
mlir/lib/Conversion/AGXToAGXIR/AGXToAGXIR.cpp | 54 +++++++-
.../Conversion/AGXToAGXIR/agx-to-agxir.mlir | 25 ++++
mlir/test/Dialect/AGX/invalid.mlir | 8 ++
mlir/test/Dialect/AGX/ops.mlir | 17 +++
.../Dialect/AGX/Inputs/identity-fragments.txt | 1 +
mlir/test/Integration/Dialect/AGX/f64.mlir | 25 ++++
.../Integration/Dialect/AGX/lit.local.cfg | 24 ++++
mlir/test/Integration/Dialect/AGX/mma.mlir | 21 ++++
mlir/test/Integration/Dialect/AGX/run-agx.py | 118 ++++++++++++++++++
.../Dialect/AGX/shuffle-ballot.mlir | 25 ++++
.../Dialect/AGX/subgroup-reduce.mlir | 20 +++
mlir/test/Target/LLVMIR/agxir.mlir | 20 +++
14 files changed, 496 insertions(+), 1 deletion(-)
create mode 100644 mlir/test/Integration/Dialect/AGX/Inputs/identity-fragments.txt
create mode 100644 mlir/test/Integration/Dialect/AGX/f64.mlir
create mode 100644 mlir/test/Integration/Dialect/AGX/lit.local.cfg
create mode 100644 mlir/test/Integration/Dialect/AGX/mma.mlir
create mode 100644 mlir/test/Integration/Dialect/AGX/run-agx.py
create mode 100644 mlir/test/Integration/Dialect/AGX/shuffle-ballot.mlir
create mode 100644 mlir/test/Integration/Dialect/AGX/subgroup-reduce.mlir
diff --git a/mlir/include/mlir/Dialect/AGX/IR/AGX.td b/mlir/include/mlir/Dialect/AGX/IR/AGX.td
index af93206b4e3cc..2b08d270ee90f 100644
--- a/mlir/include/mlir/Dialect/AGX/IR/AGX.td
+++ b/mlir/include/mlir/Dialect/AGX/IR/AGX.td
@@ -56,6 +56,19 @@ def AGX_ScopeAttr : EnumAttr<AGX_Dialect, AGX_Scope, "scope"> {
let assemblyFormat = "$value";
}
+def AGX_MemoryScope : I32EnumAttr<"MemoryScope",
+ "The threads a memory barrier orders accesses for", [
+ I32EnumAttrCase<"threadgroup", 0>,
+ I32EnumAttrCase<"device", 1>
+ ]> {
+ let genSpecializedAttr = 0;
+ let cppNamespace = "::mlir::agx";
+}
+def AGX_MemoryScopeAttr : EnumAttr<AGX_Dialect, AGX_MemoryScope,
+ "memory_scope"> {
+ let assemblyFormat = "$value";
+}
+
def AGX_ShuffleMode : I32EnumAttr<"ShuffleMode", "Lane selection of a shuffle",
[
I32EnumAttrCase<"idx", 0>,
@@ -186,4 +199,67 @@ def AGX_BallotOp : AGX_Op<"ballot"> {
let assemblyFormat = "$predicate (`scope` $scope^)? attr-dict `:` type($result)";
}
+def AGX_MemoryBarrierOp : AGX_Op<"memory_barrier"> {
+ let summary = "Order the memory accesses of the thread";
+ let description = [{
+ Makes the memory accesses of the thread before the barrier visible to
+ the other threads (of the threadgroup, or of the device) before those
+ after it, like `__threadfence_block()` and `__threadfence()`.
+
+ ```mlir
+ agx.memory_barrier
+ agx.memory_barrier scope threadgroup
+ ```
+ }];
+ let arguments = (ins DefaultValuedAttr<AGX_MemoryScopeAttr,
+ "::mlir::agx::MemoryScope::device">:$scope);
+ let assemblyFormat = "(`scope` $scope^)? attr-dict";
+}
+
+def AGX_MatrixFragment : VectorOfLengthAndType<[2], [F16, BF16, F32]>;
+
+def AGX_MMAOp : AGX_Op<"mma", [AllTypesMatch<["a", "b", "c", "result"]>]> {
+ let summary = "8x8 matrix multiply-accumulate over the SIMD-group";
+ let description = [{
+ Computes D = A * B + C on 8x8 matrices whose elements are spread over
+ the threads of the SIMD-group, like the MFMA operations of `amdgpu`.
+ Thread t holds the elements (r, c) and (r, c + 1) of each matrix, with
+ r = 4 * (t / 16) + (t / 2) % 4 and c = 4 * ((t / 8) % 2) + 2 * (t % 2).
+ The products are accumulated in order of k with fused multiply-adds,
+ starting from C.
+
+ ```mlir
+ %d = agx.mma %a * %b + %c : vector<2xf32>
+ ```
+ }];
+ let arguments = (ins AGX_MatrixFragment:$a, AGX_MatrixFragment:$b,
+ AGX_MatrixFragment:$c);
+ let results = (outs AGX_MatrixFragment:$result);
+ let assemblyFormat = "$a `*` $b `+` $c attr-dict `:` type($result)";
+}
+
+def AGX_SetFPModeOp : AGX_Op<"set_fpmode"> {
+ let summary = "Set the floating point mode";
+ let description = [{
+ Sets the rounding directions and denormal modes of the following
+ constrained floating point operations (with dynamic rounding): bits 0-1
+ are the float rounding direction (0 toward zero, 1 to nearest even, 2
+ up, 3 down), bits 2-3 the half and double one, bit 4 flushes float
+ denormals and bit 5 half and double denormals. The hardware rounds to
+ nearest even only: the other directions are computed in software.
+
+ ```mlir
+ agx.set_fpmode %mode : i32
+ ```
+ }];
+ let arguments = (ins I32:$mode);
+ let assemblyFormat = "$mode attr-dict `:` type($mode)";
+}
+
+def AGX_GetFPModeOp : AGX_Op<"get_fpmode"> {
+ let summary = "Read the floating point mode";
+ let results = (outs I32:$mode);
+ let assemblyFormat = "attr-dict `:` type($mode)";
+}
+
#endif // AGX
diff --git a/mlir/include/mlir/Dialect/LLVMIR/AGXIROps.td b/mlir/include/mlir/Dialect/LLVMIR/AGXIROps.td
index bffc5204f7c35..3f0a98d6fe017 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/AGXIROps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/AGXIROps.td
@@ -130,6 +130,14 @@ def AGXIR_ActiveThreadIndexInSIMDGroupOp
: AGXIR_SpecialRegisterOp<"active.thread.index.in.simdgroup", []>;
def AGXIR_TotalActiveThreadsInSIMDGroupOp
: AGXIR_SpecialRegisterOp<"total.active.threads.in.simdgroup", []>;
+def AGXIR_ActiveThreadIndexInQuadgroupOp
+ : AGXIR_SpecialRegisterOp<"active.thread.index.in.quadgroup", []>;
+def AGXIR_TotalActiveThreadsInQuadgroupOp
+ : AGXIR_SpecialRegisterOp<"total.active.threads.in.quadgroup", []>;
+
+def AGXIR_CoreIdOp : AGXIR_SpecialRegisterOp<"core.id", []> {
+ let summary = "Index of the GPU core that runs the thread";
+}
//===----------------------------------------------------------------------===//
// Synchronization
@@ -260,6 +268,57 @@ def AGXIR_SIMDPrefixOp : AGXIR_ReduceOp<"simd.prefix"> {
def AGXIR_QuadReduceOp : AGXIR_ReduceOp<"quad.reduce">;
def AGXIR_QuadPrefixOp : AGXIR_ReduceOp<"quad.prefix">;
+//===----------------------------------------------------------------------===//
+// Floating point mode
+//===----------------------------------------------------------------------===//
+
+def AGXIR_SetFPModeOp : AGXIR_IntrOp<"set.fpmode", [], [], [], 0>,
+ Arguments<(ins I32:$mode)> {
+ let summary = "Set the emulated floating point mode register";
+ let description = [{
+ Bits 0-1: float rounding (0 toward zero, 1 to nearest even, 2 up, 3
+ down), bits 2-3: half and double rounding, bit 4: float denormals
+ flushed, bit 5: half and double denormals flushed. It applies to the
+ constrained floating point operations with dynamic rounding.
+
+ ```mlir
+ agxir.set.fpmode %mode
+ ```
+ }];
+ let assemblyFormat = "$mode attr-dict";
+}
+
+def AGXIR_GetFPModeOp : AGXIR_IntrOp<"get.fpmode", [], [], [], 1> {
+ let summary = "Read the emulated floating point mode register";
+ let results = (outs I32:$res);
+ let assemblyFormat = "attr-dict `:` type($res)";
+}
+
+//===----------------------------------------------------------------------===//
+// SIMD-group matrices
+//===----------------------------------------------------------------------===//
+
+def AGXIR_MatrixFragment : VectorOfLengthAndType<[2], [F16, BF16, F32]>;
+
+def AGXIR_SIMDGroupMatrixFMAOp
+ : AGXIR_IntrOp<"simdgroup.matrix.fma", [0], [],
+ [AllTypesMatch<["res", "a", "b", "c"]>], 1>,
+ Arguments<(ins AGXIR_MatrixFragment:$a, AGXIR_MatrixFragment:$b,
+ AGXIR_MatrixFragment:$c)> {
+ let summary = "8x8 matrix multiply-accumulate over the SIMD-group";
+ let description = [{
+ D = A * B + C on 8x8 matrices; thread t holds the elements (r, c) and
+ (r, c + 1) with r = 4 * (t / 16) + (t / 2) % 4 and
+ c = 4 * ((t / 8) % 2) + 2 * (t % 2).
+
+ ```mlir
+ %d = agxir.simdgroup.matrix.fma %a, %b, %c : vector<2xf32>
+ ```
+ }];
+ let results = (outs AGXIR_MatrixFragment:$res);
+ let assemblyFormat = "$a `,` $b `,` $c attr-dict `:` type($res)";
+}
+
//===----------------------------------------------------------------------===//
// Special functions
//===----------------------------------------------------------------------===//
@@ -278,6 +337,10 @@ def AGXIR_RcpOp : AGXIR_FloatUnaryOp<"rcp"> {
def AGXIR_RsqrtOp : AGXIR_FloatUnaryOp<"rsqrt"> {
let summary = "Hardware reciprocal square root";
}
+def AGXIR_RsqrtSpecialOp : AGXIR_FloatUnaryOp<"rsqrt.special"> {
+ let summary = "Hardware reciprocal square root, with special cases so "
+ "that rsqrt.special(x) * x == sqrt(x)";
+}
def AGXIR_DfdxOp : AGXIR_FloatUnaryOp<"dfdx", []> {
let summary = "Derivative across the quad in x";
}
diff --git a/mlir/lib/Conversion/AGXToAGXIR/AGXToAGXIR.cpp b/mlir/lib/Conversion/AGXToAGXIR/AGXToAGXIR.cpp
index 046a6c01498e7..377974a6d4663 100644
--- a/mlir/lib/Conversion/AGXToAGXIR/AGXToAGXIR.cpp
+++ b/mlir/lib/Conversion/AGXToAGXIR/AGXToAGXIR.cpp
@@ -229,6 +229,56 @@ struct BallotOpLowering : ConvertOpToLLVMPattern<BallotOp> {
}
};
+struct MemoryBarrierOpLowering : ConvertOpToLLVMPattern<MemoryBarrierOp> {
+ using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
+
+ LogicalResult
+ matchAndRewrite(MemoryBarrierOp op, OpAdaptor adaptor,
+ ConversionPatternRewriter &rewriter) const override {
+ // A sequentially consistent fence, which the backend lowers to a memory
+ // barrier of its scope.
+ StringRef scope = op.getScope() == MemoryScope::threadgroup ? "workgroup" : "";
+ rewriter.replaceOpWithNewOp<LLVM::FenceOp>(
+ op, LLVM::AtomicOrdering::seq_cst, scope);
+ return success();
+ }
+};
+
+struct MMAOpLowering : ConvertOpToLLVMPattern<MMAOp> {
+ using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
+
+ LogicalResult
+ matchAndRewrite(MMAOp op, OpAdaptor adaptor,
+ ConversionPatternRewriter &rewriter) const override {
+ rewriter.replaceOpWithNewOp<AGXIR::SIMDGroupMatrixFMAOp>(
+ op, adaptor.getA().getType(), adaptor.getA(), adaptor.getB(),
+ adaptor.getC());
+ return success();
+ }
+};
+
+struct SetFPModeOpLowering : ConvertOpToLLVMPattern<SetFPModeOp> {
+ using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
+
+ LogicalResult
+ matchAndRewrite(SetFPModeOp op, OpAdaptor adaptor,
+ ConversionPatternRewriter &rewriter) const override {
+ rewriter.replaceOpWithNewOp<AGXIR::SetFPModeOp>(op, adaptor.getMode());
+ return success();
+ }
+};
+
+struct GetFPModeOpLowering : ConvertOpToLLVMPattern<GetFPModeOp> {
+ using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
+
+ LogicalResult
+ matchAndRewrite(GetFPModeOp op, OpAdaptor adaptor,
+ ConversionPatternRewriter &rewriter) const override {
+ rewriter.replaceOpWithNewOp<AGXIR::GetFPModeOp>(op, rewriter.getI32Type());
+ return success();
+ }
+};
+
struct ConvertAGXToAGXIRPass
: public impl::ConvertAGXToAGXIRPassBase<ConvertAGXToAGXIRPass> {
using Base::Base;
@@ -251,6 +301,8 @@ struct ConvertAGXToAGXIRPass
void mlir::populateAGXToAGXIRConversionPatterns(
const LLVMTypeConverter &converter, RewritePatternSet &patterns) {
- patterns.add<ThreadgroupBarrierOpLowering, ShuffleOpLowering,
+ patterns.add<ThreadgroupBarrierOpLowering, MemoryBarrierOpLowering,
+ MMAOpLowering, SetFPModeOpLowering, GetFPModeOpLowering,
+ ShuffleOpLowering,
ReduceOpLowering, BallotOpLowering>(converter);
}
diff --git a/mlir/test/Conversion/AGXToAGXIR/agx-to-agxir.mlir b/mlir/test/Conversion/AGXToAGXIR/agx-to-agxir.mlir
index 993d91d7c78a9..2a741a2e74e01 100644
--- a/mlir/test/Conversion/AGXToAGXIR/agx-to-agxir.mlir
+++ b/mlir/test/Conversion/AGXToAGXIR/agx-to-agxir.mlir
@@ -54,3 +54,28 @@ func.func @ballot(%p: i1) -> (i32, i32) {
%1 = agx.ballot %p scope quad : i32
return %0, %1 : i32, i32
}
+
+// CHECK-LABEL: @memory_barriers
+func.func @memory_barriers() {
+ // CHECK: llvm.fence seq_cst
+ agx.memory_barrier
+ // CHECK: llvm.fence syncscope("workgroup") seq_cst
+ agx.memory_barrier scope threadgroup
+ return
+}
+
+// CHECK-LABEL: @mma
+func.func @mma(%a: vector<2xf32>, %b: vector<2xf32>, %c: vector<2xf32>) -> vector<2xf32> {
+ // CHECK: agxir.simdgroup.matrix.fma %{{.*}}, %{{.*}}, %{{.*}} : vector<2xf32>
+ %0 = agx.mma %a * %b + %c : vector<2xf32>
+ return %0 : vector<2xf32>
+}
+
+// CHECK-LABEL: @fpmode
+func.func @fpmode(%m: i32) -> i32 {
+ // CHECK: agxir.set.fpmode %{{.*}}
+ agx.set_fpmode %m : i32
+ // CHECK: agxir.get.fpmode : i32
+ %0 = agx.get_fpmode : i32
+ return %0 : i32
+}
diff --git a/mlir/test/Dialect/AGX/invalid.mlir b/mlir/test/Dialect/AGX/invalid.mlir
index a8dff62adc702..431bb9be61761 100644
--- a/mlir/test/Dialect/AGX/invalid.mlir
+++ b/mlir/test/Dialect/AGX/invalid.mlir
@@ -29,3 +29,11 @@ func.func @reduce_int_minnumf(%v: i32) -> i32 {
%0 = agx.reduce minnumf %v : i32
return %0 : i32
}
+
+// -----
+
+func.func @mma_shape(%a: vector<4xf32>) -> vector<4xf32> {
+ // expected-error @+1 {{operand #0 must be vector of 16-bit float or bfloat16 type or 32-bit float values of length 2}}
+ %0 = agx.mma %a * %a + %a : vector<4xf32>
+ return %0 : vector<4xf32>
+}
diff --git a/mlir/test/Dialect/AGX/ops.mlir b/mlir/test/Dialect/AGX/ops.mlir
index ff783d03903f9..66206a188ff86 100644
--- a/mlir/test/Dialect/AGX/ops.mlir
+++ b/mlir/test/Dialect/AGX/ops.mlir
@@ -35,3 +35,20 @@ func.func @misc(%p: i1) -> i32 {
%0 = agx.ballot %p scope quad : i32
return %0 : i32
}
+
+// CHECK-LABEL: @matrices_and_modes
+func.func @matrices_and_modes(%a: vector<2xf32>, %h: vector<2xf16>, %m: i32) -> (vector<2xf32>, vector<2xf16>, i32) {
+ // CHECK: agx.mma %{{.*}} * %{{.*}} + %{{.*}} : vector<2xf32>
+ %0 = agx.mma %a * %a + %a : vector<2xf32>
+ // CHECK: agx.mma %{{.*}} * %{{.*}} + %{{.*}} : vector<2xf16>
+ %1 = agx.mma %h * %h + %h : vector<2xf16>
+ // CHECK: agx.memory_barrier{{$}}
+ agx.memory_barrier
+ // CHECK: agx.memory_barrier scope threadgroup
+ agx.memory_barrier scope threadgroup
+ // CHECK: agx.set_fpmode %{{.*}} : i32
+ agx.set_fpmode %m : i32
+ // CHECK: agx.get_fpmode : i32
+ %2 = agx.get_fpmode : i32
+ return %0, %1, %2 : vector<2xf32>, vector<2xf16>, i32
+}
diff --git a/mlir/test/Integration/Dialect/AGX/Inputs/identity-fragments.txt b/mlir/test/Integration/Dialect/AGX/Inputs/identity-fragments.txt
new file mode 100644
index 0000000000000..e39ecd2f8ceee
--- /dev/null
+++ b/mlir/test/Integration/Dialect/AGX/Inputs/identity-fragments.txt
@@ -0,0 +1 @@
+1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1
diff --git a/mlir/test/Integration/Dialect/AGX/f64.mlir b/mlir/test/Integration/Dialect/AGX/f64.mlir
new file mode 100644
index 0000000000000..a14158c2cd35f
--- /dev/null
+++ b/mlir/test/Integration/Dialect/AGX/f64.mlir
@@ -0,0 +1,25 @@
+// RUN: mlir-opt %s -convert-gpu-to-agxir='use-bare-ptr-memref-call-conv=1' \
+// RUN: -reconcile-unrealized-casts -o %t.mlir
+// RUN: %run_agx --grid 4 --wg 4 %t.mlir f64:1,0.1,1e308,3 f64:3,0.2,10,-0.0 f64*4 i32*4 | FileCheck %s
+
+// Double precision arithmetic is computed in software: a / b + sqrt(a),
+// with large values, a division by -0, and comparisons.
+// CHECK: 2: 1.33333 0.816228 1e+307 -inf
+// CHECK: 3: 0 0 1 1
+
+gpu.module @kernels {
+ gpu.func @f64(%a: memref<4xf64, 1>, %b: memref<4xf64, 1>, %out: memref<4xf64, 1>,
+ %cmp: memref<4xi32, 1>) kernel {
+ %i = gpu.thread_id x
+ %x = memref.load %a[%i] : memref<4xf64, 1>
+ %y = memref.load %b[%i] : memref<4xf64, 1>
+ %q = arith.divf %x, %y : f64
+ %s = math.sqrt %x : f64
+ %r = arith.addf %q, %s : f64
+ memref.store %r, %out[%i] : memref<4xf64, 1>
+ %gt = arith.cmpf ogt, %x, %y : f64
+ %z = arith.extui %gt : i1 to i32
+ memref.store %z, %cmp[%i] : memref<4xi32, 1>
+ gpu.return
+ }
+}
diff --git a/mlir/test/Integration/Dialect/AGX/lit.local.cfg b/mlir/test/Integration/Dialect/AGX/lit.local.cfg
new file mode 100644
index 0000000000000..26e9bd0183bbe
--- /dev/null
+++ b/mlir/test/Integration/Dialect/AGX/lit.local.cfg
@@ -0,0 +1,24 @@
+import os
+import shutil
+
+# The tests run kernels on the functional AGX simulator (agx_run of
+# AGX-Performance-Simulator), found through AGX_RUN or on the PATH.
+agx_run = os.environ.get("AGX_RUN") or shutil.which("agx_run")
+if not agx_run:
+ config.unsupported = True
+else:
+ config.substitutions.append(
+ (
+ "%run_agx",
+ "%s %s --llc %s --objcopy %s --mlir-translate %s --agx-run %s"
+ % (
+ config.python_executable,
+ os.path.join(os.path.dirname(__file__), "run-agx.py"),
+ os.path.join(config.llvm_tools_dir, "llc"),
+ os.path.join(config.llvm_tools_dir, "llvm-objcopy"),
+ os.path.join(config.mlir_tools_dir, "mlir-translate"),
+ agx_run,
+ ),
+ )
+ )
+config.excludes = ["run-agx.py"]
diff --git a/mlir/test/Integration/Dialect/AGX/mma.mlir b/mlir/test/Integration/Dialect/AGX/mma.mlir
new file mode 100644
index 0000000000000..b8239eb9522a4
--- /dev/null
+++ b/mlir/test/Integration/Dialect/AGX/mma.mlir
@@ -0,0 +1,21 @@
+// RUN: mlir-opt %s -convert-agx-to-agxir \
+// RUN: -convert-gpu-to-agxir='use-bare-ptr-memref-call-conv=1' \
+// RUN: -reconcile-unrealized-casts -o %t.mlir
+// RUN: %run_agx --grid 32 --wg 32 %t.mlir f32:@%S/Inputs/identity-fragments.txt \
+// RUN: f32:iota*64 f32*64 | FileCheck %s
+
+// I * B + B = 2 B: each thread's two elements, in fragment order, are
+// twice the inputs it holds.
+// CHECK: 2: 0 2 4 6 8 10 {{.*}} 122 124 126{{$}}
+
+gpu.module @kernels {
+ gpu.func @mma(%a: memref<32xvector<2xf32>, 1>, %b: memref<32xvector<2xf32>, 1>,
+ %out: memref<32xvector<2xf32>, 1>) kernel {
+ %t = gpu.lane_id
+ %x = memref.load %a[%t] : memref<32xvector<2xf32>, 1>
+ %y = memref.load %b[%t] : memref<32xvector<2xf32>, 1>
+ %d = agx.mma %x * %y + %y : vector<2xf32>
+ memref.store %d, %out[%t] : memref<32xvector<2xf32>, 1>
+ gpu.return
+ }
+}
diff --git a/mlir/test/Integration/Dialect/AGX/run-agx.py b/mlir/test/Integration/Dialect/AGX/run-agx.py
new file mode 100644
index 0000000000000..e4b7bc97532c4
--- /dev/null
+++ b/mlir/test/Integration/Dialect/AGX/run-agx.py
@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+"""Compile an AGX kernel (LLVM IR) and run it on the functional simulator.
+
+Usage:
+ run-agx.py --llc LLC --objcopy OBJCOPY --agx-run AGX_RUN
+ [--mlir-translate MLIR_TRANSLATE] kernel.{ll,mlir} --grid N --wg N ARG...
+
+A .mlir input is the output of the conversion to the LLVM and AGXIR
+dialects: the functions of its gpu.module are translated to LLVM IR.
+
+Each ARG is a kernel argument in order: `TYPE:V,V,...` is a buffer
+initialized with the values (TYPE is i32, u32, i64, f16, f32 or f64),
+`TYPE:iota*N` one with 0, 1, ..., N - 1, `TYPE:@FILE` one with the values in
+FILE, and `TYPE*N` an output buffer of N
+elements. Arguments are passed as pointers
+in uniform registers (see AGXUsage.rst). The output buffers are printed one
+per line, `NAME: v v v ...`, with NAME the argument number.
+"""
+
+import argparse
+import os
+import re
+import struct
+import subprocess
+import sys
+import tempfile
+
+FMT = {'i32': 'i', 'u32': 'I', 'f16': 'e', 'f32': 'f', 'f64': 'd', 'i64': 'q'}
+
+
+def translate(mlir_translate, path, tmp):
+ """The functions of the gpu.module of an MLIR file, as LLVM IR."""
+ lines = open(path).read().splitlines()
+ start = next(i for i, l in enumerate(lines) if l.lstrip().startswith('gpu.module'))
+ depth = lines[start].count('{') - lines[start].count('}')
+ end = start
+ while depth:
+ end += 1
+ depth += lines[end].count('{') - lines[end].count('}')
+ body = lines[start + 1:end]
+ mlir = os.path.join(tmp, 'k.mlir')
+ open(mlir, 'w').write('module {\n' + '\n'.join(body) + '\n}\n')
+ ll = os.path.join(tmp, 'k.ll')
+ subprocess.run([mlir_translate, '-mlir-to-llvmir', mlir, '-o', ll], check=True)
+ return ll
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument('--llc', required=True)
+ ap.add_argument('--objcopy', required=True)
+ ap.add_argument('--agx-run', dest='agx_run', required=True)
+ ap.add_argument('--mlir-translate', dest='mlir_translate')
+ ap.add_argument('--cpu', default='agx-g13g')
+ ap.add_argument('--grid', type=int, required=True)
+ ap.add_argument('--wg', type=int, required=True)
+ ap.add_argument('ir')
+ ap.add_argument('args', nargs='*')
+ a = ap.parse_args()
+ with tempfile.TemporaryDirectory() as tmp:
+ if a.ir.endswith('.mlir'):
+ a.ir = translate(a.mlir_translate, a.ir, tmp)
+ asm = subprocess.run([a.llc, '-mtriple=agx', '-mcpu=' + a.cpu, a.ir, '-o', '-'],
+ capture_output=True, text=True, check=True).stdout
+ obj = os.path.join(tmp, 'k.o')
+ subprocess.run([a.llc, '-mtriple=agx', '-mcpu=' + a.cpu, '-filetype=obj', a.ir,
+ '-o', obj], check=True)
+ code = os.path.join(tmp, 'k.bin')
+ subprocess.run([a.objcopy, '-O', 'binary', '--only-section=.text', obj, code],
+ check=True)
+ glob = os.path.join(tmp, 'g.bin')
+ subprocess.run([a.objcopy, '-O', 'binary', '--only-section=.agx.globals', obj, glob],
+ check=False)
+ grid = int(re.search(r'threadgroups_per_grid at u(\d+)', asm).group(1))
+ gm = re.search(r'global data buffer at u(\d+)_u\d+', asm)
+ cmd = [a.agx_run, code, '--grid', str(a.grid), '--wg', str(a.wg)]
+ uniforms = []
+ outs = []
+ for i, arg in enumerate(a.args):
+ base = 0x100000 * (i + 1)
+ if '*' in arg and ':' not in arg:
+ t, n = arg.split('*')
+ size = struct.calcsize('<' + FMT[t]) * int(n)
+ f = os.path.join(tmp, 'o%d.bin' % i)
+ cmd += ['--alloc', '%#x=%d' % (base, size), '--dump', '%#x=%d:%s' % (base, size, f)]
+ outs.append((i, t, int(n), f))
+ else:
+ t, vals = arg.split(':')
+ f = os.path.join(tmp, 'i%d.bin' % i)
+ if vals.startswith('@'):
+ vals = open(vals[1:]).read().strip()
+ if vals.startswith('iota*'):
+ vals = ','.join(str(v) for v in range(int(vals[5:])))
+ vs = [float(v) if t[0] == 'f' else int(v, 0) for v in vals.split(',')]
+ open(f, 'wb').write(struct.pack('<%d%s' % (len(vs), FMT[t]), *vs))
+ cmd += ['--map', '%#x=%s' % (base, f)]
+ uniforms += [base, 0]
+ for k, v in enumerate(uniforms):
+ cmd += ['--uniform', '%d=%d' % (k, v)]
+ groups = a.grid // a.wg
+ cmd += ['--uniform', '%d=%d' % (grid, groups), '--uniform', '%d=1' % (grid + 1),
+ '--uniform', '%d=1' % (grid + 2)]
+ if gm:
+ g = os.path.join(tmp, 'g2.bin')
+ data = open(glob, 'rb').read() if os.path.exists(glob) else b''
+ open(g, 'wb').write(data + bytes(16))
+ cmd += ['--map', '%#x=%s' % (0x4000000, g), '--uniform', '%s=%d' % (gm.group(1), 0x4000000),
+ '--uniform', '%d=0' % (int(gm.group(1)) + 1)]
+ r = subprocess.run(cmd, capture_output=True, text=True)
+ if r.returncode:
+ sys.exit('simulator error: ' + r.stdout + r.stderr)
+ for i, t, n, f in outs:
+ vals = struct.unpack('<%d%s' % (n, FMT[t]), open(f, 'rb').read())
+ print('%d: %s' % (i, ' '.join(('%g' % v) if t[0] == 'f' else str(v) for v in vals)))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/mlir/test/Integration/Dialect/AGX/shuffle-ballot.mlir b/mlir/test/Integration/Dialect/AGX/shuffle-ballot.mlir
new file mode 100644
index 0000000000000..2bad1d1b50e96
--- /dev/null
+++ b/mlir/test/Integration/Dialect/AGX/shuffle-ballot.mlir
@@ -0,0 +1,25 @@
+// RUN: mlir-opt %s -convert-agx-to-agxir \
+// RUN: -convert-gpu-to-agxir='use-bare-ptr-memref-call-conv=1' \
+// RUN: -reconcile-unrealized-casts -o %t.mlir
+// RUN: %run_agx --grid 32 --wg 32 %t.mlir i32*32 i32*32 | FileCheck %s
+
+// Each thread reads the lane index of its xor-1 neighbour, and the ballot of
+// the even lanes is 0x55555555.
+// CHECK: 0: 1 0 3 2 5 4 {{.*}} 31 30{{$}}
+// CHECK: 1: 1431655765 1431655765 {{.*}} 1431655765{{$}}
+
+gpu.module @kernels {
+ gpu.func @k(%out: memref<32xi32, 1>, %mask: memref<32xi32, 1>) kernel {
+ %t = gpu.lane_id
+ %ti = arith.index_cast %t : index to i32
+ %c1 = arith.constant 1 : i32
+ %s = agx.shuffle xor %ti, %c1 : i32
+ memref.store %s, %out[%t] : memref<32xi32, 1>
+ %odd = arith.andi %ti, %c1 : i32
+ %c0 = arith.constant 0 : i32
+ %even = arith.cmpi eq, %odd, %c0 : i32
+ %m = agx.ballot %even : i32
+ memref.store %m, %mask[%t] : memref<32xi32, 1>
+ gpu.return
+ }
+}
diff --git a/mlir/test/Integration/Dialect/AGX/subgroup-reduce.mlir b/mlir/test/Integration/Dialect/AGX/subgroup-reduce.mlir
new file mode 100644
index 0000000000000..9682aeb80126d
--- /dev/null
+++ b/mlir/test/Integration/Dialect/AGX/subgroup-reduce.mlir
@@ -0,0 +1,20 @@
+// RUN: mlir-opt %s -convert-gpu-to-agxir='use-bare-ptr-memref-call-conv=1' \
+// RUN: -reconcile-unrealized-casts -o %t.mlir
+// RUN: %run_agx --grid 64 --wg 64 %t.mlir f32:iota*64 f32*64 | FileCheck %s
+
+// Each thread scales its element and the SIMD-groups sum them: the two
+// SIMD-groups of 32 threads get 2 * (0 + ... + 31) = 992 and
+// 2 * (32 + ... + 63) = 3040.
+// CHECK: 1: 992 992 {{.*}} 992 3040 {{.*}} 3040{{$}}
+
+gpu.module @kernels {
+ gpu.func @scale(%a: memref<64xf32, 1>, %out: memref<64xf32, 1>) kernel {
+ %i = gpu.thread_id x
+ %v = memref.load %a[%i] : memref<64xf32, 1>
+ %c2 = arith.constant 2.0 : f32
+ %w = arith.mulf %v, %c2 : f32
+ %r = gpu.subgroup_reduce add %w : (f32) -> f32
+ memref.store %r, %out[%i] : memref<64xf32, 1>
+ gpu.return
+ }
+}
diff --git a/mlir/test/Target/LLVMIR/agxir.mlir b/mlir/test/Target/LLVMIR/agxir.mlir
index 232f855fc16cd..80b0bf13da17a 100644
--- a/mlir/test/Target/LLVMIR/agxir.mlir
+++ b/mlir/test/Target/LLVMIR/agxir.mlir
@@ -35,3 +35,23 @@ llvm.func @kernel(%out: !llvm.ptr<1>) attributes {agxir.kernel} {
llvm.func @helper() {
llvm.return
}
+
+// CHECK-LABEL: define void @extras
+llvm.func @extras(%out: !llvm.ptr<1>, %v: vector<2xf32>, %m: i32, %x: f32) {
+ // CHECK: call i32 @llvm.agx.core.id()
+ %0 = agxir.core.id : i32
+ // CHECK: call i32 @llvm.agx.active.thread.index.in.quadgroup()
+ %1 = agxir.active.thread.index.in.quadgroup : i32
+ // CHECK: call i32 @llvm.agx.total.active.threads.in.quadgroup()
+ %2 = agxir.total.active.threads.in.quadgroup : i32
+ // CHECK: call void @llvm.agx.set.fpmode(i32 %{{.*}})
+ agxir.set.fpmode %m
+ // CHECK: call i32 @llvm.agx.get.fpmode()
+ %3 = agxir.get.fpmode : i32
+ // CHECK: call <2 x float> @llvm.agx.simdgroup.matrix.fma.v2f32(<2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}})
+ %4 = agxir.simdgroup.matrix.fma %v, %v, %v : vector<2xf32>
+ // CHECK: call float @llvm.agx.rsqrt.special.f32(float %{{.*}})
+ %5 = agxir.rsqrt.special %x : f32
+ llvm.store %4, %out : vector<2xf32>, !llvm.ptr<1>
+ llvm.return
+}
>From 234ebf7079fb4c3600b5140fcfff97bf85fe3166 Mon Sep 17 00:00:00 2001
From: Claude <noreply at anthropic.com>
Date: Mon, 19 Oct 2026 02:42:59 +0000
Subject: [PATCH 22/22] [AGX] Select 64-bit sign_extend_inreg; limit device
index shifts to 2
- sign_extend_inreg of an i64 from i8, i16 or i1 (for example sitofp of a
sign-extended byte) had no pattern: extend the low half, then to 64
bits.
- Device loads and stores folded a scaled index with shift 3. applegpu's
notes (and the functional simulator) treat the shift field value 3 as
2, so an index scaled by 8 items addressed the wrong elements; only
shifts of 0 to 2 are folded now.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
Claude-Session: https://claude.ai/code/session_01E56KEecg6edNaS23myRWER
---
llvm/lib/Target/AGX/AGXISelDAGToDAG.cpp | 3 +-
llvm/lib/Target/AGX/AGXInstructions.td | 15 ++++++
llvm/test/CodeGen/AGX/i64.ll | 65 +++++++++++++++++++++++++
3 files changed, 82 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AGX/AGXISelDAGToDAG.cpp b/llvm/lib/Target/AGX/AGXISelDAGToDAG.cpp
index c29aa7f357b20..16ff32fdfcb2a 100644
--- a/llvm/lib/Target/AGX/AGXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AGX/AGXISelDAGToDAG.cpp
@@ -308,7 +308,8 @@ bool AGXDAGToDAGISel::SelectDeviceAddr(SDValue Addr, unsigned ElemLog2,
Scale = Log2_64(C->getZExtValue());
O = O.getOperand(0);
}
- if (Scale < ElemLog2 || Scale > ElemLog2 + 3)
+ // The shift field is 0 to 2: applegpu notes that 3 may act as 2.
+ if (Scale < ElemLog2 || Scale > ElemLog2 + 2)
continue;
bool IsSExt = O.getOpcode() == ISD::SIGN_EXTEND;
if (!IsSExt && O.getOpcode() != ISD::ZERO_EXTEND)
diff --git a/llvm/lib/Target/AGX/AGXInstructions.td b/llvm/lib/Target/AGX/AGXInstructions.td
index bb3de420d5420..725751ef5c5ac 100644
--- a/llvm/lib/Target/AGX/AGXInstructions.td
+++ b/llvm/lib/Target/AGX/AGXInstructions.td
@@ -388,6 +388,21 @@ def : Pat<(i32 (sext_inreg i32:$a, i1)),
(ISUB_I32 0, (i32 0), 0, (BITOP_I32 TT_AND, 0, $a, 0, (i32 1)), 0)>;
def : Pat<(i16 (sext_inreg i16:$a, i1)),
(ISUB_I16 0, (i16 0), 0, (BITOP_I16 TT_AND, 0, $a, 0, (i16 1)), 0)>;
+// 64-bit: the low half, sign-extended as above, then to 64 bits.
+def : Pat<(i64 (sext_inreg i64:$a, i16)),
+ (IADD_I64_FROM32 MOD_SX,
+ (IADD_I32_FROM16 MOD_SX,
+ (i16 (EXTRACT_SUBREG (i32 (EXTRACT_SUBREG $a, sub0)), lo16)), 0,
+ (i16 0), 0), 0, (i32 0), 0)>;
+def : Pat<(i64 (sext_inreg i64:$a, i8)),
+ (IADD_I64_FROM32 MOD_SX,
+ (ASR_I32 0, (BFI_I32 0, (i32 0), 0, (i32 (EXTRACT_SUBREG $a, sub0)),
+ 0, (i32 24), 0), 0, (i32 24)), 0, (i32 0), 0)>;
+def : Pat<(i64 (sext_inreg i64:$a, i1)),
+ (IADD_I64_FROM32 MOD_SX,
+ (ISUB_I32 0, (i32 0), 0,
+ (BITOP_I32 TT_AND, 0, (i32 (EXTRACT_SUBREG $a, sub0)), 0, (i32 1)), 0),
+ 0, (i32 0), 0)>;
// Count leading zeros: 31 - ffs(x); ffs(0) is -1.
def : Pat<(i32 (ctlz AGXIntSrc32:$a)),
diff --git a/llvm/test/CodeGen/AGX/i64.ll b/llvm/test/CodeGen/AGX/i64.ll
index a9328aac50484..270070041beb0 100644
--- a/llvm/test/CodeGen/AGX/i64.ll
+++ b/llvm/test/CodeGen/AGX/i64.ll
@@ -174,3 +174,68 @@ define agx_kernel void @bitcount(ptr addrspace(1) %out, i64 %a) {
ret void
}
+
+
+; Sign extensions in a 64-bit register from 8 and 16 bits extend the low half,
+; then to 64 bits.
+define agx_kernel void @sext_inreg_i64(ptr addrspace(1) %p, ptr addrspace(1) %o) {
+; CHECK-LABEL: sext_inreg_i64:
+; CHECK: // uniform registers: 14 x 16-bit (threadgroups_per_grid at u4)
+; CHECK-NEXT: // general purpose registers: 10 x 32-bit
+; CHECK-NEXT: // threadgroup memory: 0 bytes
+; CHECK-NEXT: // stack size: 0 bytes
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: device_load 0, i8, x, r1l, u0_u1, 0, signed
+; CHECK-NEXT: wait 0
+; CHECK-NEXT: bitop 0101, r1, r1l, 0
+; CHECK-NEXT: iadd r2_r3, r1, 0
+; CHECK-NEXT: bfi r1, 0, r2, 24
+; CHECK-NEXT: asr r1, r1, 24
+; CHECK-NEXT: iadd r2_r3, r1.sx, 0
+; CHECK-NEXT: asr r4, r3, 31
+; CHECK-NEXT: bitop 0101, r5, r4, 0
+; CHECK-NEXT: xor r7, r3, r4
+; CHECK-NEXT: xor r6, r2, r4
+; CHECK-NEXT: isub r2_r3, r6_r7, r4_r5
+; CHECK-NEXT: ffs r1, r2
+; CHECK-NEXT: ffs r6, r3
+; CHECK-NEXT: isub r1, 31, r1
+; CHECK-NEXT: isub r6, 31, r6
+; CHECK-NEXT: iadd r1, r1, 32
+; CHECK-NEXT: icmpsel ueq, r1, r3, 0, r1, r6
+; CHECK-NEXT: and r6, r1, 31
+; CHECK-NEXT: bfeil r7, 0, r2, 1
+; CHECK-NEXT: xor r8, r6, 31
+; CHECK-NEXT: bfi r9, 0, r3, r6
+; CHECK-NEXT: bfeil r7, 0, r7, r8
+; CHECK-NEXT: and r8, r1, 32
+; CHECK-NEXT: bfi r2, 0, r2, r6
+; CHECK-NEXT: bfeil r3, 0, r8, 5
+; CHECK-NEXT: or r6, r9, r7
+; CHECK-NEXT: icmpsel ueq, r7, r3, 0, r2, 0
+; CHECK-NEXT: icmpsel ueq, r2, r3, 0, r6, r2
+; CHECK-NEXT: icmpsel ueq, r3, r7, 0, 0, 1
+; CHECK-NEXT: or r2, r2, r3
+; CHECK-NEXT: isub r1, 159, r1
+; CHECK-NEXT: convert u32_to_f, r2, r2, rte
+; CHECK-NEXT: bfi r1, 0, r1, 23
+; CHECK-NEXT: fmul32 r1, r2, r1
+; CHECK-NEXT: mov_imm r2, -2147483648
+; CHECK-NEXT: and r2, r4, r2
+; CHECK-NEXT: or r1, r1, r2
+; CHECK-NEXT: device_store 0, i32, x, r1, u2_u3, 0, signed
+; CHECK-NEXT: stop
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+; CHECK-NEXT: trap
+ %b = load i8, ptr addrspace(1) %p
+ %e = sext i8 %b to i64
+ %f = sitofp i64 %e to float
+ store float %f, ptr addrspace(1) %o
+ ret void
+}
More information about the Mlir-commits
mailing list